Add GPFS health check

Makefile.am

@@ -14,7 +14,8 @@ nobase_dist_conf_DATA = scripts/lbnl_cmd.nhc scripts/common.nhc		\
                         scripts/lbnl_fs.nhc  scripts/lbnl_hw.nhc	\
                         scripts/lbnl_job.nhc scripts/lbnl_moab.nhc	\
                         scripts/lbnl_net.nhc scripts/lbnl_nv.nhc	\
-                        scripts/lbnl_ps.nhc
+                        scripts/lbnl_ps.nhc							\
+                        scripts/osc_gpfs.nhc
 
 MAINTAINERCLEANFILES = Makefile.in aclocal.m4 configure install-sh missing
 DISTCLEANFILES = 

README.md

@@ -452,6 +452,7 @@ The table below provides a list of the configuration variables which may be used
 | MCELOG_MAX_CORRECTED_RATE | `9` | Maximum number of **corrected** MCEs allowed before `check_hw_mcelog()` returns failure |
 | MCELOG_MAX_UNCORRECTED_RATE | `0` | Maximum number of **uncorrected** MCEs allowed before `check_hw_mcelog()` returns failure |
 | MDIAG_CMD | `mdiag` | Command to use to invoke Moab's `mdiag` command (may include path) |
+| MMHEALTH | `/usr/lpp/mmfs/bin/mmhealth` | Command to use to invoke the GPFS `mmhealth` command |
 | *NAME | `nhc` | Used to populate default paths/filenames for configuration |
 | NHC_AUTH_USERS | `root nobody` | Users authorized to have arbitrary processes running on compute nodes |
 | NHC_CHECK_ALL | `0` | Forces all checks to be non-fatal.  Displays each failure message, reports total number of failed checks, and returns that number. |
@@ -771,6 +772,40 @@ _**Example**_:  `check_fs_used / 98%`
 <br />
 
 
+##### check_gpfs_health
+`check_gpfs_health [-0] [-a] [-l] [-s] [-e <action>] <component>`
+
+Checks the health of a GPFS component.  The value for _component_ must match a component reported by mmhealth.
+
+| **Check&nbsp;Option** | **Purpose** |
+| ---------------- | ----------- |
+| `-0` | Non-fatal.  Failure of this check will be ignored. |
+| `-a` | Find, report, and act on all matching components.  Default behavior is to fail check after first matching component. |
+| `-l` | Log unhealthy component (or components, if used with `-a`) to NHC log (`$LOGFILE`). |
+| `-s` | Log unhealthy component (or components, if used with `-a`) to the syslog. |
+| `-e`_`action`_ | Execute `/bin/bash -c`_`action`_ if component is NOT healthy. |
+
+_**Example**_: `check_gpfs_health NETWORK`
+
+<br />
+
+
+##### check_gpfs_verbs_status
+`check_gpfs_verbs_status [-0] [-l] [-s]`
+
+Checks that GPFS has started Verbs and is using RDMA.
+
+| **Check&nbsp;Option** | **Purpose** |
+| ---------------- | ----------- |
+| `-0` | Non-fatal.  Failure of this check will be ignored. |
+| `-l` | Log if verbs is not started to NHC log (`$LOGFILE`). |
+| `-s` | Log if verbs is not started to the syslog. |
+
+_**Example**_: `check_gpfs_verbs_status`
+
+<br />
+
+
 ##### check_hw_cpuinfo
 `check_hw_cpuinfo [sockets] [cores] [threads]`
 

scripts/osc_gpfs.nhc

@@ -0,0 +1,173 @@
+# OSC NHC - GPFS checks
+#
+# Trey Dockendorf <[email protected]>
+# 1 November 2018
+#
+
+GPFS_COMPONENT=()
+GPFS_ENTITY=()
+GPFS_STATUS=()
+MMHEALTH="${MMHEALTH:-/usr/lpp/mmfs/bin/mmhealth}"
+MMFSADM="${MMFSADM:-/usr/lpp/mmfs/bin/mmfsadm}"
+export GPFS_COMPONENT GPFS_ENTITY GPFS_STATUS MMHEALTH MMFSADM
+
+function nhc_gpfs_health_gather_data() {
+    local LINE_CNT
+    local -a FIELD
+
+    GPFS_COMPONENT=() GPFS_ENTITY=() GPFS_STATUS=()
+
+    ((LINE_CNT=0))
+    while IFS=: read -a FIELD; do
+        if [[ "${FIELD[2]}" == "HEADER" ]]; then
+            continue
+        fi
+        if [[ "${FIELD[1]}" == "Event" ]]; then
+            continue
+        fi
+        if [[ "${FIELD[9]}" != "NODE" ]]; then
+            continue
+        fi
+        GPFS_COMPONENT[$LINE_CNT]="${FIELD[7]}"
+        GPFS_ENTITY[$LINE_CNT]="${FIELD[8]}"
+        GPFS_STATUS[$LINE_CNT]="${FIELD[10]}"
+        dbg "Got GPFS health ${GPFS_COMPONENT[$LINE_CNT]} ${GPFS_ENTITY[$LINE_CNT]} ${GPFS_STATUS[$LINE_CNT]}"
+        ((LINE_CNT++))
+    done < <($MMHEALTH node show -Y)
+
+    export GPFS_COMPONENT GPFS_ENTITY GPFS_STATUS
+}
+
+# Checks GPFS health for a given component
+#   check_gpfs_health [-0] [-a] [-l] [-s] [-e <action>] <component>
+function check_gpfs_health() {
+    local NONFATAL=0 ALL=0 LOG=0 SYSLOG=0 ACTION=""
+    local THIS_COMPONENT THIS_ENTITY THIS_STATUS MSG i
+
+    if [[ ${#GPFS_COMPONENT[*]} -eq 0 ]]; then
+        nhc_gpfs_health_gather_data
+    fi
+
+    OPTIND=1
+    while getopts ":0alse:" OPTION ; do
+        case "$OPTION" in
+            0) NONFATAL=1 ;;
+            a) ALL=1 ;;
+            l) LOG=1 ;;
+            s) SYSLOG=1 ;;
+            e) ACTION="$OPTARG" ;;
+            :) die 1 "$CHECK:  Option -$OPTARG requires an argument." ; return 1 ;;
+            \?) die 1 "$CHECK:  Invalid option:  -$OPTARG" ; return 1 ;;
+        esac
+    done
+    shift $((OPTIND-1))
+    COMPONENT="$1"
+    if [[ -z "$COMPONENT" ]]; then
+        die 1 "$CHECK: Syntax error: Must provide component to check."
+    fi
+    dbg "Looking for GPFS health component \"$COMPONENT\""
+    for ((i=0; i < ${#GPFS_COMPONENT[*]}; i++)); do
+        THIS_COMPONENT="${GPFS_COMPONENT[$i]}"
+        THIS_ENTITY="${GPFS_ENTITY[$i]}"
+        THIS_STATUS="${GPFS_STATUS[$i]}"
+        dbg "CHECKING \"$THIS_COMPONENT\" vs. \"$GPFS_COMPONENT\""
+        if ! mcheck "$THIS_COMPONENT" "$COMPONENT"; then
+            continue
+        fi
+        dbg "Matching GPFS health found: $THIS_COMPONENT: entity=$THIS_ENTITY status=$THIS_STATUS"
+        if [[ "$THIS_STATUS" == "HEALTHY" ]]; then
+            continue
+        else
+            MSG="$CHECK: GPFS health for \"$THIS_COMPONENT\" is $THIS_STATUS"
+        fi
+        # We have a winner.  Or loser, as the case may be.
+        if [[ "$LOG" == "1" ]]; then
+            log $MSG
+        fi
+        if [[ "$SYSLOG" == "1" ]]; then
+            syslog $MSG
+        fi
+        if [[ "$ACTION" != "" ]]; then
+            ${SHELL:-/bin/bash} -c "$ACTION" &
+        fi
+        if [[ $ALL -ge 1 ]]; then
+            if [[ -n "$MSG" ]]; then
+                log "$MSG ($ALL)"
+            fi
+            ((ALL++))
+            continue
+        elif [[ $NONFATAL == 1 ]]; then
+            if [[ -n "$MSG" ]]; then
+                log "$MSG (non-fatal)"
+            fi
+            return 0
+        fi
+        die 1 "$MSG"
+        return 1
+    done
+    # -a (all) does not necessarily imply -0 (non-fatal).  A value of 1 for $ALL
+    # means -a was passed in but no errors were found.  2 or above is an error.
+    if [[ $ALL -gt 1 ]]; then
+        # We had at least 1 flagged process.  Fail unless we're also non-fatal.
+        if [[ $NONFATAL == 1 ]]; then
+            if [[ -n "$MSG" ]]; then
+                log "$MSG (non-fatal)"
+            fi
+            return 0
+        fi
+        ((ALL--))
+        die $ALL "$MSG (last of $ALL)"
+        return $ALL
+    fi
+    return 0
+}
+
+# Checks GPFS verbs status
+#   check_gpfs_verbs_status [-0] [-l] [-s]
+function check_gpfs_verbs_status() {
+    local NONFATAL=0 LOG=0 SYSLOG=0 MSG=''
+    local RET OUTPUT OLD_DEBUG
+
+    OPTIND=1
+    while getopts ":0ls" OPTION ; do
+        case "$OPTION" in
+            0) NONFATAL=1 ;;
+            l) LOG=1 ;;
+            s) SYSLOG=1 ;;
+            :) die 1 "$CHECK:  Option -$OPTARG requires an argument." ; return 1 ;;
+            \?) die 1 "$CHECK:  Invalid option:  -$OPTARG" ; return 1 ;;
+        esac
+    done
+    shift $((OPTIND-1))
+
+    OLD_DEBUG=$DEBUG
+    unset DEBUG
+    check_cmd_output -t ${CMD_TIMEOUT:-5} -C "$FUNCNAME" -O OUTPUT -m '/status/' $MMFSADM test verbs status
+    RET=$?
+    export DEBUG=$OLD_DEBUG
+    if [[ $RET -ne 0 ]]; then
+        return $RET
+    fi
+
+    dbg "$MMFSADM test verbs status: \"$OUTPUT\""
+
+    if [[ "$OUTPUT" == *": started" ]]; then
+        return 0
+    else
+        MSG="$FUNCNAME: GPFS verbs is not started"
+    fi
+    if [[ "$LOG" == "1" ]]; then
+        log $MSG
+    fi
+    if [[ "$SYSLOG" == "1" ]]; then
+        syslog $MSG
+    fi
+    if [[ $NONFATAL == 1 ]]; then
+        if [[ -n "$MSG" ]]; then
+            log "$MSG (non-fatal)"
+        fi
+        return 0
+    fi
+    die 1 "$MSG"
+    return 1
+}