#!/bin/bash
# faultmonitor.sh
# Rescue CD Backup & Restore Utility fault monitoring script
# version 7.2.5
# Rod Wright 1/19/2024
# Refer to CHANGELOG file for details.
# -----------------------------------------------------------------------------

CAPACITY_WARN=90
STATUS_FILE="/root/faultstatus.txt"

function checkraid() {
    # check for RAID failure
    # takes one argument, a full raid device name
    local device=$1
    if [ "$device" != "none" ]; then
        if mdadm -D $device | grep -q degraded ; then
            return 1
        else
            return 0
        fi
    else
        return 0
    fi
}

function checksmart() {
    # check for RAID member SMART failure
    # takes a list of RAID partition full device names as positional parameters
    for drive in $* ; do
        if echo $drive|grep -q nvme ; then
            nvmesmartstat=`nvme smart-log $drive|grep critical_warning|cut -d ":" -f2|sed 's/^ *//g'`
            if [ $nvmesmartstat -ne 0 ]; then return 1 ;fi
        else
            smartctl -q silent $drive
            local smartstat=$(($? & 8))
            if [ $smartstat -ne 0 ]; then return 1 ;fi
        fi
    done
    return 0
}

function checkcapacity() {
    # check for RAID usage above a threshold
    # takes two arguments, a RAID device and a percentage
    local device=$1
    local threshold=$2
    if [ "$device" != "none" ]; then
        local pctstrg=`df $device |grep $device|awk '{print $5}'`
        local pct=${pctstrg//%}
        if [ $pct -ge $threshold ] ; then
            return 1
        else
            return 0
        fi
    else
        return 0
    fi
}

function flashuid() {
    # turn flashing UID light on or off
    # takes one argument, the word "on" or "off"
    # determine if ipmi hardware is supported and return failure if not
    if ! ipmitool chassis status >/dev/null 2>&1; then return 1 ; fi
    if [ "$1" = "on" ]; then
        local ipmiarg="force"
    elif [ "$1" = "off" ]; then
        local ipmiarg="0"
    else
        # invalid argument make no change and return failure
        return 1
    fi
    ipmitool chassis identify $ipmiarg
}

function update_faultstatusfile() {
    # maintain the fault status file
    if [ "$raiddevname" != "none" ]; then
        if [ $raidfail ]; then
            local raidline="RAID Status :   \033[1;31mFAIL\033[0m"
        else
            local raidline="RAID Status :     \033[1;32mOK\033[0m"
        fi
    else
        local raidline="RAID Status :   \033[1;33mWAIT\033[0m"
    fi
    if [ $smartfail ]; then
        local smartline="Disk Status :   \033[1;31mFAIL\033[0m"
    else
        local smartline="Disk Status :     \033[1;32mOK\033[0m"
    fi
    if [ "$raiddevname" != "none" ]; then
        if [ $lowspace ]; then
            local capline="Free Space  :    \033[1;31mLOW\033[0m"
        else
            local capline="Free Space  :     \033[1;32mOK\033[0m"
        fi
    else
        local capline="Free Space  :   \033[1;33mWAIT\033[0m"
    fi
    printf "$raidline \n$smartline \n$capline \n">$STATUS_FILE
}


# generate a list of RAID component drives
for raiddrv in `cat /proc/mdstat|grep -o -e "sd[a-z]|nvme[0-9]n[0-9]"`; do
    raidmembers=$raidmembers" /dev/$raiddrv"
done
unset raidfail
unset smartfail
unset lowspace

# main loop
while true ; do
    # determine the RAID device name and see if it's mounted
    if mount | grep -q "/dev/md" ; then
        raiddevname=`mount |grep "/dev/md"|awk '{print $1}'`
    else
        raiddevname="none"
    fi
    
    # run the tests
    if ! checkraid $raiddevname ; then raidfail=1 ; else unset raidfail ; fi
    if ! checksmart $raidmembers ; then smartfail=1 ; else unset smartfail ; fi
    if ! checkcapacity $raiddevname $CAPACITY_WARN ; then lowspace=1 ; else unset lowspace ; fi
    if [ $raidfail -o $smartfail -o $lowspace ]; then flashuid on ; else flashuid off ; fi
    update_faultstatusfile
    sleep 60
done