131 lines
3.8 KiB
Bash
131 lines
3.8 KiB
Bash
#!/bin/bash
|
|
# faultmonitor.sh
|
|
# Rescue CD Backup & Restore Utility fault monitoring script
|
|
# version 7.2.5
|
|
# Rod Wright 1/19/2024
|
|
# Refer to CHANGELOG file for details.
|
|
# -----------------------------------------------------------------------------
|
|
|
|
CAPACITY_WARN=90
|
|
STATUS_FILE="/root/faultstatus.txt"
|
|
|
|
function checkraid() {
|
|
# check for RAID failure
|
|
# takes one argument, a full raid device name
|
|
local device=$1
|
|
if [ "$device" != "none" ]; then
|
|
if mdadm -D $device | grep -q degraded ; then
|
|
return 1
|
|
else
|
|
return 0
|
|
fi
|
|
else
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
function checksmart() {
|
|
# check for RAID member SMART failure
|
|
# takes a list of RAID partition full device names as positional parameters
|
|
for drive in $* ; do
|
|
if echo $drive|grep -q nvme ; then
|
|
nvmesmartstat=`nvme smart-log $drive|grep critical_warning|cut -d ":" -f2|sed 's/^ *//g'`
|
|
if [ $nvmesmartstat -ne 0 ]; then return 1 ;fi
|
|
else
|
|
smartctl -q silent $drive
|
|
local smartstat=$(($? & 8))
|
|
if [ $smartstat -ne 0 ]; then return 1 ;fi
|
|
fi
|
|
done
|
|
return 0
|
|
}
|
|
|
|
function checkcapacity() {
|
|
# check for RAID usage above a threshold
|
|
# takes two arguments, a RAID device and a percentage
|
|
local device=$1
|
|
local threshold=$2
|
|
if [ "$device" != "none" ]; then
|
|
local pctstrg=`df $device |grep $device|awk '{print $5}'`
|
|
local pct=${pctstrg//%}
|
|
if [ $pct -ge $threshold ] ; then
|
|
return 1
|
|
else
|
|
return 0
|
|
fi
|
|
else
|
|
return 0
|
|
fi
|
|
}
|
|
|
|
function flashuid() {
|
|
# turn flashing UID light on or off
|
|
# takes one argument, the word "on" or "off"
|
|
# determine if ipmi hardware is supported and return failure if not
|
|
if ! ipmitool chassis status >/dev/null 2>&1; then return 1 ; fi
|
|
if [ "$1" = "on" ]; then
|
|
local ipmiarg="force"
|
|
elif [ "$1" = "off" ]; then
|
|
local ipmiarg="0"
|
|
else
|
|
# invalid argument make no change and return failure
|
|
return 1
|
|
fi
|
|
ipmitool chassis identify $ipmiarg
|
|
}
|
|
|
|
function update_faultstatusfile() {
|
|
# maintain the fault status file
|
|
if [ "$raiddevname" != "none" ]; then
|
|
if [ $raidfail ]; then
|
|
local raidline="RAID Status : \033[1;31mFAIL\033[0m"
|
|
else
|
|
local raidline="RAID Status : \033[1;32mOK\033[0m"
|
|
fi
|
|
else
|
|
local raidline="RAID Status : \033[1;33mWAIT\033[0m"
|
|
fi
|
|
if [ $smartfail ]; then
|
|
local smartline="Disk Status : \033[1;31mFAIL\033[0m"
|
|
else
|
|
local smartline="Disk Status : \033[1;32mOK\033[0m"
|
|
fi
|
|
if [ "$raiddevname" != "none" ]; then
|
|
if [ $lowspace ]; then
|
|
local capline="Free Space : \033[1;31mLOW\033[0m"
|
|
else
|
|
local capline="Free Space : \033[1;32mOK\033[0m"
|
|
fi
|
|
else
|
|
local capline="Free Space : \033[1;33mWAIT\033[0m"
|
|
fi
|
|
printf "$raidline \n$smartline \n$capline \n">$STATUS_FILE
|
|
}
|
|
|
|
|
|
# generate a list of RAID component drives
|
|
for raiddrv in `cat /proc/mdstat|grep -o -e "sd[a-z]|nvme[0-9]n[0-9]"`; do
|
|
raidmembers=$raidmembers" /dev/$raiddrv"
|
|
done
|
|
unset raidfail
|
|
unset smartfail
|
|
unset lowspace
|
|
|
|
# main loop
|
|
while true ; do
|
|
# determine the RAID device name and see if it's mounted
|
|
if mount | grep -q "/dev/md" ; then
|
|
raiddevname=`mount |grep "/dev/md"|awk '{print $1}'`
|
|
else
|
|
raiddevname="none"
|
|
fi
|
|
|
|
# run the tests
|
|
if ! checkraid $raiddevname ; then raidfail=1 ; else unset raidfail ; fi
|
|
if ! checksmart $raidmembers ; then smartfail=1 ; else unset smartfail ; fi
|
|
if ! checkcapacity $raiddevname $CAPACITY_WARN ; then lowspace=1 ; else unset lowspace ; fi
|
|
if [ $raidfail -o $smartfail -o $lowspace ]; then flashuid on ; else flashuid off ; fi
|
|
update_faultstatusfile
|
|
sleep 60
|
|
done
|