#!/bin/bash

SVER='1.0.6'

##############################################################################
#  sdbroker-monitor - SCA Broker Monitor
#  Copyright (C) 2018-2021 SUSE LLC
#
# Description:  Monitors the current status of Broker Server and it's Agents
# Runs on:      Broker Server
# Modified:     2021 Mar 12
#
##############################################################################
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; version 2 of the License.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, see <http://www.gnu.org/licenses/>.
#
#  Authors/Contributors:
#     Jason Record (jason.record@suse.com)
#
##############################################################################

CURRENT_SCRIPT=$(basename $0)
. /etc/sca/sdbroker.conf
[[ -s /etc/sca/sdbroker-monitor.conf ]] && . /etc/sca/sdbroker-monitor.conf

msg() {
	FACLEVEL='info'
	case $1 in
	debug) SYSTEM_LOGGER=0; CURRENT_LOGLEVEL=$LOGLEVEL_DEBUG ;;
	verbose) SYSTEM_LOGGER=1; CURRENT_LOGLEVEL=$LOGLEVEL_VERBOSE ;;
	normal) SYSTEM_LOGGER=1; CURRENT_LOGLEVEL=$LOGLEVEL_NORMAL ;;
	min*) SYSTEM_LOGGER=1; CURRENT_LOGLEVEL=$LOGLEVEL_MIN ;;
	silent) SYSTEM_LOGGER=1; CURRENT_LOGLEVEL=$LOGLEVEL_SILENT ;;
	*) SYSTEM_LOGGER=1; CURRENT_LOGLEVEL=$LOGLEVEL_MIN ;;
	esac
	shift
	if [[ $LOGLEVEL -ge $CURRENT_LOGLEVEL ]]; then
		if [[ $SYSTEM_LOGGER -gt 0 ]]; then
			[[ -n "$2" ]] && logger -p "user.${FACLEVEL}" "${CURRENT_SCRIPT}[$$]: [$1] $2" || logger -p "user.${FACLEVEL}" "${CURRENT_SCRIPT}[$$]: $1"
			if [[ $LOGLEVEL -gt $LOGLEVEL_SILENT ]]; then
				printf "%-15s %s\n" "$1" "$2"
			fi
		else
			printf "%-15s %s\n" "$1" "$2"
		fi
	fi
}

notifyAdmin() {
	EVENT_STR=$1
	shift
	for SENDTO in $EMAIL_ADMIN
	do
		echo "$*" | /usr/bin/mailx -r "SCA Broker Notification <root>" -ns "Monitor Alert: $EVENT_STR" $SENDTO
	done
}

restoreDeadAgents() {
	DEAD_AGENTS=$(mysql -NB $DB_CONNECT -e "SELECT Hostname FROM Agents WHERE AgentState='Dead'")
	if [[ -n "$DEAD_AGENTS" ]]; then
		msg verbose BROKER "Checking dead agents $DEAD_AGENTS"
		for i in $DEAD_AGENTS
		do
			if ping -c1 -w1 $i &>/dev/null; then
				AGENT_ID=$(mysql -NB $DB_CONNECT -e "SELECT AgentID FROM Agents WHERE Hostname='$i'")
				THREADS=$(mysql -NB $DB_CONNECT -e "SELECT COUNT(ArchiveAssigned) FROM AgentWorkers WHERE WorkersAgentID='$AGENT_ID' AND ArchiveAssigned IS NOT NULL")
				mysql $DB_CONNECT -e "UPDATE Agents SET AgentEvent=NOW(), AgentState='Active', AgentMessage='Agent back up, activating', ThreadsActive=$THREADS WHERE Hostname='$i'"
				msg min BROKER "Activated Agent $i"
				[[ $EMAIL_LEVEL -ge $EMAIL_MIN ]] && notifyAdmin "Activated Agent $i, Ready for archive assignments."
			else
				msg min BROKER "Still Cannot Connect Agent: $i"
				[[ $EMAIL_LEVEL -ge $EMAIL_MIN ]] && notifyAdmin "Agent $i Dead, still cannot ping the agent."
			fi
		done
	else
		msg verbose BROKER "No dead agents found"
	fi
}

pingAgents() {
	PING_AGENTS=$(mysql -NB $DB_CONNECT -e "SELECT Hostname FROM Agents WHERE AgentState='Active'")
	if [[ -n "$PING_AGENTS" ]]; then
		msg verbose BROKER "Checking agent connectivity $PING_AGENTS"
		for i in $PING_AGENTS
		do
			if ping -c1 -w1 $i &>/dev/null; then
				msg verbose BROKER "Ping Agent $i Successful"
				[[ $EMAIL_LEVEL -ge $EMAIL_VERBOSE ]] && notifyAdmin "Agent $i, Network connectivity confirmed."
			else
				msg min BROKER "Ping Agent $i FAILED"
				AGENT_ID=$(mysql -NB $DB_CONNECT -e "UPDATE Agents SET AgentEvent=NOW(), AgentState='Dead', AgentMessage='Cannot ping agent' WHERE Hostname='$i'")
				[[ $EMAIL_LEVEL -ge $EMAIL_MIN ]] && notifyAdmin "Agent $i Dead, cannot ping agent."
				exit
			fi
		done
	else
		msg verbose BROKER "No active agents found"
	fi
}

###########################################################################################################
### main
###########################################################################################################

[[ -s /etc/HOSTNAME ]] && SERVER_NAME=$(head -1 /etc/HOSTNAME) || SERVER_NAME=$(hostname)
mysql $DB_CONNECT -e "USE $DB_NAME" &>/dev/null
if [[ $? -gt 0 ]]; then
	msg normal SQL "ERROR: Broker $SERVER_NAME cannot connect to the $DB_NAME database on $DB_HOSTNAME"
	[[ $EMAIL_LEVEL -ge $EMAIL_MIN ]] && notifyAdmin "DB Connection Failed" "ERROR: Broker $SERVER_NAME cannot connect to the $DB_NAME database on $DB_HOSTNAME"
	exit 2
else
	pingAgents
	restoreDeadAgents
fi

