#!/bin/bash

SVER='1.0.8'

##############################################################################
# sdagent - SCA Agent Manager
# Copyright (c) 2014-2018 SUSE LLC
#
# Description:  Starts agent threads for new and retry archives
# Runs on:      Agent Server
# Modified:     2018 Jan 03
##############################################################################
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; version 2 of the License.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, see <http://www.gnu.org/licenses/>.
#
#  Authors/Contributors:
#     Jason Record (jason.record@suse.com)
#
##############################################################################

CURRENT_SCRIPT=$(basename $0)
. /etc/sca/sdagent.conf

##############################################################################
# Global Variables
##############################################################################

AGENT_BIN='/usr/sbin/sdagent-supportconfig'
ASSIGNED_ARCHIVES=0
SUPPRESS_ADDITIONAL_EMAILS="${SCA_WORK}/emails-suppressed.txt"
DEACTIVATE=0
ACTIVATE=0
PID_CHECK_DELAY=5

##############################################################################
# Local Function Definitions
##############################################################################

title() {
	test $LOGLEVEL -ge $LOGLEVEL_NORMAL && clear
	msg normal "SCA_AGENT_SERVER" "$CURRENT_SCRIPT v$SVER"
}

showHelp() {
	[[ -n "$1" ]] && { echo "$1"; echo; }
	echo "Usage: $CURRENT_SCRIPT [OPTIONS]"
	echo "Description:"
	echo "  Assigns new supportconfig archives for analysis."
	echo
	echo "Options:"
	echo "  -h  Show this screen and exit"
	echo "  -l  Broker log: 0=Syslog(default) 1=Minimal 2=Verbose 3=Debug"
	echo "  -w  Worker log: 1=Minimal 2=Verbose(default) 3=Debug"
	echo "  -a  Activate the agent"
	echo "  -d  Deactivate the agent"
	echo
}

msg() {
	FACLEVEL='info'
	case $1 in
	debug) SYSTEM_LOGGER=1; CURRENT_LOGLEVEL=$LOGLEVEL_DEBUG ;;
	verbose) SYSTEM_LOGGER=1; CURRENT_LOGLEVEL=$LOGLEVEL_VERBOSE ;;
	normal) SYSTEM_LOGGER=1; CURRENT_LOGLEVEL=$LOGLEVEL_NORMAL ;;
	min*) SYSTEM_LOGGER=1; CURRENT_LOGLEVEL=$LOGLEVEL_MIN ;;
	silent) SYSTEM_LOGGER=1; CURRENT_LOGLEVEL=$LOGLEVEL_SILENT ;;
	*) SYSTEM_LOGGER=1; CURRENT_LOGLEVEL=$LOGLEVEL_MIN ;;
	esac
	shift
	if [[ $LOGLEVEL -ge $CURRENT_LOGLEVEL ]]; then
		if [[ $SYSTEM_LOGGER -gt 0 ]]; then
			[[ -n "$2" ]] && logger -p "user.${FACLEVEL}" "${CURRENT_SCRIPT}[$$]: [$1] $2" || logger -p "user.${FACLEVEL}" "${CURRENT_SCRIPT}[$$]: $1"
			if [[ $LOGLEVEL -gt $LOGLEVEL_SILENT ]]; then
				printf "%-15s %s\n" "$1" "$2"
			fi
		else
			printf "%-15s %s\n" "$1" "$2"
		fi
	fi
}

notifyAdmin() {
	msg debug '> notifyAdmin'
	if [[ -e $SUPPRESS_ADDITIONAL_EMAILS ]]; then
		msg normal NOTIFY "Agent Manager Emails Suppressed, Found $SUPPRESS_ADDITIONAL_EMAILS"
	else
		msg normal NOTIFY "Sending Agent Manager Alert to Email(s): $EMAIL_ADMIN"
		EVENT_STR=$1
		shift
		for SENDTO in $EMAIL_ADMIN
		do
			msg debug EMAIL "Send To: $SENDTO, Agent Manager: $EVENT_STR, $*"
			echo "$*" | /usr/bin/mailx -r "SCA Agent Notification<root>" -ns "Agent Manager: $EVENT_STR" $SENDTO
		done
	fi
	msg debug '< notifyAdmin'
}

agentAccess() {
	msg debug '> agentAccess'
	# returns 0 NO_ACCESS, 1 INACTIVE, 2 ACTIVE
	CHECK_AGENT_STATE=$(mysql -NB $DB_CONNECT -e "SELECT AgentState FROM Agents WHERE AgentID=$AGENT_ID")
	RCODE=$?
	if [[ $RCODE -gt 0 ]]; then
		msg normal DB "ERROR Cannot access administrative database $DB_NAME on localhost"
		RCODE=$AGENT_NOACCESS
	else
		case $CHECK_AGENT_STATE in
		Active) RCODE=$AGENT_ACTIVE ;;
		Error) RCODE=$AGENT_NOACCESS ;;
		*) RCODE=$AGENT_INACTIVE ;;
		esac
		msg verbose ACCESS "Agent State '$CHECK_AGENT_STATE', setting agentAccess = $RCODE"
	fi
	msg debug "< agentAccess, RCODE = $RCODE"
	return $RCODE
}

assignNewArchives() {
	msg debug '> assignNewArchives'
	mysql $DB_CONNECT -e "LOCK TABLES Archives WRITE" #Establish MUTEX
	ARCHIDS=$(mysql -BN $DB_CONNECT -e "SELECT ArchiveID FROM Archives WHERE ArchiveState='New' ORDER BY ArchiveEvent DESC LIMIT $LIMIT")
	msg debug ARCHIDS "$ARCHIDS"
	if [[ -n "$ARCHIDS" ]]; then
		mysql $DB_CONNECT -e "UPDATE Archives SET ArchiveState='Assigned', ArchiveEvent=NOW(), AssignedAgentID=$AGENT_ID WHERE ArchiveID IN (${ARCHIDS//[[:space:]]/,})"
		mysql $DB_CONNECT -e "UNLOCK TABLES"
		for ARCHIVE_ID in $ARCHIDS
		do
			msg debug 'SQL AGENT_ID' "Agent ID '$AGENT_ID' assigned to Archive ID '$ARCHIVE_ID'"
			WORKER_ID=$(mysql -BN $DB_CONNECT -e "SELECT WorkerID FROM AgentWorkers WHERE WorkersAgentID='$AGENT_ID' AND ArchiveAssigned IS NULL LIMIT 1")
			if [[ -n "$WORKER_ID" ]]; then
				msg debug 'SQL WORKER_ID' "Assigned Agent Worker ID: $WORKER_ID"
				ARCHIVE_FILE=$(mysql -BN $DB_CONNECT -e "SELECT Filename FROM Archives WHERE ArchiveID='$ARCHIVE_ID'")
				mysql $DB_CONNECT -e "UPDATE Archives SET AssignedWorkerID=$WORKER_ID WHERE ArchiveID='$ARCHIVE_ID'"
				mysql $DB_CONNECT -e "UPDATE Agents SET ThreadsActive=ThreadsActive+1, AgentEvent=NOW() WHERE AgentID='$AGENT_ID'"
				mysql $DB_CONNECT -e "UPDATE AgentWorkers SET ArchiveAssigned='$ARCHIVE_ID' WHERE WorkersAgentID='$AGENT_ID' AND WorkerID=$WORKER_ID"
				((ASSIGNED_ARCHIVES++))
				msg debug CMD "$AGENT_BIN -l${LOGLEVEL_WORKER} -i \"${AGENT_ID},${WORKER_ID},${ARCHIVE_ID}\" -p worker${WORKER_ID} -a $ARCHIVE_FILE &>/dev/null"
				nohup $AGENT_BIN -l${LOGLEVEL_WORKER} -i "${AGENT_ID},${WORKER_ID},${ARCHIVE_ID}" -p worker${WORKER_ID} -a ${ARCHIVE_FILE} &>/dev/null &
			else
				mysql $DB_CONNECT -e "UPDATE Archives SET ArchiveState='Retry', AssignedWorkerID=NULL, AssignedAgentID=NULL WHERE ArchiveID='$ARCHIVE_ID'"
				ACTIVE_WORKERS=$(mysql -NB $DB_CONNECT -e "SELECT COUNT(WorkerID) FROM AgentWorkers WHERE ArchiveAssigned IS NOT NULL")
				msg verbose AGENTS "Total active agent workers: $ACTIVE_WORKERS"
				msg verbose AGENTS "Warning: No available agent threads for new archives"
				[[ $EMAIL_LEVEL -ge $EMAIL_NORMAL ]] && notifyAdmin "NEW: All Agents Busy" "Warning: All agents are busy, total active agent workers: $ACTIVE_WORKERS"
				break
			fi
		done
		msg normal NEW "New archives assigned to agents: $ASSIGNED_ARCHIVES"
	else
		msg verbose ARCHIVES "No new archives to process"
		mysql $DB_CONNECT -e "UNLOCK TABLES"
	fi
	msg debug '< assignNewArchives'
}

agentBusy() {
	msg debug '> agentBusy'
	RCODE=0
	LIMIT=$(mysql -BN $DB_CONNECT -e "SELECT COUNT(WorkerID) FROM AgentWorkers WHERE ArchiveAssigned IS NULL AND WorkersAgentID=$AGENT_ID")
	export LIMIT
	if [[ $LIMIT -gt 0 ]]; then
		CPU_MAX=$(mysql -BN $DB_CONNECT -e "SELECT CPUMax FROM Agents WHERE AgentID='$AGENT_ID'")
		CPU_IDLE=$(vmstat 1 2 | tail -1 | awk '{print $15}')
		CPU_UTIL=$((100-CPU_IDLE))
		if [[ $CPU_UTIL -ge $CPU_MAX ]]; then
			msg normal AGENT "CPU Utilization of ${CPU_UTIL}% exceeds ${CPU_MAX}% threshold"
			mysql $DB_CONNECT -e "UPDATE Agents SET AgentEvent=NOW(), AgentMessage='Busy: Max CPU', CPUCurrent=$CPU_UTIL WHERE AgentID=$AGENT_ID"
			SQLERR=$?
			((RCODE++))
		else
			mysql $DB_CONNECT -e "UPDATE Agents SET AgentEvent=NOW(), AgentMessage='Ready', CPUCurrent=$CPU_UTIL WHERE AgentID=$AGENT_ID"
			SQLERR=$?
		fi
		if [[ $SQLERR -gt 0 ]]; then
			msg normal SQL "ERROR: Failed to update AgentID $AGENT_ID on $HOSTNAME"
			[[ $EMAIL_LEVEL -ge $EMAIL_MIN ]] && notifyAdmin "Agent $AGENT_ID Update Failed" "ERROR: Failed to update AgentID $AGENT_ID on $HOSTNAME"
			exit 3
		else
			msg normal CPU "$HOSTNAME CPU Utilization: ${CPU_UTIL}%"
		fi
	else
		msg normal AGENT "No available agent worker threads"
		mysql $DB_CONNECT -e "UPDATE Agents SET AgentEvent=NOW(), AgentMessage='Busy: Max Threads' WHERE AgentID=$AGENT_ID"
		((RCODE++))
	fi
	msg debug "< agentBusy, RCODE=$RCODE, CPU Utilization: ${CPU_UTIL}%, Available Worker Threads: $LIMIT"
	return $RCODE
}

deactivateAgent() {
	msg debug "> deactivateAgent"
	if [[ $AGENT_STATE -gt 1 ]]; then	
		msg min DEACTIVATE "Deactivating Agent $AGENT_ID"
		mysql $DB_CONNECT -e "UPDATE Agents SET AgentState='Inactive', AgentEvent=NOW(), AgentMessage='Deactivating Agent'  WHERE AgentID=$AGENT_ID"
		msg verbose DEACTIVATE "Stopping all worker thread processes on agent $AGENT_ID"
		STOP_WORKERS=1
		while (( STOP_WORKERS )); 
		do
			PIDS=$(ps axwwo pid,cmd | grep $AGENT_BIN | grep -v grep | cut -d' ' -f1)
			if [[ -n "$PIDS" ]]; then
				msg min DEACTIVATE "Stopping Active Threads: $PIDS"
				kill -9 $PIDS
				sleep $PID_CHECK_DELAY
			else
				msg debug DEACTIVATE "All $(basename $AGENT_BIN) threads stopped"
				STOP_WORKERS=0
			fi
		done
		msg verbose DEACTIVATE "Deleting all agent $AGENT_ID worker directories"
		WORKER_DIRS=$(mysql -NB $DB_CONNECT -e "SELECT HomePath FROM AgentWorkers WHERE WorkersAgentID='$AGENT_ID'")
		[[ -n "$WORKER_DIRS" ]] && rm -rf $WORKER_DIRS &>/dev/null
		msg verbose DEACTIVATE "Releasing all archives assigned to agent $AGENT_ID in the database"
		mysql $DB_CONNECT -e "UPDATE Archives SET ArchiveState='Retry', ArchiveMessage='Releasing archive, agent deactivated', RetryCount=0, AssignedWorkerID=NULL WHERE AssignedAgentID='$AGENT_ID' AND ArchiveState='Assigned' OR ArchiveState='Downloading' OR ArchiveState='Extracting' OR ArchiveState='Analyzing' OR ArchiveState='Reporting'"
		msg verbose DEACTIVATE "Resetting agent $AGENT_ID worker threads in the database"
		mysql $DB_CONNECT -e "UPDATE AgentWorkers SET ArchiveAssigned=NULL WHERE WorkersAgentID=$AGENT_ID"
		msg verbose DEACTIVATE "Updating database agent $AGENT_ID metadata"
		mysql $DB_CONNECT -e "UPDATE Agents SET ThreadsActive=0, AgentMessage='Deactivated by Agent Administrator' WHERE AgentID=$AGENT_ID"
	else
		msg normal DEACTIVATE "Cannot Deactivate Agent $AGENT_ID, Invalid AGENT_STATE $AGENT_STATE, must be greater than 1"
	fi
	msg debug "< deactivateAgent"
}

activateAgent() {
	msg debug "> activateAgent"
	if [[ $AGENT_STATE -eq 1 ]]; then	
		msg min ACTIVATE "Activating Agent $AGENT_ID"
		mysql $DB_CONNECT -e "UPDATE Agents SET AgentEvent=NOW(), AgentState='Active', AgentMessage='Activated by Agent Administrator', ThreadsActive=0, CPUCurrent=0 WHERE AgentID='$AGENT_ID'"
		msg verbose ACTIVATE "Resetting agent $AGENT_ID worker threads in the database"
		mysql $DB_CONNECT -e "UPDATE AgentWorkers SET ArchiveAssigned=NULL WHERE WorkersAgentID=$AGENT_ID"
		rm -f $SUPPRESS_ADDITIONAL_EMAILS
	else
		msg normal ACTIVATE "Cannot Activate Agent $AGENT_ID, Invalid AGENT_STATE $AGENT_STATE, must be equal to 1"
	fi
	msg debug "< activateAgent"
}

##############################################################################
# Main Program Execution
##############################################################################

title
[[ -d $SCA_WORK ]] || mkdir -p $SCA_WORK
[[ $AGENT_PRIORITY_GROUP -gt 0 ]] && sleep $AGENT_PRIORITY_GROUP
while getopts ':hl:w:ad' TMPOPT
do
	case $TMPOPT in
		\:) showHelp "ERROR: Missing Argument -$OPTARG"; exit 1 ;; 
		\?) showHelp "ERROR: Invalid Option -$OPTARG"; exit 2 ;;
		h) LOGLEVEL=$LOGLEVEL_NORMAL; showHelp; exit 0 ;;
		l) LOGLEVEL=$OPTARG ;;
		w) LOGLEVEL_WORKER=$OPTARG ;;
		a) ACTIVATE=1 ;;
		d) DEACTIVATE=1 ;;
	esac
done

agentAccess
AGENT_STATE=$?
if [[ $AGENT_STATE -eq $AGENT_NOACCESS ]]; then
	[[ $EMAIL_LEVEL -ge $EMAIL_MIN ]] && notifyAdmin "No DB Access" "ERROR Cannot access administrative database $DB_NAME on localhost"
	exit 5
else
	if [[ $ACTIVATE -gt 0 ]]; then
		activateAgent
	elif [[ $DEACTIVATE -gt 0 ]]; then
		deactivateAgent
	else
		if [[ $AGENT_STATE -ge $AGENT_ACTIVE ]]; then
			agentBusy
			if [[ $? -gt 0 ]]; then
				msg normal AGENT "Agent $AGENT_ID on $HOSTNAME busy"
			else
				assignNewArchives
				sleep 1
				[[ $ASSIGNED_ARCHIVES -gt 0 ]] && agentBusy
			fi
		else
			msg normal AGENT "ERROR The agent is not active"
			[[ $EMAIL_LEVEL -ge $EMAIL_MIN ]] && notifyAdmin 'AGENT NOT ACTIVE' "ERROR: The agent on $HOSTNAME is not Active, no supportconfig archives will be processed on this agent."
			date > $SUPPRESS_ADDITIONAL_EMAILS
			exit 6
		fi
	fi
fi
