#!/bin/bash
#############################################################
# Name:        Supportconfig Plugin for SUSE SAP 
# Description: Gathers important troubleshooting information
#              about a SUSE SAP HANA/NetWeaver configurations
# License:     GPLv2
#############################################################

SVER='1.0.7'
RCFILE="/usr/lib/supportconfig/resources/supportconfig.rc"
TITLE="SUSE SAP"
LF=plugin-ha_sap.txt
INSTANCES=()

[ -s $RCFILE ] && . $RCFILE || { echo "ERROR: Initializing resource file: $RCFILE"; exit 1; }

validate_rpm_if_installed() {
	THISRPM=$1
	echo "#==[ Validating RPM $THISRPM ]===#"
	log_write $LF "#==[ Validating RPM ]===============================#"
	if log_cmd $LF "rpm -qi $THISRPM"; then
		echo "# rpm -V $THISRPM"
		if log_cmd $LF "rpm -V $THISRPM"; then
			echo "Status: Passed"
		else
			echo "Status: WARNING"
		fi
	else
		echo "package $THISRPM is not installed"
		echo "Status: Skipped"
	fi
}

collect_instance_log() {
	local LFILE
	for LFILE in "${@}" ; do
		log_write $LF "#==[ instance Log File ]============================#"
		log_write $LF "# ${LFILE}"
		log_cmd $LF "cat ${LFILE}"
		log_write $LF "######"
	done
}

#############################################################
log_write $LF "#==[ Supportconfig Plugin for $TITLE, v${SVER} ]====#"
log_write $LF

## Docs  ##
log_write $LF "#==[ Referencee Documentation ]=====================#"
echo -e "Current Documentation:
\t https://documentation.suse.com/en-us/sles-sap/15-SP6/
\t https://documentation.suse.com/en-us/sles-sap/15-SP5/
\t https://documentation.suse.com/en-us/sles-sap/15-SP4/
\t https://documentation.suse.com/en-us/sles-sap/15-SP3/
\t https://documentation.suse.com/en-us/sles-sap/15-SP2/
\t https://documentation.suse.com/en-us/sles-sap/15-SP1/

Tuning: 
\t https:://documentation.suse.com/sles-sap/15-SP5/single-html/SLES-SAP-guide/index.html#cha-tune

Best Practice Guides: \t https://documentation.suse.com/sbp/sap/
\tSAP HANA System Replication Scale-Out - Performance-Optimized Scenario:\t https://documentation.suse.com/sbp/sap-15/html/SLES4SAP-hana-angi-scaleout-perfopt-15/index.html
\tHow to upgrade to SAPHanaSR-angi:\t https://www.suse.com/c/how-to-upgrade-to-saphanasr-angi/
\tSAP HANA SR Performance Optimized Scale up:\t https://documentation.suse.com/sbp/sap-15/pdf/SLES4SAP-hana-sr-guide-PerfOpt-15_en.pdf
\tSAP HANA SR Performance Optimized Scale out:\t https://documentation.suse.com/sbp/sap-15/pdf/SLES4SAP-hana-scaleOut-PerfOpt-15_en.pdf
\tSAP HANA SR Performance Optimized Scale out Multi-Target:\t https://documentation.suse.com/sbp/sap-15/pdf/SLES4SAP-hana-scaleout-multitarget-perfopt-15_en.pdf
\tSAP HANA SR Cost Optimized Scale Up:\t https://documentation.suse.com/sbp/sap-15/pdf/SLES4SAP-hana-sr-guide-costopt-15_en.pdf
\tENSA1 SAP NetWeaver Enqueue Replication:\t https://documentation.suse.com/sbp/sap-15/pdf/SAP-nw740-sle15-setupguide_en.pdf
\tENSA2 SAP S/4 HANA - Enqueue Replication:\t https://documentation.suse.com/sbp/sap-15/pdf/SAP-S4HA10-setupguide-simplemount-sle15_en.pdf
" 2>&1 >> "$LOG"/$LF

RPMLIST="
SAPHanaSR-angi
SAPHanaSR
sapconf
tuned
saptune
sap-suse-cluster-connector
sap_suse_cluster_connector
SAPHanaSR-ScaleOut
resource-agents
sapstartsrv-resource-agents
saphana-checks
"

for THISRPM in $RPMLIST; do
	validate_rpm_if_installed "$THISRPM"
done
log_write $LF

## product information for Trento (offline) checks ##
log_write $LF "#==[ product information ]==========================#"
log_cmd $LF "ls -lA --time-style=long-iso /etc/products.d"
for f in /etc/products.d/*; do
	log_cmd $LF "/usr/bin/cat $f"
done

## Authentication ## 
log_write $LF "#==[ Authentication information ]===================#"
log_cmd $LF "grep -v '^\s*\#\|^$' /etc/nsswitch.conf | head -5"
log_cmd $LF "grep -E '^[[:alnum:]]{3}adm:' /etc/passwd"
log_cmd $LF "grep -E '^sap' /etc/passwd"
log_cmd $LF "grep -E '^[[:alnum:]]{3}adm:' /etc/group"
log_cmd $LF "grep -E -e '^sap' -e '^sdba:' -e '^dba:' /etc/group"
log_cmd $LF "grep -E -R -e '[[:alnum:]]{3}adm' -e 'crm_attribute' -e 'SAPHanaSR' /etc/sudoers /etc/sudoers.d/"

## SAP BASIC Tuning ##
log_write $LF "#==[ SAP Tuning information ]=======================#"

RELEASE=$(grep ^VERSION= /etc/os-release)
log_write $LF "OS Release: $RELEASE"
if [ "$RELEASE" == 'VERSION="11.4"' ]; then
	log_cmd $LF "chkconfig -l boot.sapconf"
	log_cmd $LF "cat /etc/sysconfig/sapconf | grep -v '^\s*\#\|^$'"
else 
	log_cmd $LF "systemctl status sapconf"
	if log_cmd $LF "systemctl status tuned.service"; then
		noSaptuneProfile="true"
		if log_cmd $LF "tuned-adm active"; then
			profile=$(cat /etc/tuned/active_profile)
			if [ "$profile" == "saptune" ]; then
				log_cmd $LF "saptune version"
				log_cmd $LF "saptune solution list"
				log_cmd $LF "saptune note list"
				log_cmd $LF "saptune solution verify"
				noSaptuneProfile=""
			fi
		fi
	else
		noTuned="true"
        fi
	if log_cmd $LF "systemctl status saptune.service"; then
		log_cmd $LF "saptune version"
		echo -e "HINT: since saptune 3.0 a special saptune supportconfig plugin exists\n"
		log_cmd $LF "saptune status"
		log_cmd $LF "saptune solution list"
		log_cmd $LF "saptune note list"
		log_cmd $LF "saptune solution verify"
	else
		noSaptune="true"
	fi
	if [[ "$noSaptune" == "true"  && ( "$noTuned" == "true" || "$noSaptuneProfile" == "true" ) ]]; then
		echo "WARNING: tuned.service or saptune not configured"
	fi
fi 

## SAP Host Agent ##
log_write $LF "#==[ SAP Host Agent Information ]===================#"
log_cmd $LF "/usr/sap/hostctrl/exe/saphostexec -version |tail -30"
if log_cmd $LF "/usr/sap/hostctrl/exe/saphostexec -status"; then
	echo -e "ERROR: Please make sure 'sapinit.service' is enabled and running. \n"
fi
log_cmd $LF "systemctl status sapinit"
log_cmd $LF "/usr/bin/cat /usr/sap/sapservices"
log_cmd $LF "systemctl status saphostagent.service"
log_cmd $LF "systemctl status sapping.service"
log_cmd $LF "systemctl status sappong.service"
log_cmd $LF "systemctl list-unit-files | grep -i sap"
log_cmd $LF "systemd-cgls -u SAP.slice"
log_cmd $LF "ps -ef | grep sapstartsrv"
log_cmd $LF "ls -lA --time-style=long-iso /etc/polkit-1/rules.d/[0-9][0-9]-SAP[A-Z][A-Z0-9][A-Z0-9]-[0-9][0-9].rules"

## SAP Exit Code Index ##
SAPCONTROLEXIT="sapcontrol return codes: \n 
  0  Last webmethod call successful \n 
  1  Last webmethod call failed, invalid parameter \n 
  2  StartWait, StopWait, WaitforStarted, WaitforStopped, RestartServiceWait timed out \n 
  3  GetProcessList succeeded, all processes running correctly \n 
  4  GetProcessList succeeded, all processes stopped \n"
SAPHDB1="lanscapeHostConfiguration.py return codes: \n 
  0  Fatal \n 
  1  Error \n
  2  Warning \n 
  3  Info \n 
  4  OK \n 
  5  Ignore \n"
SAPHDB2="systemReplicationStatus.py return codes: \n 
  10  NoHSR \n
  11  Error \n 
  12  Unknown ( Normal rc for Secondaray/slave node ) \n 
  13  Initializing \n 
  14  Syncing \n 
  15  Active ( Desired state for Primary/master node ) \n"

ENSAMSG1="## ENSA1 ##  (Default Standalone Enqueue Server) \n
    Service names: 'msg_server', 'enserver', and 'enrepserver' \n
    Process names: 'ms.sap*', 'er.sap*' \n"
ENSAMSG2="## ENSA2 ##  (Default starting with S/4HANA ABAP Platform 1809) \n
    Service names: 'msg_server', 'enq_server', and 'enq_replicator' \n
    Process names: 'ms.sap*', 'enq.sap*', 'enqr.sap*' \n"

log_write $LF "#==[ $TITLE retrieve SAP Instances ]==============#"
# Get SAP Instances using saphostctrl function. Relies on sap interface to be working. 
if [ -x /usr/sap/hostctrl/exe/saphostctrl ]; then
	SAP_CONFIG="sap_config.txt"
	/usr/sap/hostctrl/exe/saphostctrl -function ListInstances | awk '{print $4,$6}' &> $SAP_CONFIG

	while read -r LINE; do
		SID=$(echo "$LINE" | awk '{print $1}')
		SIDADM="$(tr '[:upper:]' '[:lower:]' <<< $SID)adm"
		INST=$(echo "$LINE" | awk '{print $2}')
		INST_NAME="SID=$SID:InstanceNumber=$INST"
		INSTANCES+=("${INST_NAME} ${SIDADM} ${SID} ${INST}")
	done < $SAP_CONFIG
	rm -r $SAP_CONFIG
fi

if [ -z "$SID" ]; then
	if [ -f /usr/sap/sapservices ]; then
		# saphostctrl does not report instances, search for systemd services
		# or read from /usr/sap/sapservices
		sapservices=$(systemctl -t service list-unit-files | sed -n 's/\(^SAP.*\).service.*/\1/p')
		if [ -z "$sapservices" ]; then
			# no systemd services available, try /usr/sap/sapservices
			sapservices=$(awk '$1!~/^#/ {print $3}' /usr/sap/sapservices | awk -F= '{print $2}')
			for s in $sapservices; do
				# /usr/sap/HA1/SYS/profile/HA1_HDB10_node11
				service=${s##*/}
				SID=${service:0:3}
				SIDADM=${SID,,}adm
				INST=${service:7:2}
				INST_NAME="SID=$SID:InstanceNumber=$INST"
				INSTANCES+=("${INST_NAME} ${SIDADM} ${SID} ${INST}")
			done
		else
			# SAP naming schema for services - SAP<SID>_<INSTNO>.service
			for service in $sapservices; do
				# SAPHA1_00
				SID=${service:3:3}
				SIDADM=${SID,,}adm
				INST=${service:7:2}
				INST_NAME="SID=$SID:InstanceNumber=$INST"
				INSTANCES+=("${INST_NAME} ${SIDADM} ${SID} ${INST}")
			done
		fi
	fi
fi
if [ -z "$SID" ]; then
	log_write $LF "NO SIDs found!"
	log_write $LF
fi

log_write $LF "#==[ $TITLE Troubleshooting Commands ]============#"
log_cmd $LF "/usr/sap/hostctrl/exe/saphostctrl -function ListInstances"
log_cmd $LF "/usr/sap/hostctrl/exe/saphostctrl -function Ping"
if [ -n "$SIDADM" ]; then
	log_cmd $LF "ps -U $SIDADM -f"
fi

# Run through each instance detected and display information about it
VHOST=$(hostname)
INST_CNT=1
LABEL=0
ADM=1
SID=2
NUM=3
for INSTANCE in "${INSTANCES[@]}"
do
	# take the INSTANCE variable and turn it into a new array called ELEMENT. Rather than refer to the items in the array by their numeric placement the above 4 variables will be used as reference.
	read -r -a ELEMENT <<< $INSTANCE
	log_entry $LF section "$TITLE Instance $INST_CNT of ${#INSTANCES[@]}: ${ELEMENT[${LABEL}]}"
	HDBEXIST=true
	log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'HDB version'"
	log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'python --version'"
	if log_cmd $LF "/usr/bin/id ${ELEMENT[${ADM}]}"; then
		log_write $LF "RETURN CODE: $?"; log_write $LF
		log_cmd $LF "SAPHanaSR-monitor"
		log_write $LF "RETURN_CODE: $?"; log_write $LF
		log_cmd $LF "SAPHanaSR-showAttr"
		log_write $LF "RETURN_CODE: $?"; log_write $LF
		log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'hdbnsutil -sr_state'"
		log_write $LF "RETURN CODE: $?"; log_write $LF
		log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'HDBSettings.sh landscapeHostConfiguration.py --sapcontrol=1'"
		log_write $LF "RETURN CODE: $?"; echo -e "$SAPHDB1" >> "$LOG"/$LF; log_write $LF
		log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'HDBSettings.sh systemReplicationStatus.py'"
		log_write $LF "RETURN CODE: $?"; echo -e "$SAPHDB2" >> "$LOG"/$LF; log_write $LF
		log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'HDBSettings.sh systemOverview.py'"
		log_write $LF "RETURN CODE: $?"; log_write $LF
		log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'HDB info'"
		log_write $LF "RETURN CODE: $?"; log_write $LF
		log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'python getParameter.py --key=global.ini/system_replication/operation_mode --sapcontrol=1'"
		log_write $LF "RETURN CODE: $?"; log_write $LF
	else
		log_write $LF "RETURN CODE: $?"; log_write $LF
		HDBEXIST=false
	fi
	log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'sapcontrol -nr ${ELEMENT[${NUM}]} -function GetVersionInfo'"
	log_write $LF "RETURN CODE: $?"
	log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'sapcontrol -nr ${ELEMENT[${NUM}]} -function GetStartProfile'"
	log_write $LF "RETURN CODE: $?"
	log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'sapcontrol -nr ${ELEMENT[${NUM}]} -function GetSystemInstanceList'"
	log_write $LF "RETURN CODE: $?"
	log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'sapcontrol -nr ${ELEMENT[${NUM}]} -function GetProcessList'"
	log_write $LF "RETURN CODE: $?"; echo -e "$SAPCONTROLEXIT" >> "$LOG"/$LF; log_write $LF; echo -e "$ENSAMSG1" >> "$LOG"/$LF; echo -e "$ENSAMSG2" >> "$LOG"/$LF
	log_cmd $LF "su - ${ELEMENT[${ADM}]} -c disp+work"

	INSTANCE_PROFILE_DIR=$(su - "${ELEMENT[${ADM}]}" -c 'cdpro && pwd' 2>/dev/null)
	INSTANCE_CONFIG_DIR=$(su - "${ELEMENT[${ADM}]}" -c 'cdcoc && pwd' 2>/dev/null)
	INSTANCE_TRACE_DIR=$(su - "${ELEMENT[${ADM}]}" -c 'cdtarce && pwd' 2>/dev/null)
	if [[ -d $INSTANCE_CONFIG_DIR ]] && [[ $HDBEXIST == true ]]; then
		conf_files $LF  "${INSTANCE_CONFIG_DIR}"/global.ini "${INSTANCE_CONFIG_DIR}"/nameserver.ini "${INSTANCE_CONFIG_DIR}"/daemon.ini
	fi
	if [[ -d $INSTANCE_PROFILE_DIR ]]; then
		conf_files $LF "${INSTANCE_PROFILE_DIR}"/"${ELEMENT[${SID}]}"_*
	fi
	if [[ -d $INSTANCE_TRACE_DIR ]]; then
		log_cmd $LF "ls -ltra $INSTANCE_TRACE_DIR"
	fi
	
	log_cmd $LF "systemctl status SAP${ELEMENT[${SID}]}_${ELEMENT[${NUM}]}.service"

	# SUSE HANA hook script logs
	trace_dir=/usr/sap/${ELEMENT[${SID}]}/HDB${ELEMENT[${NUM}]}/${VHOST}/trace
	# /usr/sap/HA1/HDB10/hanacost1/trace/nameserver_suschksrv.trc
	collect_instance_log "${trace_dir}"/nameserver_suschksrv.trc
	# /usr/sap/HA1/HDB10/hanacost1/trace/nameserver_saphanasr_multitarget_hook.trc
	collect_instance_log "${trace_dir}"/nameserver_saphanasr_multitarget_hook.trc

	## Adding support for sap-suse-cluster-connector  ##
        if log_cmd $LF "rpm -qa | grep -E 'sap-suse-cluster-connector|sap_suse_cluster_connector'"; then
                log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'sapcontrol -nr ${ELEMENT[${NUM}]} -function HAGetFailoverConfig'"
                log_write $LF "RETURN CODE: $?"; log_write $LF
                log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'sapcontrol -nr ${ELEMENT[${NUM}]} -function HACheckFailoverConfig'"
                log_write $LF "RETURN CODE: $?"; log_write $LF
                log_cmd $LF "su - ${ELEMENT[${ADM}]} -c 'sapcontrol -nr ${ELEMENT[${NUM}]} -function HACheckConfig'"
                log_write $LF "RETURN CODE: $?"; log_write $LF

		# collect instance logs
		work_dir=/usr/sap/${ELEMENT[${SID}]}/ASCS${ELEMENT[${NUM}]}/work
		# /usr/sap/EN2/ASCS00/work/sap_suse_cluster_connector.[old|log]
		collect_instance_log "${work_dir}"/sap_suse_cluster_connector.???
		# /usr/sap/EN2/ASCS00/work/sapstartsrv.???
		collect_instance_log "${work_dir}"/sapstartsrv.???
        fi

	 INST_CNT=$((INST_CNT + 1))
done
## Adding some basic cluster commands.  Refer to ha.txt in supportconfig for additional ##
log_write $LF "#==[ $TITLE Basic Cluster Configuration ]=========#"
log_cmd $LF "crm_mon -A -r -1"
log_cmd $LF "crm configure show obscure:passw*"
log_cmd $LF "corosync-cmapctl -b"

log_write $LF "#==[ $TITLE Log Files ]===========================#"
log_cmd $LF "grep -E -i 'saphana|SAPDatabase|SAPInstance|SAPStartsrv|sapcontrol|saphostctrl|sap_suse_cluster_connector' /var/log/messages | tail -1000"
if ! [ "$RELEASE" == 'VERSION="11.4"' ]; then
	log_cmd $LF "find /var/log/ -name pacemaker.log -exec grep -E -i 'saphana|SAPDatabase|SAPInstance|SAPStartsrv|sapcontrol|saphostctrl' {} \; | tail -1000"
fi

## running saphana-checks ##
log_write $LF "#==[ running saphana-checks ]=======================#"
if [ -x /usr/lib/saphana-checks/bin/saphana-check.sh ]; then
	log_write $LF "saphana-checks provided by SUSE"
	log_cmd $LF "/usr/lib/saphana-checks/bin/saphana-check.sh"
elif [ -x /opt/sap/saphana-checks/bin/saphana-check.sh ]; then
	log_write $LF "saphana-checks provided by SAP"
	log_cmd $LF "/opt/sap/saphana-checks/bin/saphana-check.sh"
else
	log_write $LF "saphana-checks are not installed"
fi
