Subject: [PATCH] [BZ 184025] dbginfo.sh: Extend data collection
From: Sa Liu <saliu@de.ibm.com>

Description:  dbginfo.sh: Extend data collection 
Symptom:      This update covers various symptoms on dbginfo.sh data
              collection: 
              - There is no data collection for ethtool, tc and bridge
                output for debugging network issues
              - There is no data collection for hyptop output
              - There is no data collection for nvme devices
              - There is no data collection for smc devices
              - Sometimes the lsof output is too long which makes the 
                runtime.out hard to be opened
              - The journalctl was limited to 50000 lines
Problem:      Following problems exist:
              - Missing data collection for ethtool, tc and bridge  
              - Missing data collection for hyptop output
              - Missing data collection for nvme devices
              - Missing data collection for smc devices
              - Sometimes the lsof output is too long which makes the
                runtime.out hard to be opened
              - The journalctl was limited to 50000 lines. A very long
                journalctl will overwrite the beginning part
Solution:     - Extend the data collection to collect output for 
                ethtool, tc, bridge and hyptop output
              - Extend the data collection to collect nvme and smc 
                device information
              - Write the output of lsof to a separate file
              - Extend journalctl to 100000 lines
Reproduction: Run this script and verify the output 
Upstream-ID:   -
Problem-ID:    184025

Signed-off-by: Sa Liu <saliu@de.ibm.com>
Index: s390-tools-service/scripts/dbginfo.sh
===================================================================
--- s390-tools-service.orig/scripts/dbginfo.sh
+++ s390-tools-service/scripts/dbginfo.sh
@@ -2,7 +2,7 @@
 #
 # dbginfo.sh - Tool to collect runtime, configuration, and trace information
 #
-# Copyright IBM Corp. 2002, 2017
+# Copyright IBM Corp. 2002, 2020
 #
 # s390-tools is free software; you can redistribute it and/or modify
 # it under the terms of the MIT license. See LICENSE for details.
@@ -21,7 +21,7 @@ readonly SCRIPTNAME="${0##*/}"
 print_version() {
     cat <<EOF
 ${SCRIPTNAME}: Debug information script version %S390_TOOLS_VERSION%
-Copyright IBM Corp. 2002, 2017
+Copyright IBM Corp. 2002, 2020
 EOF
 }
 
@@ -167,9 +167,21 @@ readonly OUTPUT_FILE_VMCMD="${WORKPATH}z
 # File that includes content of files from sysfs
 readonly OUTPUT_FILE_SYSFS="${WORKPATH}sysfsfiles.out"
 
+# File that includes the output of lsof
+readonly OUTPUT_FILE_LSOF="${WORKPATH}open_files.out"
+
 # File that includes content of OSA OAT
 readonly OUTPUT_FILE_OSAOAT="${WORKPATH}osa_oat"
 
+# File that includes content of Ethtool commands
+readonly OUTPUT_FILE_ETHTOOL="${WORKPATH}ethtool.out"
+
+# File that includes content of tc commands
+readonly OUTPUT_FILE_TC="${WORKPATH}tc.out"
+
+# File that includes content of bridge commands
+readonly OUTPUT_FILE_BRIDGE="${WORKPATH}bridge.out"
+
 # File that includes the output of journalctl
 readonly OUTPUT_FILE_JOURNALCTL="${WORKPATH}journalctl.out"
 
@@ -189,7 +201,7 @@ readonly OUTPUT_FILE_NVME="${WORKPATH}nv
 readonly MOUNT_POINT_DEBUGFS="/sys/kernel/debug"
 
 # The amount of steps running the whole collections
-readonly COLLECTION_COUNT=12
+readonly COLLECTION_COUNT=15
 
 # The kernel version (e.g. '2' from 2.6.32 or '3' from 3.2.1)
 readonly KERNEL_VERSION=$(uname -r 2>/dev/null | cut -d'.' -f1)
@@ -312,7 +324,7 @@ LOGFILES="\
   /var/log/yum.log\
   /var/log/openvswitch/ovs-vswitchd.log\
   /var/log/openvswitch/ovsdb-server.log\
-  /var/run/docker/libcontainerd/containerd/events.log\
+  /run/docker/libcontainerd/containerd/events.log\
   /run/containerd/events.log\
   "
 
@@ -367,7 +379,7 @@ CONFIGFILES="\
   $(find /lib/modules -name modules.dep 2>/dev/null)\
   /etc/docker\
   /lib/systemd/system/docker.service\
-  /usr/lib/systemd/system/docker.service\
+  /usr/lib/systemd/system\
   /etc/apparmor.d\
   "
 
@@ -394,6 +406,7 @@ CMDS="uname -a\
   :ip link show\
   :ip ntable\
   :ip a sh\
+  :ip -s -s link\
   :firewall-cmd --list-all\
   :ipcs -a\
   :netstat -pantu\
@@ -441,7 +454,8 @@ CMDS="uname -a\
   :SPident\
   :rpm -qa | sort\
   :sysctl -a\
-  :lsof\
+  :lsof \
+   > '${OUTPUT_FILE_LSOF}'\
   :mount\
   :df -h\
   :df -i\
@@ -451,7 +465,7 @@ CMDS="uname -a\
   :java -version\
   :cat /root/.bash_history\
   :env\
-  :journalctl --all --no-pager --since=$(date -d '5 day ago' +%Y-%m-%d) --until=now --lines=50000 \
+  :journalctl --all --no-pager --lines=100000 --output=short-precise\
    > '${OUTPUT_FILE_JOURNALCTL}'\
   :openssl engine\
   :systemd-delta\
@@ -465,6 +479,10 @@ CMDS="uname -a\
   :docker version\
   :docker stats --no-stream\
   :systemctl status docker.service\
+  :blockdev --report\
+  :lvdisplay\
+  :lspci -vv\
+  :smc_dbg\
   "
 
 ########################################
@@ -480,6 +498,7 @@ VM_CMDS="q userid\
   :q cpus\
   :q srm\
   :q vtod\
+  :q time full\
   :q timezone\
   :q loaddev\
   :q v osa\
@@ -548,6 +567,11 @@ collect_cmdsout() {
     done
     IFS="${ifs_orig}"
 
+    if echo "${RUNTIME_ENVIRONMENT}" | grep -qi "z/VM" >/dev/null 2>&1; then
+        call_run_command "hyptop -b -d 1 -n 5 -f \#,c,m,C:s,M:s,o -S c" "${OUTPUT_FILE_CMD}"
+    else call_run_command "hyptop -b -d 1 -n 5 -f \#,T,c,e,m,C:s,E:s,M:s,o -S c" "${OUTPUT_FILE_CMD}"
+    fi
+
     pr_log_stdout " "
 }
 
@@ -743,6 +767,83 @@ collect_osaoat() {
 }
 
 ########################################
+collect_ethtool() {
+    local network_devices
+    local network_device
+
+    network_devices=$(ls /sys/class/net 2>/dev/null)
+    if which ethtool >/dev/null 2>&1; then
+	if test -n "${network_devices}"; then
+	    pr_syslog_stdout "8 of ${COLLECTION_COUNT}: Collecting ethtool output"
+	    for network_device in ${network_devices}; do
+		call_run_command "ethtool ${network_device}" "${OUTPUT_FILE_ETHTOOL}"
+		call_run_command "ethtool -k ${network_device}" "${OUTPUT_FILE_ETHTOOL}"
+		call_run_command "ethtool -a ${network_device}" "${OUTPUT_FILE_ETHTOOL}"
+		call_run_command "ethtool -c ${network_device}" "${OUTPUT_FILE_ETHTOOL}"
+		call_run_command "ethtool -g ${network_device}" "${OUTPUT_FILE_ETHTOOL}"
+		call_run_command "ethtool -i ${network_device}" "${OUTPUT_FILE_ETHTOOL}"
+		call_run_command "ethtool -l ${network_device}" "${OUTPUT_FILE_ETHTOOL}"
+		call_run_command "ethtool -P ${network_device}" "${OUTPUT_FILE_ETHTOOL}"
+		call_run_command "ethtool -S ${network_device}" "${OUTPUT_FILE_ETHTOOL}"
+		call_run_command "ethtool -T ${network_device}" "${OUTPUT_FILE_ETHTOOL}"
+	    done
+	else
+	    pr_syslog_stdout "8 of ${COLLECTION_COUNT}: Collecting ethtool output skipped - no devices"
+	fi
+    else
+	pr_syslog_stdout "8 of ${COLLECTION_COUNT}: Collecting ethtool output skipped - not available"
+    fi
+
+    pr_log_stdout " "
+}
+
+########################################
+collect_tc() {
+    local network_devices
+    local network_device
+
+    network_devices=$(ls /sys/class/net 2>/dev/null)
+    if which tc >/dev/null 2>&1; then
+	if test -n "${network_devices}"; then
+	    pr_syslog_stdout "9 of ${COLLECTION_COUNT}: Collecting tc output"
+	    for network_device in ${network_devices}; do
+		call_run_command "tc -s qdisc show dev ${network_device}" "${OUTPUT_FILE_TC}"
+	    done
+	else
+	    pr_syslog_stdout "9 of ${COLLECTION_COUNT}: Collecting tc output skipped - no devices"
+	fi
+    else
+	pr_syslog_stdout "9 of ${COLLECTION_COUNT}: Collecting tc output skipped - not available"
+    fi
+
+    pr_log_stdout " "
+}
+
+########################################
+collect_bridge() {
+    local network_devices
+    local network_device
+
+    network_devices=$(ls /sys/class/net 2>/dev/null)
+    if which bridge >/dev/null 2>&1; then
+	if test -n "${network_devices}"; then
+	    pr_syslog_stdout "10 of ${COLLECTION_COUNT}: Collecting bridge output"
+	    for network_device in ${network_devices}; do
+		call_run_command "bridge -d link show dev ${network_device}" "${OUTPUT_FILE_BRIDGE}"
+		call_run_command "bridge -s fdb show dev ${network_device}" "${OUTPUT_FILE_BRIDGE}"
+		call_run_command "bridge -d mdb show dev ${network_device}" "${OUTPUT_FILE_BRIDGE}"
+	    done
+	else
+	    pr_syslog_stdout "10 of ${COLLECTION_COUNT}: Collecting bridge output skipped - no devices"
+	fi
+    else
+	pr_syslog_stdout "10 of ${COLLECTION_COUNT}: Collecting bridge output skipped - not available"
+    fi
+
+    pr_log_stdout " "
+}
+
+########################################
 # OpenVSwitch
 collect_ovs() {
     local br_list
@@ -758,7 +859,7 @@ collect_ovs() {
             :ovsdb-client dump\
             "
     if test -n "${br_list}"; then
-        pr_syslog_stdout "8 of ${COLLECTION_COUNT}: Collecting OpenVSwitch output"
+        pr_syslog_stdout "11 of ${COLLECTION_COUNT}: Collecting OpenVSwitch output"
         IFS=:
           for ovscmd in ${ovscmds}; do
             IFS=${ifs_orig} call_run_command "${ovscmd}" "${OUTPUT_FILE_OVS}.out"
@@ -777,7 +878,7 @@ collect_ovs() {
          IFS="${ifs_orig}"
         done
     else
-        pr_syslog_stdout "8 of ${COLLECTION_COUNT}: Collecting OpenVSwitch output skipped"
+        pr_syslog_stdout "11 of ${COLLECTION_COUNT}: Collecting OpenVSwitch output skipped"
     fi
 
     pr_log_stdout " "
@@ -790,12 +891,12 @@ collect_domain_xml() {
 
     domain_list=$(virsh list --all --name)
     if test -n "${domain_list}"; then
-        pr_syslog_stdout "9 of ${COLLECTION_COUNT}: Collecting domain xml files"
+        pr_syslog_stdout "12 of ${COLLECTION_COUNT}: Collecting domain xml files"
 	  for domain in ${domain_list}; do
 	    call_run_command "virsh dumpxml ${domain}" "${OUTPUT_FILE_XML}_${domain}.xml"
           done
     else
-        pr_syslog_stdout "9 of ${COLLECTION_COUNT}: Collecting domain xml files skipped"
+        pr_syslog_stdout "12 of ${COLLECTION_COUNT}: Collecting domain xml files skipped"
     fi
 
     pr_log_stdout " "
@@ -809,23 +910,23 @@ collect_docker() {
     # call docker inspect for all containers
     item_list=$(docker ps -qa)
     if test -n "${item_list}"; then
-        pr_syslog_stdout "10a of ${COLLECTION_COUNT}: Collecting docker container output"
+        pr_syslog_stdout "13a of ${COLLECTION_COUNT}: Collecting docker container output"
         for item in ${item_list}; do
             call_run_command "docker inspect ${item}" "${OUTPUT_FILE_DOCKER}"
         done
     else
-        pr_syslog_stdout "10a of ${COLLECTION_COUNT}: Collecting docker container output skipped"
+        pr_syslog_stdout "13a of ${COLLECTION_COUNT}: Collecting docker container output skipped"
     fi
 
     # call docker inspect for all networks
     item_list=$(docker network ls -q)
     if test -n "${item_list}"; then
-        pr_syslog_stdout "10b of ${COLLECTION_COUNT}: Collecting docker network output"
+        pr_syslog_stdout "13b of ${COLLECTION_COUNT}: Collecting docker network output"
         for item in ${item_list}; do
             call_run_command "docker network inspect ${item}" "${OUTPUT_FILE_DOCKER}"
         done
     else
-        pr_syslog_stdout "10b of ${COLLECTION_COUNT}: Collecting docker network output skipped"
+        pr_syslog_stdout "13b of ${COLLECTION_COUNT}: Collecting docker network output skipped"
     fi
 
     pr_log_stdout " "
@@ -835,7 +936,7 @@ collect_docker() {
 collect_nvme() {
     local NVME
 
-    pr_syslog_stdout "11 of ${COLLECTION_COUNT}: Collecting nvme output"
+    pr_syslog_stdout "14 of ${COLLECTION_COUNT}: Collecting nvme output"
     call_run_command "nvme list" "${OUTPUT_FILE_NVME}"
 
     for NVME in /dev/nvme[0-9]*; do
@@ -1016,25 +1117,31 @@ environment_setup()
 # create gzip-ped tar file
 create_package()
 {
+    local rc_tar
     pr_stdout "Finalizing: Creating archive with collected data"
     cd "${WORKDIR_BASE}"
 
-    if ! tar -czf "${WORKARCHIVE}" "${WORKDIR_CURRENT}"; then
-	pr_stdout " "
-	pr_stdout "${SCRIPTNAME}: Error: Collection of data failed!"
-	pr_stdout "       The creation of \"${WORKARCHIVE}\" was not successful."
-	pr_stdout "       Please check the directory \"${WORKDIR_BASE}\""
-	pr_stdout "       to provide enough free available space."
+    tar -czf "${WORKARCHIVE}" "${WORKDIR_CURRENT}"
+    rc_tar=$?
+    if [ $rc_tar -eq 0 ]; then
+        chmod 0600 "${WORKARCHIVE}"
+        pr_stdout " "
+        pr_stdout "Collected data was saved to:"
+        pr_stdout " >>  ${WORKARCHIVE}  <<"
+        pr_stdout " "
+        pr_stdout "Review the collected data before sending to your service organization. "
+        pr_stdout " "
+    elif [ $rc_tar -eq 127 ]; then
+        pr_stdout " "
+        pr_stdout "${SCRIPTNAME}: Error: tar command is not available!"
+        pr_stdout "     Please install the corresponding package!"
     else
-	chmod 0600 "${WORKARCHIVE}"
-	pr_stdout " "
-	pr_stdout "Collected data was saved to:"
-	pr_stdout " >>  ${WORKARCHIVE}  <<"
+        pr_stdout " "
+        pr_stdout "${SCRIPTNAME}: Error: Collection of data failed!"
+        pr_stdout "       The creation of \"${WORKARCHIVE}\" was not successful."
+        pr_stdout "       Please check the directory \"${WORKDIR_BASE}\""
+        pr_stdout "       to provide enough free available space."
     fi
-
-    pr_stdout " "
-    pr_stdout "Review the collected data before sending to your service organization. "
-    pr_stdout " "
 }
 
 
@@ -1136,6 +1243,12 @@ collect_configfiles
 
 collect_osaoat
 
+collect_ethtool
+
+collect_tc
+
+collect_bridge
+
 collect_ovs
 
 collect_domain_xml
