Subject: [PATCH] [BZ 87349] dbginfo.sh: Improvements on data collection and speed
From: Wolfgang Taphorn <taphorn@de.ibm.com>

Description: dbginfo.sh: Improvements on data collection and speed
Symptom:     Some runtime information is missing in the output of dbginfo.sh.
             Apart of the functional symptoms, the coding style in the script
             has changed from time to time.
Problem:     * The missing data in the collected output leads to incomplete
               information for debugging an issue.
             * Issue on some distributions to detect if the debugfs is mounted
             * Different coding styles introduced quick fixes the affected the
               speed of the data collection.
Solution:    * Adding the following additional tools for execution to collect
               further information:
               - env
               - last
               - lsshut
               - ls /dev
               - multipath -t
               - nm-tool
               - pvscan -vvv
               - runlevel
             * Adding the following additional z/VM CP commands for execution
               to collect further information:
               - 'q dumpdev'
               - 'q loaddev'
               - 'q reorder %VMUSERID%'
               - 'q quickdsp %VMUSERID%'
             * Collecting the following additional configuration files:
               - /etc/anacrontab
               - /etc/auto.*
               - /etc/crypttab
               - /etc/dasd.conf
               - /etc/depmod.d
               - /etc/dracut.conf
               - /etc/dracut.conf.d
               - /etc/hba.conf
               - /etc/inittab
               - /etc/iscsi
               - /etc/mdadm.conf
               - /etc/mtab
               - /etc/multipath
               - /etc/networks
               - /etc/rc.local
               - /etc/*release
               - /etc/rsyslog.conf
               - /etc/rsyslog.d
               - /etc/sysctl.d
               - /etc/udev*
             * Collecting the following additional log files:
               - /var/log/audit
               - /var/log/anaconda.*
               - /var/log/audit
               - /var/log/boot*
               - /var/log/cron*
               - /var/log/dmesg*
               - /var/log/dracut.log*
               - /var/log/yum.log
             * Collecting the following additional files in proc:
               - /proc/cio_ignore
               - /proc/iomem
               - /proc/mdstat
               - /proc/swaps
               - /proc/sys/kernel/*
               - /proc/sys/vm/*
               - /proc/zoneinfo
             * Sorting the output for some commands like 'rpm', 'find /dev..'
               and 'find /boot...'.
             * Unify the coding style within the script.
             * Lookup for mounted debugfs is now in /proc/mounts instead of
               parsing the output of 'mount'.
             * Simplified some loops to speed up the script in various
               scenarios.
             * Sorted the list of variables 'PROCFILES', 'LOGFILES' and
               'CONFIGFILES'.
Reproduction: -
Upstream-ID:  -
Problem-ID:   87349

Signed-off-by: Wolfgang Taphorn <taphorn@de.ibm.com>
---
 scripts/dbginfo.sh | 1276 +++++++++++++++++++++++++++++------------------------
 1 file changed, 705 insertions(+), 571 deletions(-)

--- a/scripts/dbginfo.sh
+++ b/scripts/dbginfo.sh
@@ -1,658 +1,792 @@
-#!/bin/bash 
+#!/bin/sh
 ###############################################################################
-# collect some configuration, trace, and debug
-# information about the S390 Linux system
+# Copyright IBM Corp. 2002, 2012
 #
-# Copyright IBM Corp. 2002, 2011.
+# Collect some configuration, trace, and debug information about the
+# Linux on System z machine
+#
+# Author(s): Sven Schuetz                                <sven[at]de.ibm.com>
+#            Wolfgang Taphorn                         <taphorn[at]de.ibm.com>
+#            Stefan Reimbold                  <stefan.reimbold[at]de.ibm.com>
+#            Susanne Wintenberger                     <swinten[at]de.ibm.com>
+#            Michael Mueller                             <mimu[at]de.ibm.com>
+#
+# This file is part of the s390-tools
+#
+# s390-tools is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# s390-tools is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with s390-tools; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 ###############################################################################
 
+# The kernel release version as delivered from uname -r
+readonly KERNEL_RELEASE_VERSION="$(uname -r 2>/dev/null)"
 
-# variables
+########################################
+# Global used variables
 #
 
-SCRIPTNAME="dbginfo.sh"
-WORKDIR=DBGINFO-`date +%Y-%m-%d-%H-%M-%S`-`hostname`
-rc_check_zvm=0
-# write output to following directory
-WORKPFX=/tmp
-WORKPATH=$WORKPFX/$WORKDIR
-# write output to following files
-CMDOUTPUT=runtime.out
-VM_CMDOUTPUT=zvm_runtime.out
-SYSFSFILELIST=sysfsfiles.out
-FCPCONFOUTPUT=scsi
-MOUNT_POINT_DEBUGFS="/sys/kernel/debug"
-kernel_version_tmp=0
-
-LOGFILE=dbginfo.log
-
-# procfs entries to be collected (except s390dbf and scsi)
-PROCFILES=" \
-    /proc/sysinfo \
-    /proc/version \
-    /proc/cpuinfo \
-    /proc/meminfo \
-    /proc/buddyinfo \
-    /proc/slabinfo \
-    /proc/modules \
-    /proc/mounts \
-    /proc/partitions \
-    /proc/stat \
-    /proc/devices \
-    /proc/misc \
-    /proc/qeth \
-    /proc/qeth_perf \
-    /proc/qeth_ipa_takeover \
-    /proc/cmdline \
-    /proc/crypto \
-    /proc/diskstats \
-    /proc/interrupts \
-    /proc/dasd/devices \
-    /proc/dasd/statistics \
-    /proc/sys/kernel/spin_retry \
-    /proc/sys/vm/cmm_pages \
-    /proc/sys/vm/cmm_timed_pages \
-    /proc/sys/vm/cmm_timeout \
-    /proc/sys/vm/swappiness \
-    /proc/net/vlan/* \
-    /proc/net/bonding/* \
-    "
-PROCFILES_24=" \
-    /proc/subchannels \
-    /proc/chpids \
-    /proc/chandev \
-    /proc/ksyms \
-    /proc/lvm/global \
-    "
-
-# log files to be collected
-LOGFILES=" \
-    /var/log/messages* \
-    /var/log/boot.log* \
-    /var/log/IBMtape.trace \
-    /var/log/IBMtape.errorlog \
-    /var/log/lin_tape.trace \
-    /var/log/lin_tape.errorlog \
-    /var/log/sa/* \
-    "
-# config files to be collected;
-# additional all files "modules.dep" are collected
-CONFIGFILES=" \
-    /etc/ccwgroup.conf \
-    /etc/chandev.conf \
-    /etc/modules.conf \
-    /etc/modprobe.conf* \ 
-    /etc/fstab \
-    /etc/syslog.conf \
-    /etc/sysconfig \
-    /etc/crontab \
-    /etc/cron.* \
-    /etc/exports \
-    /etc/sysctl.conf \
-    /etc/zipl.conf \
-    /etc/lvm \
-    /etc/IBMtaped.conf \
-    /etc/lin_taped.conf \
-    /etc/multipath.conf \
-    /etc/rc.d \
-    /etc/zfcp.conf \
-    /etc/udev/rules.d \
-    "
-
-#sysfs files NOT to be collected
-SYSFSFILEEXCLUDES="\
-	trace_pipe\
-	"
+# The general name of this script
+readonly SCRIPTNAME="${0##*/}"
+
+# The base working directory
+readonly WORKDIR_BASE="/tmp/"
+
+# The current working directory for the actual script execution
+readonly WORKDIR_CURRENT="DBGINFO-$(date +%Y-%m-%d-%H-%M-%S)-$(hostname)"
+
+# The current path where the collected information is put together
+readonly WORKPATH="${WORKDIR_BASE}${WORKDIR_CURRENT}/"
+
+# The current TAR archive that finally includes all collected information
+readonly WORKARCHIVE="${WORKDIR_BASE}${WORKDIR_CURRENT}.tgz"
+
+# The log file of activities from this script execution
+readonly LOGFILE="${WORKPATH}dbginfo.log"
+
+# File that includes output of Linux commands
+readonly OUTPUT_FILE_CMD="${WORKPATH}runtime.out"
+
+# File that includes output of z/VM commands (if running in z/VM)
+readonly OUTPUT_FILE_VMCMD="${WORKPATH}zvm_runtime.out"
+
+# File that includes content of files from sysfs
+readonly OUTPUT_FILE_SYSFS="${WORKPATH}sysfsfiles.out"
+
+# File that includes content of zFCP settings
+readonly OUTPUT_FILE_FCPCONF="${WORKPATH}scsi"
+
+# File that includes content of OSA OAT
+readonly OUTPUT_FILE_OSAOAT="${WORKPATH}osa_oat"
+
+# Mount point of the debug file system
+readonly MOUNT_POINT_DEBUGFS="/sys/kernel/debug"
+
+# The amount of steps running the whole collections
+readonly COLLECTION_COUNT=7
+
+# The kernel version (e.g. '2' from 2.6.32 or '3' from 3.2.1)
+readonly KERNEL_VERSION="$(uname -r 2>/dev/null | cut -d'.' -f1)"
+
+# The kernel major revision number (e.g. '6' from 2.6.32 or '2' from 3.2.1)
+readonly KERNEL_MAJOR_REVISION="$(uname -r 2>/dev/null | cut -d'.' -f2)"
+
+# The kernel mainor revision number (e.g. '32' from 2.6.32 or '1' from 3.2.1)
+readonly KERNEL_MINOR_REVISION="$(uname -r 2>/dev/null | cut -d'.' -f3 | sed s/[^0-9].*//g)"
+
+# Is this kernel supporting sysfs - since 2.4 (0=yes, 1=no)
+if test ${KERNEL_VERSION} -lt 2 ||
+    ( test  ${KERNEL_VERSION} -eq 2 && test ${KERNEL_MAJOR_REVISION} -le 4 ); then
+    readonly LINUX_SUPPORT_SYSFS=1
+else
+    readonly LINUX_SUPPORT_SYSFS=0
+fi
+
+# Is this kernel potentially using the /sys/kernel/debug feature - since 2.6.13 (0=yes, 1=no)
+if test ${KERNEL_VERSION} -lt 2 ||
+    ( test ${KERNEL_VERSION} -eq 2 &&
+	( test ${KERNEL_MAJOR_REVISION} -lt 6 ||
+	    ( test ${KERNEL_MAJOR_REVISION} -eq 6 && test ${KERNEL_MINOR_REVISION} -lt 13 ))); then
+    readonly LINUX_SUPPORT_SYSFSDBF=1
+else
+    readonly LINUX_SUPPORT_SYSFSDBF=0
+fi
+
+# Is this Linux on System z under z/VM (0=yes, 1=no)
+if grep -q 'z/VM' /proc/sysinfo 2>/dev/null; then
+    readonly LINUX_ON_ZVM=0
+else
+    readonly LINUX_ON_ZVM=1
+fi
+
+########################################
+
+# Collection of proc fs entries
+PROCFILES="\
+  /proc/buddyinfo\
+  /proc/cio_ignore\
+  /proc/cmdline\
+  /proc/cpuinfo\
+  /proc/crypto\
+  /proc/dasd/devices\
+  /proc/dasd/statistics\
+  /proc/devices\
+  /proc/diskstats\
+  /proc/interrupts\
+  /proc/iomem\
+  /proc/mdstat\
+  /proc/meminfo\
+  /proc/misc\
+  /proc/modules\
+  /proc/mounts\
+  /proc/net/vlan\
+  /proc/net/bonding\
+  /proc/partitions\
+  /proc/qeth\
+  /proc/qeth_perf\
+  /proc/qeth_ipa_takeover\
+  /proc/slabinfo\
+  /proc/stat\
+  /proc/swaps\
+  /proc/sys/kernel\
+  /proc/sys/vm\
+  /proc/sysinfo\
+  /proc/version\
+  /proc/zoneinfo\
+  "
+
+# Adding files to PROCFILES in case scsi devices are available
+if test -e /proc/scsi; then
+    PROCFILES="${PROCFILES}\
+      `find /proc/scsi -type f -perm +0444`\
+      "
+fi
+
+# Adding files to PROCFILES in case we run on Kernel 2.4 or older
+if test ${LINUX_SUPPORT_SYSFS} -eq 1; then
+    PROCFILES="${PROCFILES}\
+      /proc/chpids\
+      /proc/chandev\
+      /proc/ksyms\
+      /proc/lvm/global\
+      /proc/subchannels\
+      "
+fi
+
+# Adding s390dbf files to PROCFILE in case we run on Kernel lower than 2.6.13
+if test ${LINUX_SUPPORT_SYSFSDBF} -eq 1; then
+    if test -e /proc/s390dbf; then
+	PROCFILES="${PROCFILES}\
+          `find /proc/s390dbf -type f -not -path \"*/raw\" -not -path \"*/flush\"`\
+          "
+    fi
+fi
+
+########################################
+
+LOGFILES="\
+  /var/log/anaconda.*\
+  /var/log/audit\
+  /var/log/boot*\
+  /var/log/cron*\
+  /var/log/dmesg*\
+  /var/log/dracut.log*\
+  /var/log/IBMtape.trace\
+  /var/log/IBMtape.errorlog\
+  /var/log/lin_tape.trace\
+  /var/log/lin_tape.errorlog\
+  /var/log/messages*\
+  /var/log/sa\
+  /var/log/yum.log\
+  "
+
+########################################
+
+CONFIGFILES="\
+  /etc/anacrontab\
+  /etc/auto.*\
+  /etc/ccwgroup.conf\
+  /etc/chandev.conf\
+  /etc/cron.*\
+  /etc/crontab\
+  /etc/crypttab\
+  /etc/dasd.conf\
+  /etc/depmod.d\
+  /etc/dracut.conf\
+  /etc/dracut.conf.d\
+  /etc/exports\
+  /etc/fstab\
+  /etc/hba.conf\
+  /etc/IBMtaped.conf\
+  /etc/iscsi\
+  /etc/lin_taped.conf\
+  /etc/lvm\
+  /etc/mdadm.conf\
+  /etc/modprobe.conf*\
+  /etc/modprobe.d\
+  /etc/modules.conf\
+  /etc/mtab\
+  /etc/multipath.conf\
+  /etc/multipath\
+  /etc/networks\
+  /etc/rc.d\
+  /etc/rc.local\
+  /etc/rsyslog.conf\
+  /etc/rsyslog.d\
+  /etc/syslog.conf\
+  /etc/sysconfig\
+  /etc/sysctl.conf\
+  /etc/sysctl.d\
+  /etc/udev*\
+  /etc/zipl.conf\
+  /etc/zfcp.conf\
+  /etc/*release\
+  `find /lib/modules -name modules.dep`\
+  "
+
+########################################
 
-# collect output of following commands;
-# commands are separated by ':'
 CMDS="uname -a\
-    :uptime\
-    :iptables -L\
-    :ulimit -a\
-    :ps -eo pid,tid,nlwp,policy,user,tname,ni,pri,psr,sgi_p,stat,wchan,start_time,time,pcpu,pmem,vsize,size,rss,share,command\
-    :ps axX\
-    :dmesg -s 1048576\
-    :ifconfig -a\
-    :route -n\
-    :ip route list\
-    :ip rule list\
-    :ip neigh list\
-    :ip link show\
-    :ipcs -a\
-    :netstat -pantu\
-    :netstat -s\
-    :dmsetup ls\
-    :dmsetup ls --tree\
-    :dmsetup table\
-    :dmsetup table --target multipath\
-    :dmsetup status\
-    :multipath -v6 -ll\
-    :multipath -d\
-    :lsqeth\
-    :lscss\
-    :lsdasd\
-    :ziorep_config -ADM\
-    :lsmod\
-    :lsdev\
-    :lsscsi\
-    :lstape\
-    :lszfcp\
-    :lszfcp -D\
-    :lszfcp -V\
-    :SPident\
-    :rpm -qa\
-    :sysctl -a\
-    :lsof\
-    :mount\
-    :df -h\
-    :pvpath -qa
-    :ls -la /boot\
-    :java -version\
-    :cat /root/.bash_history\
-    "
+  :uptime\
+  :runlevel\
+  :iptables -L\
+  :ulimit -a\
+  :ps -eo pid,tid,nlwp,policy,user,tname,ni,pri,psr,sgi_p,stat,wchan,start_time,time,pcpu,pmem,vsize,size,rss,share,command\
+  :ps axX\
+  :dmesg -s 1048576\
+  :last\
+  :lsshut\
+  :ifconfig -a\
+  :nm-tool
+  :route -n\
+  :ip route list\
+  :ip rule list\
+  :ip neigh list\
+  :ip link show\
+  :ipcs -a\
+  :netstat -pantu\
+  :netstat -s\
+  :dmsetup ls\
+  :dmsetup ls --tree\
+  :dmsetup table\
+  :dmsetup table --target multipath\
+  :dmsetup status\
+  :multipath -v6 -ll\
+  :multipath -d\
+  :multipath -t\
+  :lsqeth\
+  :lscss\
+  :lsdasd\
+  :ziorep_config -ADM\
+  :lsmod\
+  :lsdev\
+  :lsscsi\
+  :lstape\
+  :lszfcp\
+  :lszfcp -D\
+  :lszfcp -V\
+  :pvscan -vvv\
+  :SPident\
+  :rpm -qa | sort\
+  :sysctl -a\
+  :lsof\
+  :mount\
+  :df -h\
+  :pvpath -qa\
+  :find /boot -print0 | sort -z | xargs -0 -n 10 ls -ld\
+  :find /dev -print0 | sort -z | xargs -0 -n 10 ls -ld\
+  :java -version\
+  :cat /root/.bash_history\
+  :env\
+  :ziomon_fcpconf -o ${OUTPUT_FILE_FCPCONF}\
+  "
 
-#z/VM commands
+########################################
 
 VM_CMDS="q userid\
-	:q users\
-	:q v osa\
-	:q v dasd\
-	:q v crypto\
-	:q v fcp\
-	:q v pav\
-	:q v st\
-	:q st\
-	:q xstore\
-	:q xstore user system\
-	:q sxspages\
-	:q v sw\
-	:q vmlan\
-	:q vswitch\
-	:q vswitch details\
-	:q vswitch access\
-	:q vswitch active\
-	:q vswitch accesslist\
-	:q vswitch promiscuous\
-	:q vswitch controller\
-	:q port group all active details\
-	:q set\
-	:q comm\
-	:q controller all\
-	:q cplevel\
-	:q cpus\
-	:q fcp\
-	:q frames\
-	:q lan\
-	:q lan all details\
-	:q lan all access\
-	:q cache\
-	:q nic\
-	:q pav\
-	:q privclass\
-	:q proc\
-	:q qioass\
-	:q spaces\
-	:q swch all\
-	:q timezone\
-	:q trace\
-	:q vtod\
-	:q srm\
-	:q mdcache\
-	:q alloc page\
-	:q alloc spool\
-	:q dump\
-	:q reorder \
-	:ind load\
-	:ind sp\
-	:ind user\
-	"
-#
-# function definitions
-#
+  :q users\
+  :q privclass\
+  :q cplevel\
+  :q cpus\
+  :q srm\
+  :q vtod\
+  :q timezone\
+  :q loaddev\
+  :q v osa\
+  :q v dasd\
+  :q v crypto\
+  :q v fcp\
+  :q v pav\
+  :q v sw\
+  :q v st\
+  :q st\
+  :q xstore\
+  :q xstore user system\
+  :q sxspages\
+  :q vmlan\
+  :q vswitch\
+  :q vswitch details\
+  :q vswitch access\
+  :q vswitch active\
+  :q vswitch accesslist\
+  :q vswitch promiscuous\
+  :q vswitch controller\
+  :q port group all active details\
+  :q set\
+  :q comm\
+  :q controller all\
+  :q fcp\
+  :q frames\
+  :q lan\
+  :q lan all details\
+  :q lan all access\
+  :q cache\
+  :q nic\
+  :q pav\
+  :q proc\
+  :q qioass\
+  :q spaces\
+  :q swch all\
+  :q trace\
+  :q mdcache\
+  :q alloc page\
+  :q alloc spool\
+  :q dump\
+  :q dumpdev\
+  :q reorder VMUSERID\
+  :q quickdsp VMUSERID\
+  :ind load\
+  :ind sp\
+  :ind user\
+  "
 
-# print version info
-printversion()
-{
-    cat <<EOF
-$SCRIPTNAME: Debug information script version %S390_TOOLS_VERSION%
-Copyright IBM Corp. 2002, 2011
-EOF
-}
 
-# print usage and help
-printhelp()
-{
-    cat <<EOF
-Usage: $SCRIPTNAME [OPTIONS]
+###############################################################################
 
-This script collects some runtime, configuration and
-trace information about your Linux for zSeries system
-for debugging purposes. It also traces information 
-about z/VM if the Linux runs under z/VM.
-This information is written to a file
-	/tmp/DBGINFO-[date]-[time]-[hostname].tgz
-where [date] and [time] are the date and time when debug
-data is collected. [hostname] indicates the hostname of the
-system the data was collected from.
+########################################
+collect_cmdsout() {
+    local cmd
+    local ifs_orig="${IFS}"
 
-Options:
+    echo " 1 of ${COLLECTION_COUNT}: Collecting command output"
 
-        -h, --help       print this help
-        -v, --version    print version information
+    IFS=:
+    for cmd in ${CMDS}; do
+	IFS=${ifs_orig}	call_run_command "${cmd}" "${OUTPUT_FILE_CMD}"
+    done
+    IFS="${ifs_orig}"
 
-Please report bugs to: linux390@de.ibm.com
-EOF
+    echo
 }
 
-# copy file $1 to $WORKPATH
-collect_file_contents()
-{
-    echo "  $1" >>  $LOGFILE
-    if [ ! -e $1 ]
-    then
-	echo "  WARNING: No such file: \"$1\"" >> $LOGFILE
-	return 1
-    elif [ ! -r $1 ]
-    then
-	echo "  WARNING: Permission denied: \"$1\"" >> $LOGFILE
-	return 1
-    else
-	if [ ! -e $WORKPATH`dirname $1` ]
-	then
-	    mkdir --parents $WORKPATH`dirname $1`
-	fi
-	cp -r -d -L --parents $1 $WORKPATH 2>> $LOGFILE
-# 	head -c 10m $1 >> $2
-	if [ $? -ne 0 ]
-	then
-	    echo "  WARNING: cp failed for file: \"$1\"" >> $LOGFILE
-	    return 1
+
+########################################
+collect_vmcmdsout() {
+    local vm_command
+    local cp_command
+    local vm_cmds
+    local vm_userid
+    local module_loaded=1
+    local ifs_orig="${IFS}"
+
+    if test ${LINUX_ON_ZVM} -eq 0; then
+	echo " 2 of ${COLLECTION_COUNT}: Collecting z/VM command output"
+
+	if type vmcp >/dev/null 2>&1; then
+	    cp_command="vmcp"
+	    if ! lsmod 2>/dev/null | grep -q vmcp; then
+		modprobe vmcp && module_loaded=0 && sleep 2
+	    fi
+	elif type hcp >/dev/null 2>&1; then
+	    cp_command="hcp"
+	    if ! lsmod 2>/dev/null | grep -q cpint; then
+		modprobe cpint && module_loaded=0 && sleep 2
+	    fi
 	else
-	    return 0
+	    echo "  WARNING: No program found to communicate to z/VM CP"
+	    echo "  WARNING: Skipping the collection of z/VM command output"
+	    echo ""
+	    return 1
 	fi
+	VMUSERID="$(${cp_command} q userid | sed -ne 's/^\([^[:space:]]*\).*$/\1/p')"
+
+	vm_cmds="$(echo ${VM_CMDS} | sed "s/VMUSERID/${VMUSERID}/g")"
+
+	IFS=:
+	for vm_command in ${vm_cmds}; do
+	    IFS=${ifs_orig}
+	    local cp_buffer_size=2
+	    local rc_buffer_size=2
+	    while test ${rc_buffer_size} -eq 2 && test ${cp_buffer_size} -lt 1024; do
+		cp_buffer_size=$(( ${cp_buffer_size} * 2 ))
+
+		eval ${cp_command} -b ${cp_buffer_size}k ${vm_command} >/dev/null 2>&1
+		rc_buffer_size=$?
+	    done
+	    call_run_command "${cp_command} -b ${cp_buffer_size}k ${vm_command}" "${OUTPUT_FILE_VMCMD}"
+	    IFS=:
+	done
+	IFS=${ifs_orig}
+
+	if test ${module_loaded} -eq 0 && test "x${cp_command}" = "xhcp"; then
+	    rmmod cpint
+	elif test ${module_loaded} -eq 0 && test "x${cp_command}" = "xvmcp"; then
+	    rmmod vmcp
+	fi
+    else
+	echo " 2 of ${COLLECTION_COUNT}: Running in LPAR, no z/VM command output collected"
     fi
+
+    echo
 }
 
-# append output of command $1 to file $2
-collect_cmd_output()
-{
-    local raw_cmd=`echo $1 | awk '{ print $1 }'`;
 
-    echo "#######################################################">>$2
+########################################
+collect_procfs() {
+    local file_name
 
-    # check if command exists
-    which $raw_cmd >/dev/null 2>&1;
-    if [ $? -ne 0 ]
-    then
-        # check if command is a builtin
-        command -v $raw_cmd >/dev/null 2>&1;
-        [ $? -ne 0 ] && echo "command '$raw_cmd' not available">>$2 && return 1;
-    fi
+    echo " 3 of ${COLLECTION_COUNT}: Collecting procfs"
 
-    echo "$USER@$HOST> $1">>$2
-    $1 1>>$2 2>&1
-    if [ $? -ne 0 ]
-    then
-	echo "  WARNING: Command not successfully completed: \"$1\"">>  $2
-        echo "" >>$2
-	return 1
-    else
-        echo "" >>$2
-	return 0
-    fi
-    
+    for file_name in ${PROCFILES}; do
+	call_collect_file "${file_name}" "${LOGFILE}"
+    done
+
+    echo
 }
 
-# check cmd line arguments
-check_cmdline()
-{
-    # currently no options available
-    if [ $# -eq 1 ]
-    then
-	if [ $1 = '-h' ] ||  [ $1 = '--help' ]
-	then
-	    printhelp
-	elif [ $1 = '-v' ] ||  [ $1 = '--version' ]
-	then
-	    printversion
-	else
-	    echo 
-	    echo "$SCRIPTNAME: invalid option $1"
-	    echo "Try '$SCRIPTNAME --help' for more information"
-	    echo
-	    exit 1
+
+########################################
+collect_sysfs() {
+    local debugfs_mounted=0;
+    local file_name
+    local file_names
+    local rc_mount
+
+    # Requires kernel version newer then 2.4
+    if test ${LINUX_SUPPORT_SYSFS} -eq 0; then
+	echo " 4 of ${COLLECTION_COUNT}: Collecting sysfs"
+	# Requires kernel version of 2.6.13 or newer
+	if test ${LINUX_SUPPORT_SYSFSDBF} -eq 0; then
+	    if ! grep -qE "${MOUNT_POINT_DEBUGFS}.*debugfs" /proc/mounts; then
+		if mount -t debugfs debugfs ${MOUNT_POINT_DEBUGFS} >/dev/null 2>&1; then
+		    sleep 2
+		    debugfs_mounted=1;
+		else
+		    echo "  WARNING: \"Unable to mount debugfs ${MOUNT_POINT_DEBUGFS}\""
+		fi
+	    fi
 	fi
-	exit 0
-    elif [[ $# -ge 1 ]]
-    then
-	echo "ERROR: Invalid number of arguments"
-	echo 
-	printhelp
-	exit 1
-    fi    
-}
 
-# change into temporary directory; if necessary create the directory
-prepare_workdir()
-{
-    if [ ! -e $WORKPFX ]
-    then
-    	mkdir $WORKPFX
-    fi
-	    	
-    if [ -e $WORKPATH ]
-    then
-	# remove old stuff
-	echo "Clean up target directory $WORKPATH"
-	rm -rf $WORKPATH/*
+	call_run_command "find /sys -print0 | sort -z | xargs -0 -n 10 ls -ld" ${OUTPUT_FILE_SYSFS}
+
+	find /sys -noleaf -type f -perm +444 -a -not -name '*trace_pipe' | while IFS= read -r file_name; do
+	    call_collect_file "${file_name}" "${LOGFILE}"
+	done
+
+	if test ${debugfs_mounted} -eq 1; then
+	    umount ${MOUNT_POINT_DEBUGFS}
+	fi
     else
-	echo "Create target directory $WORKPATH"
-	mkdir $WORKPATH
+	echo " 4 of ${COLLECTION_COUNT}: Collecting sysfs skipped. Kernel $(uname -r) must be newer than 2.4"
     fi
-    echo "Change to target directory $WORKPATH"
-    cd $WORKPATH
+
+    echo
 }
 
-# collect single proc fs entries
-# (PRCFILES should not contain /proc/scsi and /proc/s390dbf)
-collect_procfs()
-{
-    echo "Get procfs entries" | tee -a $LOGFILE
-    for i in $*
-    do
-	collect_file_contents $i
+
+########################################
+collect_logfiles() {
+    local file_name
+
+    echo " 5 of ${COLLECTION_COUNT}: Collecting log files"
+
+    for file_name in ${LOGFILES}; do
+	call_collect_file "${file_name}" "${LOGFILE}"
     done
+
+    echo
 }
 
-# collect procfs entries of /proc/s390dbf
-collect_s390dbf()
-{
-    echo "Get entries of /proc/s390dbf" | tee -a $LOGFILE
-    if [ -e /proc/s390dbf ]
-    then
-	for i in `find /proc/s390dbf -type f \
-                  -not -path "*/raw" -not -path "*/flush"`
-	do
-	    collect_file_contents $i
- 	done
-    else
-	echo "  WARNING: /proc/s390dbf not found" | tee -a $LOGFILE
-    fi
+########################################
+collect_configfiles() {
+    local file_name
+
+    echo " 6 of ${COLLECTION_COUNT}: Collecting config files"
+
+    for file_name in ${CONFIGFILES}; do
+	call_collect_file "${file_name}" "${LOGFILE}"
+    done
+
+    echo
 }
 
-# collect procfs entries of /proc/scsi
-collect_procfs_scsi()
-{
-    echo "Get entries of /proc/scsi" | tee -a $LOGFILE
-    if [ -e /proc/scsi ]
-    then
-	for i in `find /proc/scsi -type f \
-                  -perm +0444`
-	do
-	    collect_file_contents $i
-	done
+########################################
+collect_osaoat() {
+    local network_devices="$(lsqeth 2>&1 | grep "Device name" | sed 's/.*:[[:space:]]\+\(.*\)[[:space:]]\+/\1/g')"
+    local network_device
+
+    if which qethqoat >/dev/null 2>&1; then
+	if test -n "${network_devices}"; then
+	    echo " 7 of ${COLLECTION_COUNT}: Collecting osa oat output"
+	    for network_device in ${network_devices}; do
+		call_run_command "qethqoat ${network_device}" ${OUTPUT_FILE_OSAOAT}.out &&
+		call_run_command "qethqoat -r ${network_device}" ${OUTPUT_FILE_OSAOAT}_${network_device}.raw
+	    done
+	else
+	    echo " 7 of ${COLLECTION_COUNT}: Collecting osa oat output skipped - no devices"
+	fi
     else
-	echo "  WARNING: /proc/scsi not found" >> $LOGFILE
+	echo " 7 of ${COLLECTION_COUNT}: Collecting osa oat output skipped - not available"
     fi
-}
 
-#check for excluded files in sysfs
-check_for_excludes()
-{
-	for filename in ${SYSFSFILEEXCLUDES[@]}
-	do
-		if [ `basename $1` = $filename ]
-		then
-			return 1
-		fi
-	done
-	return 0
+    echo
 }
 
-# collect sysfs entries
-collect_sysfs()
-{
-    local rc_mount=""
-    # check if debugfs is mounted
-    mount | grep -q $MOUNT_POINT_DEBUGFS
-    rc_mount=$?
-    if [ $rc_mount -eq 1 ]; then
-	if [ $kernel_version_tmp -ge 13 ] || [ "${kernel_version:0:1}" \> 2 ] ; then
-		mount -t debugfs debugfs $MOUNT_POINT_DEBUGFS
-	fi
-    fi
 
-    echo "Get file list of /sys" | tee -a $LOGFILE
-	collect_cmd_output "ls -Rl /sys" $SYSFSFILELIST
-    
-    echo "Get entries of /sys" | tee -a $LOGFILE
-    for i in `find /sys -noleaf -type f -perm +444`
-    	do
-    		if [ -e $i ]
-		then
-			if check_for_excludes $i
-			then
-		    		collect_file_contents $i
-			fi
-		else
-			echo "  WARNING: $i not found" | tee -a $LOGFILE
-    		fi
-	done
-    #unmount debugfs if not mounted at the beginning
-    if [ $rc_mount -eq 1 ]; then
-	if [ $kernel_version_tmp -ge 13 ] || [ "${kernel_version:0:1}" \> 2 ] ; then
-		umount $MOUNT_POINT_DEBUGFS
+########################################
+call_run_command() {
+    local cmd="${1}"
+    local logfile="${2}"
+    local raw_cmd="$(echo ${cmd} | sed -ne 's/^\([^[:space:]]*\).*$/\1/p')"
+
+    echo "#######################################################" >> ${logfile}
+    echo "${USER}@${HOSTNAME}> ${cmd}" >> ${logfile}
+
+    # check if command exists
+    if ! which ${raw_cmd} >/dev/null 2>&1; then
+        # check if command is a builtin
+	if ! command -v ${raw_cmd} >/dev/null 2>&1; then
+	    echo "  WARNING: Command \"${raw_cmd}\" not available" >> ${logfile}
+	    echo "" >> ${logfile}
+	    return 1;
 	fi
     fi
 
-}
-
-# collect output of commands
-collect_cmdsout()
-{
-    local commands=$1
-    local outputfile=$2
-    local buffersize=2
-    local rc_buffer_size=2
-    local check_buffer=""
-    if [ $rc_check_zvm -eq 1 ]; then
-    	echo "Saving z/VM runtime information into $outputfile" | tee -a $LOGFILE
+    if ! eval ${cmd} 1>>${logfile} 2>&1; then
+	echo "  WARNING: Command \"${cmd}\" failed" >> ${logfile}
+	echo "" >> ${logfile}
+	return 1
     else
-    	echo "Saving runtime information into $outputfile" | tee -a $LOGFILE
+	echo "" >> ${logfile}
+	return 0
     fi
-    _IFS_ORIG=$IFS
-    IFS=:
-    for i in $commands
-    do
-        IFS=$_IFS_ORIG
-        if [ $rc_check_zvm -eq 1 ]; then 
-            buffersize=2
-            rc_buffer_size=2
-            echo "$i" | grep reorder > /dev/null 2>&1
-            reorder=$?
-            if [ $reorder -eq 0 ]; then
-                vmcp_userid=`vmcp q userid 2>/dev/null | awk '{print $1}'`
-            fi
-            while [ $rc_buffer_size -eq 2 ]; do
-                if [ $buffersize -lt 1024 ]; then
-                    let buffersize=$buffersize*2
-                else
-                    break
-                fi
-                if [ $reorder -eq 0 ]; then
-                    check_buffer="$cp_tool -b "$buffersize"k $i $vmcp_userid"
-                else
-                    check_buffer="$cp_tool -b "$buffersize"k $i"
-                fi
-                $check_buffer >/dev/null 2>&1
-                rc_buffer_size=$?
-            done
-            if [ $reorder -eq 0 ]; then
-                i="$cp_tool -b "$buffersize"k $i $vmcp_userid"
-            else
-                i="$cp_tool -b "$buffersize"k $i"
-            fi
-        fi
-	collect_cmd_output "$i" "$outputfile"
-	IFS=:
-    done
-    IFS=$_IFS_ORIG
 }
 
-# config files and module dependencies
-collect_config()
-{
-    echo "Copy config files" | tee -a $LOGFILE
-    for i in $CONFIGFILES
-    do
-	collect_file_contents $i
-    done
-    for i in `find /lib/modules -name modules.dep`
-    do
-	collect_file_contents $i
-    done
-}
 
-# Check if we run under z/VM and which Linux cp tool is installed
-check_zvm()
-{
-    cp_tool=''
-    echo "Check if we run under z/VM" | tee -a $LOGFILE
-    # Are we running under z/VM 
-    cat /proc/sysinfo | grep -q "z/VM"
-    if [ $? -eq 1 ] 
-    then
-	echo " Running in LPAR" | tee -a $LOGFILE
-	return 0 
+########################################
+call_collect_file() {
+    local directory_name
+    local file_name="${1}"
+    local logfile="${2}"
+
+    echo "  ${file_name}" >> ${logfile}
+    if test ! -e "${file_name}"; then
+	echo "  WARNING: No such file: \"${file_name}\"" >> ${logfile}
+	return 1
+    elif test ! -r "${file_name}"; then
+	echo "  WARNING: Permission denied: \"${file_name}\"" >> ${logfile}
+	return 1
     else
-	echo " Running under z/VM" | tee -a $LOGFILE
-	# Is vmcp installed
-	which vmcp > /dev/null 2>&1
-    	if [ $? -eq 1 ] 
-    	then
-		# is hcp installed
-		which hcp > /dev/null 2>&1
-		if [ $? -eq 1 ]
- 		then
-			echo " No cp tool installed" | tee -a $LOGFILE 
-			return 0
-		else
-			cp_tool=hcp
-			echo " Installed CP tool: $cp_tool" | tee -a $LOGFILE 
-			return 1
-		fi
+	directory_name=$(dirname "${file_name}")
+	if test ! -e "${WORKPATH}${directory_name}"; then
+	    mkdir -p "${WORKPATH}${directory_name}"
+	fi
+	if ! cp -r -d -L --parents "${file_name}" "${WORKPATH}" 2>> ${logfile}; then
+	    echo "  WARNING: cp failed for file: \"${file_name}\"" >> ${logfile}
+	    return 1
 	else
-		cp_tool=vmcp
-		echo " Installed CP tool: $cp_tool" | tee -a $LOGFILE 
-		return 1
+	    return 0
 	fi
-     fi
+    fi
 }
 
-# Capture z/VM data
-get_zvm_data()
-{
-	vmcp_dev_exists=0
-	if [ -e "/dev/vmcp" ]; then
-		vmcp_dev_exists=1
-	fi
-	lsmod | grep -q $cp_tool
-	rc_lsmod=$?
-   	if [ $rc_lsmod -eq 1 ] && [ $vmcp_dev_exists -eq 0 ]; then
-                if [ $cp_tool = "hcp" ]; then
-                        modprobe cpint
-                        sleep 2
-                else
-                        modprobe $cp_tool
-                        sleep 2
-                fi
-        fi
-        collect_cmdsout "$VM_CMDS" "$VM_CMDOUTPUT"
-        if [ $rc_lsmod -eq 1 ] && [ $vmcp_dev_exists -eq 0 ]; then
-                if [ $cp_tool = "hcp" ]; then
-                        rmmod cpint
-                else
-                        rmmod $cp_tool
-                fi
-        fi
+
+###############################################################################
+
+
+########################################
+# print version info
+print_version() {
+    cat <<EOF
+ ${SCRIPTNAME}: Debug information script version %S390_TOOLS_VERSION%
+ Copyright IBM Corp. 2002, 2012
+EOF
 }
 
-# Caputure FCP configuration data if possible
-get_fcp_data()
+
+########################################
+# print how to use this script
+print_usage()
 {
-    which ziomon_fcpconf >/dev/null 2>&1;
-    [ $? -eq 0 ] && ziomon_fcpconf -o $FCPCONFOUTPUT;
+    print_version
+
+    cat <<EOF
+
+
+ Usage: ${SCRIPTNAME} [OPTIONS]
+
+ This script collects runtime, configuration and trace information about
+ your Linux on System z installation for debugging purposes.
+
+ It also traces information about z/VM if the Linux runs under z/VM.
+
+
+ The collected information is written to a TAR archive named
+
+     /tmp/DBGINFO-[date]-[time]-[hostname].tgz
+
+ where [date] and [time] are the date and time when debug data is collected.
+ [hostname] indicates the hostname of the system the data was collected from.
+
+
+ Options:
+
+        -h|--help          print this help
+        -v|--version       print version information
+
+
+ Please report bugs to: linux390@de.ibm.com
+
+EOF
 }
 
-# log files
-collect_log()
+
+########################################
+# print that an instance is already running
+print_alreadyrunning() {
+    print_version
+
+    cat <<EOF
+
+
+ Please check the system if another instance of ${SCRIPTNAME} is already
+ running. If this is not the case, please remove the lock file
+ '${WORKDIR_BASE}${SCRIPTNAME}.lock'.
+EOF
+}
+
+
+########################################
+#
+commandline_parse()
 {
-    echo "Copy log files" | tee -a $LOGFILE
-    for i in $LOGFILES
-    do
-	collect_file_contents $i
-    done
+    local cmdline_arg1=${1}
+    local cmdline_count=${#}
+
+    if test ${cmdline_count} -eq 1; then
+	if test ${cmdline_arg1} = '-h' -o ${cmdline_arg1} = '--help'; then
+	    print_usage
+	elif test ${cmdline_arg1} = '-v' -o ${cmdline_arg1} = '--version'; then
+	    print_version
+	else
+	    echo
+	    echo " ${SCRIPTNAME}: invalid option ${cmdline_arg1}"
+	    echo " Try '${SCRIPTNAME} --help' for more information"
+	    echo
+	    exit 1
+	fi
+	exit 0
+    elif test ${cmdline_count} -ge 1; then
+	echo
+	echo " ERROR: Invalid number of arguments!"
+	echo
+	print_usage
+	exit 1
+    fi
+}
+
+
+########################################
+# Setup the environment
+environment_setup()
+{
+    if test -e ${WORKDIR_BASE}${SCRIPTNAME}.lock; then
+	print_alreadyrunning
+        exit 1
+    else
+	touch ${WORKDIR_BASE}${SCRIPTNAME}.lock
+    fi
+
+    # Generating a random delay up to 0.999 second
+    #sleep .$[ ( ${RANDOM} % 1000 ) + 1 ]s
+
+    mkdir -p ${WORKDIR_BASE}
+
+    if ! mkdir ${WORKPATH}; then
+	echo " ERROR: target directory ${WORKPATH} already exists or ${WORKDIR_BASE} does not exist!"
+	exit 1
+    fi
 }
 
+
+########################################
 # create gzip-ped tar file
 create_package()
 {
-    cd $WORKPATH/..
-    tar -czf $WORKDIR.tgz $WORKDIR 2>/dev/null
-    cleanup;
+    echo " Finalizing: Creating archive with collected data"
+    cd ${WORKDIR_BASE}
+
+    if ! tar -czf ${WORKARCHIVE} ${WORKDIR_CURRENT}; then
+	echo " "
+	echo " ERROR: Collection of data failed!"
+    else
+	echo " "
+	echo " Collected data was saved to:"
+	echo "   ${WORKARCHIVE}"
+    fi
+
     echo
-    echo "Collected data was saved to:"
-    echo "  $WORKPFX/$WORKDIR.tgz"
 }
 
-cleanup()
+
+########################################
+# Cleaning up the prepared/collected information
+environment_cleanup()
 {
-    [ -d $WORKPATH ] && rm -rf $WORKPATH;
+    test -d ${WORKPATH} && rm -rf ${WORKPATH};
+    test -e ${WORKDIR_BASE}${SCRIPTNAME}.lock && rm -f ${WORKDIR_BASE}${SCRIPTNAME}.lock;
 }
 
+
+########################################
+# Function to perform a cleanup in case of a received signal
 emergency_exit()
 {
-    cleanup;
+    echo " INFO: Cleanup of temporary collected data"
+    environment_cleanup
+
+    echo " INFO: Emergency exit processed"
+    echo " INFO: Data collection has been interrupted"
+    echo " "
     exit;
 }
 
-#
-# start of script
-#
 
-trap emergency_exit SIGHUP SIGTERM SIGINT
+###############################################################################
+# Running the script
 
-kernel_version=`uname -r`
-kernel_version_tmp=`echo ${kernel_version:4:2} | sed s/[^0-9]//g`
-check_cmdline $*
-prepare_workdir
-printversion >$LOGFILE
-if [ $kernel_version_tmp -lt 13 ] && [ "${kernel_version:0:1}" \< 3 ]
-	then
-		collect_s390dbf
-	fi
-collect_procfs $PROCFILES
-collect_cmdsout "$CMDS" "$CMDOUTPUT"
-if [ "${kernel_version:2:1}" \> 4 ] || [ "${kernel_version:0:1}" \> 2 ]
-    then
-	collect_sysfs
-    else
-	collect_procfs $PROCFILES_24
-fi
-collect_procfs_scsi
-check_zvm
-rc_check_zvm=$?
-if [ $rc_check_zvm -eq 1 ]; then
-	get_zvm_data
-fi
-get_fcp_data
-collect_config
-collect_log
-create_package
+commandline_parse ${*}
+environment_setup
 
-#
-# end of script
-#
+# trap on SIGHUP SIGINT SIGTERM
+trap emergency_exit 1 2 15
+
+(
+    print_version
+    echo " "
+    echo " Kernel version        = ${KERNEL_VERSION}.${KERNEL_MAJOR_REVISION}.${KERNEL_MINOR_REVISION} ($(uname -r))"
+    echo " Runtime environment   = $(test ${LINUX_ON_ZVM} -eq 0 && echo 'z/VM' || echo 'LPAR')"
+    echo " "
+
+    collect_cmdsout
+
+    collect_vmcmdsout
+
+    # Collecting the proc file system (content is specific based on kernel version)
+    collect_procfs
+
+    # Collecting sysfs in case we run on Kernel 2.4 or newer
+    collect_sysfs
+
+    collect_logfiles
+
+    collect_configfiles
+
+    collect_osaoat
+
+) | tee -a ${LOGFILE}
+
+create_package
+echo ""
+environment_cleanup
 
 #EOF
