Subject: [PATCH] [FEAT 79969] s390: Fuzzy live dump for System z
From: Michael Holzheu <holzheu@linux.vnet.ibm.com>

Summary:     s390: Fuzzy live dump for System z
Description: A new dump mechanism is established to create a completely
             non-disruptive kernel dump. Because the Linux system
             continues running while the dump is written and kernel data
             structures are changing during the procedure, the resulting
             dump will not be consistent (fuzzy).

             This patch adds the following features to the zgetdump tool:
               - Live dump creation using new devmem source dump format
               - Live dump detection using --info option
Problem-ID:  79969
---
 zdump/Makefile     |    2 
 zdump/dfi.c        |   22 +++++
 zdump/dfi.h        |   10 ++
 zdump/dfi_devmem.c |  155 +++++++++++++++++++++++++++++++++++++
 zdump/zgetdump.8   |  222 ++++++++++++++++++++++++++++++++++++-----------------
 zdump/zgetdump.h   |    1 
 6 files changed, 339 insertions(+), 73 deletions(-)

--- a/zdump/Makefile
+++ b/zdump/Makefile
@@ -8,7 +8,7 @@ all: zgetdump
 OBJECTS = zgetdump.o opts.o zg.o \
 	  dfi.o dfi_vmcoreinfo.o \
 	  dfi_lkcd.o dfi_elf.o dfi_s390.o dfi_s390mv.o dfi_s390tape.o \
-	  dfi_kdump.o \
+	  dfi_kdump.o dfi_devmem.o \
 	  dfo.o dfo_elf.o dfo_s390.o \
 	  df_s390.o \
 	  dt.o dt_s390sv.o dt_s390mv.o \
--- a/zdump/dfi.c
+++ b/zdump/dfi.c
@@ -14,10 +14,11 @@
 #define PROGRESS_HASH_CNT 50
 
 /*
- * DFI vector - ensure that tape is the first in the list!
+ * DFI vector - ensure that tape is the first in the list and devmem the second!
  */
 static struct dfi *dfi_vec[] = {
 	&dfi_s390tape,
+	&dfi_devmem,
 	&dfi_s390mv,
 	&dfi_s390,
 	&dfi_lkcd,
@@ -28,6 +29,11 @@ static struct dfi *dfi_vec[] = {
 };
 
 /*
+ * Live dump magic
+ */
+u64 dfi_live_dump_magic = 0x4c49564544554d50ULL; /* LIVEDUMP */
+
+/*
  * CPU information
  */
 struct cpus {
@@ -810,6 +816,19 @@ static void utsname_init(void)
 }
 
 /*
+ * Try to get livedump magic
+ */
+static void livedump_init(void)
+{
+	u64 magic;
+
+	if (dfi_mem_read_rc(0, &magic, sizeof(magic)))
+		return;
+	if (magic == dfi_live_dump_magic)
+		dfi_attr_dump_method_set(DFI_DUMP_METHOD_LIVE);
+}
+
+/*
  * Initialize input dump format.
  */
 int dfi_init(void)
@@ -830,6 +849,7 @@ int dfi_init(void)
 			dfi_vmcoreinfo_init();
 			kdump_swap_init();
 			utsname_init();
+			livedump_init();
 		}
 		if (rc == 0 || rc == -EINVAL)
 			return rc;
--- a/zdump/dfi.h
+++ b/zdump/dfi.h
@@ -237,4 +237,14 @@ extern const char *dfi_name(void);
 extern int dfi_init(void);
 extern void dfi_exit(void);
 
+/*
+ * Live dump memory magic
+ */
+extern u64 dfi_live_dump_magic;
+
+/*
+ * Dump methods
+ */
+#define DFI_DUMP_METHOD_LIVE	"live"
+
 #endif /* DFI_H */
--- /dev/null
+++ b/zdump/dfi_devmem.c
@@ -0,0 +1,155 @@
+/*
+ * zgetdump - Tool for copying and converting System z dumps
+ *
+ * /dev/mem and /dev/crash dump input format
+ *
+ * Copyright IBM Corp. 2012
+ * Author(s): Michael Holzheu <holzheu@linux.vnet.ibm.com>
+ */
+
+#include <string.h>
+#include <time.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <limits.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/utsname.h>
+#include <linux/fs.h>
+#include "zgetdump.h"
+
+/*
+ * Add live dump magic to buffer
+ */
+static void add_live_magic(void *buf, u64 off, u64 cnt)
+{
+	if (off >= sizeof(dfi_live_dump_magic))
+		return;
+	memcpy(buf, &dfi_live_dump_magic,
+	       MIN(cnt, sizeof(dfi_live_dump_magic)));
+}
+
+/*
+ * "devmem" mem chunk read callback
+ *
+ * This functions reads page wise in order to detect unreadable pages
+ */
+static void dfi_devmem_mem_chunk_read(struct dfi_mem_chunk *mem_chunk, u64 off,
+				      void *buf, u64 cnt)
+{
+	u64 copied = 0, len = PAGE_SIZE;
+	(void) mem_chunk;
+
+	if (off % PAGE_SIZE)
+		len = MIN(len, PAGE_SIZE - off % PAGE_SIZE);
+	len = MIN(len, cnt);
+	do {
+		zg_seek(g.fh, mem_chunk->start + off + copied, ZG_CHECK);
+		if (zg_read(g.fh, buf + copied, len, ZG_CHECK_NONE) < 0) {
+			if (errno == EFAULT) {
+				/* This can happen when using CMM */
+				memset(buf + copied, 0, len);
+			} else {
+				ERR_EXIT_ERRNO("Could not read %s",
+					       g.opts.device);
+			}
+		}
+		copied += len;
+		len = MIN(PAGE_SIZE, cnt - copied);
+	} while (len);
+	add_live_magic(buf, mem_chunk->start + off, cnt);
+}
+
+/*
+ * Detect memory chunks via /proc/iomem
+ *
+ * check = 0: Initialize memory map
+ * check = 1: Verifiy that current system memory map is same as DFI memory map
+ */
+static int detect_mem_chunks(int check)
+{
+	char line[4096], type1[4096], type2[4096];
+	unsigned long start, end, cnt = 0;
+	struct dfi_mem_chunk *mem_chunk;
+	struct zg_fh *fh;
+	ssize_t rc;
+
+	fh = zg_open("/proc/iomem", O_RDONLY, ZG_CHECK);
+	do {
+		rc = zg_gets(fh, line, sizeof(line), ZG_CHECK);
+		if (rc == 0)
+			break;
+		sscanf(line, "%lx-%lx : %s %s", &start, &end, type1, type2);
+		if (strcmp(type1, "System") != 0)
+			continue;
+		if (strcmp(type2, "RAM") != 0 && strcmp(type2, "ROM") != 0)
+			continue;
+		if (check) {
+			mem_chunk = dfi_mem_chunk_find(start);
+			if (!mem_chunk)
+				return -EINVAL;
+			if (mem_chunk->start != start)
+				return -EINVAL;
+			if (mem_chunk->end != end)
+				return -EINVAL;
+			cnt++;
+		} else {
+			dfi_mem_chunk_add(start, end - start + 1, NULL,
+					  dfi_devmem_mem_chunk_read);
+		}
+	} while (1);
+	if (check && cnt != dfi_mem_chunk_cnt())
+		return -EINVAL;
+	return 0;
+}
+
+/*
+ * Return architecture of running system
+ */
+static enum dfi_arch system_arch(void)
+{
+	struct utsname utsname;
+
+	uname(&utsname);
+	if (memcmp(utsname.machine, "s390x", 5) == 0)
+		return DFI_ARCH_64;
+	if (memcmp(utsname.machine, "s390", 4) == 0)
+		return DFI_ARCH_32;
+	return DFI_ARCH_UNKNOWN;
+}
+
+/*
+ * Initialize devmem DFI
+ */
+static int dfi_devmem_init(void)
+{
+	if (strcmp(g.fh->path, "/dev/mem") != 0 &&
+	    strcmp(g.fh->path, "/dev/crash") != 0)
+		return -ENODEV;
+	dfi_arch_set(system_arch());
+	dfi_cpu_info_init(DFI_CPU_CONTENT_NONE);
+	detect_mem_chunks(0);
+	dfi_attr_dump_method_set(DFI_DUMP_METHOD_LIVE);
+	zg_seek(g.fh, 0, ZG_CHECK);
+	return 0;
+}
+
+/*
+ * Cleanup devmem DFI
+ */
+static void dfi_devmem_exit(void)
+{
+	if (detect_mem_chunks(1))
+		STDERR("Warning: memory map has changed\n");
+}
+
+/*
+ * devmem DFI operations
+ */
+struct dfi dfi_devmem = {
+	.name		= "devmem",
+	.init		= dfi_devmem_init,
+	.exit		= dfi_devmem_exit,
+	.feat_bits	= DFI_FEAT_COPY | DFI_FEAT_SEEK,
+};
--- a/zdump/zgetdump.8
+++ b/zdump/zgetdump.8
@@ -1,14 +1,27 @@
-.TH ZGETDUMP 8 "Jan 2010" "s390-tools"
+.TH ZGETDUMP 8 "April 2012" "s390-tools"
 .SH NAME
 zgetdump \- Tool for copying and converting System z dumps
 .SH SYNOPSIS
-\fBzgetdump\fR [OPTIONS] [DUMP/DUMPDEV] [DIR]
+
+\fBzgetdump\fR    DUMP [-s SYS] [-f FMT] > DUMP_FILE
+.br
+         -m DUMP [-s SYS] [-f FMT] DIR
+.br
+         -i DUMP [-s SYS]
+.br
+         -d DUMPDEV
+.br
+         -u DIR
+.br
+         -h|-v
 .SH DESCRIPTION
-The \fBzgetdump\fR tool reads or converts a dump. The dump can be located
-either on a dump device or on a file system. By default the dump content is
+The \fBzgetdump\fR tool copies a source dump into a target dump with a
+configurable dump format. The source dump can be located either on a dump
+device or on a file system. By default the source dump content is
 written to standard output, which you can redirect to a specific file. You
-can also mount the dump content, print dump information, or check
-whether a DASD device contains a valid dump tool.
+can also mount the dump content, print dump information, check
+whether a DASD device contains a valid dump tool, or create a non-disruptive
+dump on a live system.
 .SH OPTIONS
 .TP
 .BR "\-h" " or " "\-\-help"
@@ -20,7 +33,7 @@ Print version information, then exit.
 
 .TP
 .BR "\-m <DUMP> <DIR>" " or " "\-\-mount <DUMP> <DIR>"
-Mount the DUMP to mount point DIR and generate a virtual target
+Mount the source dump DUMP to mount point DIR and generate a virtual target
 dump file instead of writing the content to standard output. The virtual dump
 file gets the name "dump.FMT", where FMT is the name of the specified
 dump format (see "--fmt" option).
@@ -28,12 +41,12 @@ dump format (see "--fmt" option).
 .TP
 .BR "\-u <DIR>" " or " "\-\-umount <DIR>"
 Unmount the dump that is mounted at mount point DIR. This option is a wrapper
-for "fusermount -u". Instead of DIR also the the DUMP (e.g. /dev/dasdd1)
+for "fusermount -u". Instead of DIR also the DUMP (for example /dev/dasdd1)
 can be specified.
 
 .TP
 .BR "\-d <DUMPDEV>" " or " "\-\-device <DUMPDEV>"
-Check DASD device DUMPDEV for valid dump tool and print information about it.
+Check DASD device DUMPDEV for a valid dump tool and print information about it.
 
 .TP
 .BR "\-i <DUMP>" " or " "\-\-info <DUMP>"
@@ -51,12 +64,27 @@ Executable and Linking Format core dump
 s390 dump (default)
 
 .TP
+.BR "\-s <SYS>" " or " "\-\-select <SYS>"
+If kdump fails and a stand-alone dump is created, the resulting dump captures
+two systems: The crashed kdump system and the previously crashed Linux
+instance. With the "--select" option you can choose which system data
+to use:
+
+.BR "- prod:"
+Data for the initial crashed Linux instance
+
+.BR "- kdump:"
+Data for the crashed kdump system
+
+The "-s" option returns an error for dumps that capture only a single crashed system.
+
+.TP
 \fBDUMP\fR
 This parameter specifies the file, partition or tape device node where the
-dump is located:
+source dump is located:
 .TP
 .BR
-- Regular dump file (e.g. /testdir/dump.0)
+- Regular dump file (e.g. /dumps/dump.0)
 .TP
 .BR
 - DASD partition device node (e.g. /dev/dasdc1)
@@ -66,6 +94,9 @@ dump is located:
 .TP
 .BR
 - Tape device node (e.g. /dev/ntibm0)
+.TP
+.BR
+- Device node for live system (/dev/mem or /dev/crash)
 
 Note: For DASD multi-volume dump it is sufficient to specify only one of the
 multi-volume DASD partitions as DUMP.
@@ -76,37 +107,49 @@ When using the "--device" option, DUMPDE
 the dump disk that should be verified.
 
 .SH COPY DUMP
-The default action of zgetdump is to copy the DUMP to standard output. Read
+The default action of zgetdump is to copy the source dump to standard output in
+the target format specified by the \-\-fmt option. Read
 the examples section below for more information.
 
 .SH MOUNT DUMP
-Instead of writing the dump content to standard output you can also mount the
-dump using the "--mount" option. With that option it is possible to convert
-the dump without the need of copying it. The zgetdump tool generates a
-virtual target dump file that contains the dump in the requested target
-format. The virtual dump file is generated by mounting the source dump as a
-user space file system to the directory specified by the "--mount" option.
-The virtual target dump file is called dump.<FMT> where FMT denotes
-the format of the target dump. The virtual dump file exists as long as the
-directory containing the file is not unmounted.
-
-Mounting can be useful when you want to process the dump with a tool that
-cannot read the original dump format. To do this, mount the dump and
-specify the required target dump format with the "--fmt" option. Mounting is
-also for useful for multi-volume DASD dumps. After a multi-volume dump has been
-mounted, it is shown as a single dump file that can be accessed directly with
-dump processing tools like "makedumpfile", "crash" or "lcrash".
-
-Mounting is implemented with "fuse" (file system in user space). Therefore the
-"fuse" kernel module must to be loaded on the system before the "--mount"
-option can be used.
-
-A DASD dump can be mounted e.g. with "zgetdump /dev/dasdd1 -m
-/mnt" and unmounted with either "zgetdump -u /mnt", "fusermount -u /mnt" or
-"umount /mnt" (root only).
+Use the "--mount" option to make a source dump accessible to tools that cannot
+directly read the original dump format. Rather than creating a converted
+copy of the dump, zgetdump creates a virtual dump file with the requested
+target format. This is fast and does not consume any additional disk space.
+Also multi-volume dumps can be assimilated into a single virtual dump file,
+which can then be accessed directly with dump-processing tools like
+makedumpfile or crash.
+
+Specify a command of this form to mount and convert a dump:
+
+.br
+# zgetdump --mount <DUMP> <DIR>  --fmt <FMT>
+.br
+
+Where:
+.TP
+.BR DUMP
+is the source dump or dump device
+.TP
+.BR DIR
+is the mount point where the virtual dump file is created
+.TP
+.BR FMT
+is the target dump format to which the virtual dump file is converted.
+The resulting virtual dump file is <DIR>/dump.<FMT>
+.P
+The virtual dump file exists until the directory is unmounted.
+Use zgetdump -u <DIR> to unmount a dump.
 
+The zgetdump tool uses the file system in user space (fuse) to mount the source
+dump. Therefore, the fuse kernel module must to be loaded before using
+the "--mount" option.
+
+Read the examples section below for more information.
 .SH DUMP FORMATS
-zgetdump supports the following dump formats:
+The default target format of zgetdump is "s390". Use the "--fmt" option to
+change the target format. The following dump formats are supported for
+target and source dump:
 .TP
 .BR "s390"
 This dump format is System z specific and is used for DASD and tape dumps.
@@ -116,17 +159,25 @@ Executable and Linking Format core dump.
 Linux user space core dumps. The zgetdump tool supports this dump format only
 for 64 bit.
 .TP
-.BR "lkcd"
-This dump format has been used by the Linux Kernel Crash Dumps (LKCD) project
-and is used on System z for the vmconvert and zfcp (SCSI) dump tool. The
-zgetdump tool supports "lkcd" only as source format.
 
+The following dump formats are supported for the source dump only:
+
+.TP
+.BR "lkcd"
+This dump format is used by the Linux Kernel Crash Dumps (LKCD) project
+and also on System z for the "vmconvert" and "zfcp" (SCSI) dump tool.
 .TP
-The default target format of zgetdump is "s390". Use the "--fmt" option to change the target format.
+.BR "devmem"
+On live systems the /dev/mem or /dev/crash device nodes can be used as source
+dumps for creating live dumps.
+.TP
+.BR "kdump" / "kdump_flat"
+Dump formats created by the "makedumpfile" tool. For these formats only the
+"--info" option can be used.
 
 .SH DUMP INFORMATION
-When calling zgetdump with the "--info" option depending on the dump format
-the following dump attributes are available:
+Depending on the dump format, the following dump attributes are available
+when calling zgetdump with the "--info" option:
 .TP
 .BR "Dump format"
 Name of the dump format.
@@ -134,11 +185,16 @@ Name of the dump format.
 .BR Version
 Version number of the dump format.
 .TP
+.BR "Dump method"
+Dump method that has been used to create the dump. Currently the only
+supported value for this attribute is "live" which indicates that the
+dump has been created from a live system and therefore is not consistent.
+.TP
 .BR "Dump created/ended"
 Time when the dump process was started or ended. The dump time information is
 printed in your local time zone. E.g. "Wed, 03 Feb 2010 10:47:37 +0100" shows
 the time at your location. The meaning of "+0100" is that your time zone is one
-hour behind GMT (Greenwich Mean Time). You can use the "TZ" environment
+hour behind GMT. You can use the "TZ" environment
 variable or use the "tzselect" tool to change the time zone. For example, if you
 know that the dump has been created in Hawaii, you can get the correct
 time information with:
@@ -147,13 +203,22 @@ time information with:
 # TZ='Pacific/Honolulu' zgetdump -i DUMP
 .TP
 .BR "Dump CPU ID"
-Identifier of the CPU that executed the dump tool.
+Identifier of the CPU that ran the dump tool.
+.TP
+.BR "UTS node name"
+The network node hostname of the Linux system.
+.TP
+.BR "UTS kernel release"
+The kernel release of the Linux system.
+.TP
+.BR "UTS kernel version"
+The kernel version of the Linux system.
 .TP
 .BR "Build arch"
 Architecture (s390 or s390x) on which the dump tool was built.
 .TP
 .BR "System arch"
-Architecture (s390 or s390x) of the dumped Linux system.
+Architecture (s390 or s390x) of the Linux system.
 .TP
 .BR "CPU count (online)"
 Number of online CPUs.
@@ -166,19 +231,20 @@ Memory range that was dumped. This value
 dumped and the first dumped memory address.
 .TP
 .BR "Real memory range"
-Memory range that was available on system. This value is the difference
-between the last and the first memory address of the dumped system.
+Memory range that was available on the system. This value is the difference
+between the last and the first memory address of the system on which the
+dump was created.
 The "real memory range" can differ from the "dump memory range" when
 the SIZE parameter was used when preparing the dump device with the zipl
 tool (see man zipl).
 .TP
 .BR "Memory map"
-Available memory chunks in the dump. Depending on the dump tool there
-can be multiple memory chunks, when a system with memory holes is dumped.
+Available memory chunks in the dump. Some dump tools create multiple memory
+chunks when creating a dump on a system with memory gaps
 
 .SH DUMP DEVICE INFORMATION
-When calling zgetdump with the "--device" option depending on the dump tool
-the following attributes are available:
+Depending on the dump tool, the following attributes are available
+when calling zgetdump with the "--device" option:
 .TP
 .BR "Dump tool"
 Name of the dump tool.
@@ -197,7 +263,7 @@ If this attribute is set, the dump tool
 limit even if there is more memory available.
 .TP
 .BR "Force specified"
-If that attribute is set to "yes", the multi-volume DASD dump tool will not
+If this attribute is set to "yes", the multi-volume DASD dump tool will not
 verify the dump signature on dump partitions. This can be useful, if the dump
 partition is also used for swap.
 
@@ -211,8 +277,9 @@ The DASD partition /dev/dasdx1 was prepa
   # zipl -d /dev/dasdx1
 
 .br
-The corresponding single-volume dump tool was IPLed. The respective zgetdump
-call to copy the dump from the DASD partition to file dump.s390 is:
+An IPL was performed on the corresponding single-volume dump tool and a dump
+has been created. To copy the dump from the DASD partition to file dump.s390
+issue:
 .br
 
   # zgetdump /dev/dasdx1 > dump.s390
@@ -227,8 +294,9 @@ were prepared for multi-volume dump with
   # zipl -M dev_list.conf
 
 .br
-The corresponding multi-volume dump tool was IPLed. The respective zgetdump
-call to copy the dump from the DASD partitions to file dump.s390 is:
+An IPL was performed on the corresponding multi-volume dump tool and a dump
+has been created. To copy the dump from the DASD partitions to file dump.s390
+issue:
 .br
 
   # zgetdump /dev/dasdx > dump.s390
@@ -243,14 +311,25 @@ Tape device /dev/ntibm0 was prepared wit
   # zipl -d /dev/ntibm0
 
 .br
-The corresponding tape dump tool was IPLed. The respective zgetdump call to
-copy the dump from the tape to file dump.s390 is:
+An IPL was performed on the corresponding tape dump tool and a dump
+has been created. To copy the dump from the tape to file dump.s390
+issue:
 .br
 
   # zgetdump /dev/ntibm0 > dump.s390
 
 .br
 .TP
+.B Create live dump
+
+To store an ELF-format dump from a live system in a file called dump.elf
+issue:
+.br
+
+  # nice -n -20 zgetdump /dev/mem -f elf > dump.elf
+
+.br
+.TP
 .B Using pipes for network transfer
 
 You can redirect standard output to tools like ftp or ssh in order to
@@ -277,28 +356,29 @@ scp or ftp directly on the mounted virtu
 .TP
 .B Using the "--mount" option
 
-Mount multi-volume DASD dump, process it with the "crash" tool and unmout
-it with zgetdump afterwards.
+Mount a single-volume DASD dump as virtual ELF dump file, compress
+it with the makedumpfile tool, and unmount it with zgetdump:
 .br
 
-  # zgetdump -m -f elf /dev/dasdx /dumps
-  # crash vmlinux /dumps/dump.elf
+  # zgetdump -m -f elf /dev/dasdc1 /dumps
+  # makedumpfile -c -d 31 -x vmlinux.debug \\
+       /dumps/dump.elf dump.kdump
   # zgetdump -u /dumps
 
 .br
-Convert an ELF dump to an s390 dump by mounting it with the "--fmt" option,
-process it with lcrash and unmount it with fusermount afterwards.
+Mount a multi-volume DASD dump, process it with the "crash" tool, and
+unmount it with fusermount:
 .br
 
-  # zgetdump -m -f s390 dump.elf /dumps
-  # lcrash System.map /dumps/dump.s390 Kerntypes
+  # zgetdump -m /dev/dasdx /dumps
+  # crash vmlinux /dumps/dump.s390
   # fusermount -u /dumps
 
 .br
 .TP
 .B Print dump information (--info)
 
-Print information on DASD dump on /dev/dasdd1:
+Print information about a DASD dump on /dev/dasdd1:
 .br
 
   # zgetdump -i /dev/dasdd1
@@ -307,11 +387,11 @@ Print information on DASD dump on /dev/d
 .TP
 .B Print DASD dump tool information (--device)
 
-Print information on DASD dump tool on /dev/dasdd:
+Print information about a DASD dump tool on /dev/dasdd:
 .br
 
   # zgetdump -d /dev/dasdd
 
 .br
 .SH SEE ALSO
-.BR zipl (8), crash (8),  lcrash (8), dumpconf (8), vmconvert (8), vmur (8)
+.BR zipl (8), crash (8), makedumpfile (8), dumpconf (8), vmconvert (8), vmur (8)
--- a/zdump/zgetdump.h
+++ b/zdump/zgetdump.h
@@ -78,6 +78,7 @@ extern struct dfi dfi_lkcd;
 extern struct dfi dfi_elf;
 extern struct dfi dfi_kdump;
 extern struct dfi dfi_kdump_flat;
+extern struct dfi dfi_devmem;
 
 /*
  * Supported DFO dump formats
