Subject: [PATCH] [FEAT KRN1311] hyptop: Exploit diag 0c data
From: Michael Holzheu <holzheu@linux.vnet.ibm.com>

Summary:     hyptop: Exploit diag 0c data
Description: The hyptop tool now reads the binary debugfs file
             "/sys/kernel/debug/diag_0c" and provides the following
             two new metrics under z/VM:
               * mgm : Management time per second
               * mgm+: Total management time
             In addition to this also includes several minor cleanup patches
             from upstream s390-tools are included.
Upstream-ID: -
Problem-ID:  KRN1311

Signed-off-by: Michael Holzheu <holzheu@linux.vnet.ibm.com>
---
 hyptop/Makefile           |    2 
 hyptop/dg_debugfs.c       |   10 ++-
 hyptop/dg_debugfs.h       |    9 ++
 hyptop/dg_debugfs_lpar.c  |    7 --
 hyptop/dg_debugfs_vm.c    |  116 ++++++++++++++++++++++++++++++++----
 hyptop/dg_debugfs_vmd0c.c |  147 ++++++++++++++++++++++++++++++++++++++++++++++
 hyptop/helper.c           |    8 +-
 hyptop/hyptop.8           |   17 +++++
 hyptop/sd.h               |   19 +++++
 hyptop/sd_core.c          |    5 -
 hyptop/sd_cpu_items.c     |   21 +++++-
 hyptop/sd_sys_items.c     |   26 ++++++--
 hyptop/table.c            |    6 -
 hyptop/table.h            |    7 +-
 hyptop/table_col_unit.c   |   11 +++
 hyptop/win_sys.c          |   17 +++--
 hyptop/win_sys_list.c     |    5 +
 17 files changed, 385 insertions(+), 48 deletions(-)

--- a/hyptop/Makefile
+++ b/hyptop/Makefile
@@ -9,7 +9,7 @@ LDLIBS += -lncurses
 OBJECTS = hyptop.o opts.o helper.o \
 	  sd_core.o sd_sys_items.o sd_cpu_items.o \
 	  tbox.o table.o table_col_unit.o \
-	  dg_debugfs.o dg_debugfs_lpar.o dg_debugfs_vm.o \
+	  dg_debugfs.o dg_debugfs_lpar.o dg_debugfs_vm.o dg_debugfs_vmd0c.o \
 	  win_sys_list.o win_sys.o win_fields.o \
 	  win_cpu_types.o win_help.o nav_desc.o
 
--- a/hyptop/dg_debugfs.c
+++ b/hyptop/dg_debugfs.c
@@ -16,6 +16,8 @@
 #include "hyptop.h"
 #include "dg_debugfs.h"
 
+#define HYPFS_SUBDIR "/s390_hypfs/"
+
 static char *l_debugfs_dir;
 
 static void l_check_rc(int rc, int exit_on_err)
@@ -70,17 +72,19 @@ int dg_debugfs_init(int exit_on_err)
  */
 int dg_debugfs_open(const char *file)
 {
-	char path[PATH_MAX];
+	char *path;
 	int fh;
 
+	path = ht_alloc(strlen(l_debugfs_dir) + strlen(HYPFS_SUBDIR) +
+			strlen(file) + 1);
 	path[0] = 0;
 	strcat(path, l_debugfs_dir);
-	strcat(path, "/s390_hypfs/");
+	strcat(path, HYPFS_SUBDIR);
 	strcat(path, file);
 	fh = open(path, O_RDONLY);
+	ht_free(path);
 	if (fh == -1)
 		return -errno;
 	else
 		return fh;
 }
-
--- a/hyptop/dg_debugfs.h
+++ b/hyptop/dg_debugfs.h
@@ -10,6 +10,8 @@
 #ifndef DG_DEBUGFS_H
 #define DG_DEBUGFS_H
 
+#include "sd.h"
+
 #define DBFS_WAIT_TIME_US 10000
 
 extern int dg_debugfs_init(int exit_on_err);
@@ -17,4 +19,11 @@ extern int dg_debugfs_vm_init(void);
 extern int dg_debugfs_lpar_init(void);
 extern int dg_debugfs_open(const char *file);
 
+/*
+ * z/VM diag 0C prototypes
+ */
+int dg_debugfs_vmd0c_init(void);
+void dg_debugfs_vmd0c_sys_cpu_fill(struct sd_sys *sys, u64 online_time,
+				   unsigned int cpu_cnt);
+
 #endif /* DG_DEBUGFS_H */
--- a/hyptop/dg_debugfs_lpar.c
+++ b/hyptop/dg_debugfs_lpar.c
@@ -105,6 +105,7 @@ static void l_sd_cpu_fill(struct sd_cpu
 	sd_cpu_cpu_time_us_set(cpu, cpu_info->lp_time);
 	sd_cpu_mgm_time_us_set(cpu, G0(cpu_info->acc_time - cpu_info->lp_time));
 	sd_cpu_online_time_us_set(cpu, cpu_info->online_time);
+	sd_cpu_state_set(cpu, SD_CPU_STATE_UNKNOWN);
 }
 
 /*
@@ -153,6 +154,7 @@ static void l_sd_cpu_phys_fill(struct sd
 	if (!cpu) {
 		l_idx2name(cpu_info->ctidx, cpu_type);
 		cpu = sd_cpu_new(sys, cpu_id, cpu_type, 1);
+		sd_cpu_state_set(cpu, SD_CPU_STATE_UNKNOWN);
 	}
 	sd_cpu_mgm_time_us_set(cpu, cpu_info->mgm_time);
 	sd_cpu_real_type_set(cpu, cpu_type);
@@ -224,7 +226,6 @@ static void l_sd_sys_root_fill(struct sd
 	struct l_x_info_blk_hdr *time_hdr;
 	struct l_debugfs_d204_hdr *hdr;
 	struct l_x_sys_hdr *sys_hdr;
-	static int first = 1;
 	struct sd_sys *lpar;
 	char lpar_id[10];
 	int i;
@@ -252,10 +253,8 @@ static void l_sd_sys_root_fill(struct sd
 		sd_sys_commit(lpar);
 	}
 
-	if (first && (time_hdr->flags & LPAR_PHYS_FLG)) {
+	if (time_hdr->flags & LPAR_PHYS_FLG)
 		l_sd_sys_root_cpu_phys_fill(sys, (void *) sys_hdr);
-		first = 0;
-	}
 	ht_free(hdr);
 	sd_sys_commit(sys);
 }
--- a/hyptop/dg_debugfs_vm.c
+++ b/hyptop/dg_debugfs_vm.c
@@ -19,9 +19,13 @@
 #define VM_CPU_ID	"ALL"
 #define NAME_LEN	8
 #define DEBUGFS_FILE	"diag_2fc"
+#define VM_CPU_ID_OPERATING	"0"
+#define VM_CPU_ID_STOPPED	"1"
 
 static u64 l_update_time_us;
 static long l_2fc_buf_size;
+static int l_use_debugfs_vmd0c;
+static char l_guest_name[64];
 
 /*
  * Diag 2fc data structure definition
@@ -38,7 +42,7 @@ struct l_diag2fc_data {
 	u32	pcpus;
 	u32	lcpus;
 	u32	vcpus;
-	u32	cpu_min;
+	u32	ocpus;
 	u32	cpu_max;
 	u32	cpu_shares;
 	u32	cpu_use_samp;
@@ -66,24 +70,103 @@ struct l_debugfs_d2fc {
 	char			diag2fc_buf[];
 } __attribute__ ((packed));
 
+ /*
+ * Get local guest name
+ */
+static void l_guest_name_init(void)
+{
+	int level, found = 0;
+	char line[1024];
+	FILE *fh;
+
+	fh = fopen("/proc/sysinfo", "r");
+	if (!fh)
+		ERR_EXIT_ERRNO("Could not open '/proc/sysinfo'");
+	while (fgets(line, sizeof(line), fh)) {
+		if (sscanf(line, "VM%02d Name: %s", &level, l_guest_name) == 2)
+			found = 1;
+	}
+	if (!found)
+		ERR_EXIT("Could find guest name in '/proc/sysinfo'");
+	fclose(fh);
+}
+
 /*
- * Fill "guest" with data
+ * Get existing or create new CPU
  */
-static void l_sd_sys_fill(struct sd_sys *guest, struct l_diag2fc_data *data)
+static struct sd_cpu *l_cpu_alloc(struct sd_sys *guest, const char *id, int cnt)
 {
-	struct sd_cpu *cpu;
+	struct sd_cpu *cpu = sd_cpu_get(guest, id);
 
-	cpu = sd_cpu_get(guest, VM_CPU_ID);
-	if (!cpu)
-		cpu = sd_cpu_new(guest, VM_CPU_ID, SD_CPU_TYPE_STR_UN,
-				 data->vcpus);
+	return cpu ? cpu : sd_cpu_new(guest, id, SD_CPU_TYPE_STR_UN, cnt);
+}
+
+/*
+ * Get number of operating CPUs
+ */
+static int l_ocpus(struct l_diag2fc_data *data)
+{
+	/*
+	 * For guests with ABSOLUTE or limit SHARE, ocpus and cpu_max is zero.
+	 * In this case we return vcpus.
+	 */
+	return (data->cpu_max == 0) ? data->vcpus : data->ocpus;
+}
+
+/*
+ * Fill operating CPUs with data
+ */
+static void l_cpu_oper_fill(struct sd_sys *guest, struct l_diag2fc_data *data)
+{
+	struct sd_cpu *cpu;
 
-	sd_cpu_cnt(cpu) = data->vcpus;
+	cpu = l_cpu_alloc(guest, VM_CPU_ID_OPERATING, l_ocpus(data));
+	sd_cpu_state_set(cpu, SD_CPU_STATE_OPERATING);
 	sd_cpu_cpu_time_us_set(cpu, data->used_cpu);
 	sd_cpu_online_time_us_set(cpu, data->el_time);
+	sd_cpu_cnt(cpu) = l_ocpus(data);
+	sd_cpu_commit(cpu);
+}
+
+/*
+ * Fill stopped CPUs with data
+ */
+static void l_cpu_stop_fill(struct sd_sys *guest, struct l_diag2fc_data *data)
+{
+	int cnt = data->vcpus - l_ocpus(data);
+	struct sd_cpu *cpu;
+
+	cpu = l_cpu_alloc(guest, VM_CPU_ID_STOPPED, cnt);
+	sd_cpu_state_set(cpu, SD_CPU_STATE_STOPPED);
+	sd_cpu_cpu_time_us_set(cpu, 0);
+	sd_cpu_online_time_us_set(cpu, 0);
+	sd_cpu_cnt(cpu) = cnt;
+	sd_cpu_commit(cpu);
+}
+
+/*
+ * Fill CPUs will data
+ */
+static void l_cpu_fill(struct sd_sys *guest, struct l_diag2fc_data *data)
+{
+	if (l_ocpus(data) > 0)
+		l_cpu_oper_fill(guest, data);
+	if (data->vcpus - l_ocpus(data) > 0)
+		l_cpu_stop_fill(guest, data);
+}
+
+/*
+ * Fill "guest" with data
+ */
+static void l_sd_sys_fill(struct sd_sys *guest, struct l_diag2fc_data *data)
+{
+	if (l_use_debugfs_vmd0c && (strcmp(guest->id, l_guest_name) == 0))
+		dg_debugfs_vmd0c_sys_cpu_fill(guest, data->el_time,
+					      data->vcpus);
+	else
+		l_cpu_fill(guest, data);
 
 	sd_sys_weight_cur_set(guest, data->cpu_shares);
-	sd_sys_weight_min_set(guest, data->cpu_min);
 	sd_sys_weight_max_set(guest, data->cpu_max);
 
 	sd_sys_mem_min_kib_set(guest, data->mem_min_kb);
@@ -149,6 +232,9 @@ static void l_sd_sys_root_fill(struct sd
 	if (!cpu)
 		cpu = sd_cpu_new(sys, VM_CPU_ID, SD_CPU_TYPE_STR_UN,
 				 d2fc_data[0].lcpus);
+	sd_cpu_state_set(cpu, SD_CPU_STATE_UNKNOWN);
+	sd_cpu_cnt(cpu) = d2fc_data[0].lcpus;
+	sd_cpu_commit(cpu);
 
 	for (i = 0; i < hdr->count; i++) {
 		struct l_diag2fc_data *data = &d2fc_data[i];
@@ -186,12 +272,14 @@ static void l_sd_update(void)
  */
 static struct sd_sys_item *l_sys_item_vec[] = {
 	&sd_sys_item_cpu_cnt,
+	&sd_sys_item_cpu_oper_cnt,
 	&sd_sys_item_cpu_diff,
+	&sd_sys_item_mgm_diff,
 	&sd_sys_item_cpu,
+	&sd_sys_item_mgm,
 	&sd_sys_item_online,
 	&sd_sys_item_mem_use,
 	&sd_sys_item_mem_max,
-	&sd_sys_item_weight_min,
 	&sd_sys_item_weight_cur,
 	&sd_sys_item_weight_max,
 	NULL,
@@ -216,7 +304,9 @@ static struct sd_sys_item *l_sys_item_en
  */
 static struct sd_cpu_item *l_cpu_item_vec[] = {
 	&sd_cpu_item_cpu_diff,
+	&sd_cpu_item_mgm_diff,
 	&sd_cpu_item_cpu,
+	&sd_cpu_item_mgm,
 	&sd_cpu_item_online,
 	NULL,
 };
@@ -256,12 +346,16 @@ int dg_debugfs_vm_init(void)
 {
 	int fh;
 
+	fh = dg_debugfs_vmd0c_init();
+	if (fh == 0)
+		l_use_debugfs_vmd0c = 1;
 	fh = dg_debugfs_open(DEBUGFS_FILE);
 	if (fh < 0)
 		return fh;
 	else
 		close(fh);
 	l_2fc_buf_size = sizeof(struct l_debugfs_d2fc_hdr);
+	l_guest_name_init();
 	sd_dg_register(&dg_debugfs_vm_dg);
 	return 0;
 }
--- /dev/null
+++ b/hyptop/dg_debugfs_vmd0c.c
@@ -0,0 +1,147 @@
+/*
+ * hyptop - Show hypervisor performance data on System z
+ *
+ * Hyptop z/VM data gatherer for diag 0c that operates on debugfs
+ *
+ * Copyright IBM Corp. 2010
+ * Author(s): Michael Holzheu <holzheu@linux.vnet.ibm.com>
+ */
+
+#include <stdlib.h>
+#include <unistd.h>
+#include <errno.h>
+#include <linux/types.h>
+#include "hyptop.h"
+#include "sd.h"
+#include "helper.h"
+#include "dg_debugfs.h"
+
+#define DEBUGFS_FILE	"diag_0c"
+
+static long l_0c_buf_size;
+
+/*
+ * Diag 0c entry structure definition
+ */
+struct hypfs_diag0c_entry {
+	char	date[8];	/* MM/DD/YY in EBCDIC */
+	char	time[8];	/* HH:MM:SS in EBCDIC */
+	__u64	virtcpu;	/* Virtual time consumed by the virt CPU (us) */
+	__u64	totalproc;	/* Total of virtual and simulation time (us) */
+	__u32	cpu;		/* Linux logical CPU number */
+	__u32	reserved;	/* Align to 8 byte */
+};
+
+/*
+ * Header for debugfs file "diag_0c"
+ */
+struct hypfs_diag0c_hdr {
+	__u64	len;		/* Length of diag0c buffer without header */
+	__u16	version;	/* Version of header */
+	char	reserved1[6];	/* Reserved */
+	char	tod_ext[16];	/* TOD clock for diag0c */
+	__u64	count;		/* Number of entries (CPUs) in diag0c array */
+	char	reserved2[24];	/* Reserved */
+};
+
+struct hypfs_diag0c_data {
+	struct hypfs_diag0c_hdr		hdr;		/* 64 byte header */
+	struct hypfs_diag0c_entry	entry[];	/* diag0c entry array */
+};
+
+/*
+ * Fill one CPU with data
+ */
+static void l_sd_cpu_fill(struct sd_sys *sys, unsigned int cpu_nr,
+			  u64 online_time, u64 cpu_time,
+			  u64 mgm_time, enum sd_cpu_state state)
+{
+	struct sd_cpu *cpu;
+	char cpu_id[16];
+
+	sprintf(cpu_id, "%d", cpu_nr);
+	cpu = sd_cpu_get(sys, cpu_id);
+	if (!cpu)
+		cpu = sd_cpu_new(sys, cpu_id, SD_CPU_TYPE_STR_UN, 1);
+	sd_cpu_cpu_time_us_set(cpu, cpu_time);
+	sd_cpu_mgm_time_us_set(cpu, mgm_time);
+	sd_cpu_online_time_us_set(cpu, online_time);
+	sd_cpu_state_set(cpu, state);
+	sd_cpu_cnt(cpu) = 1;
+	sd_cpu_commit(cpu);
+}
+
+/*
+ * Read debugfs file
+ */
+static void l_read_debugfs(struct hypfs_diag0c_hdr **hdr,
+			   struct hypfs_diag0c_entry **entry)
+{
+	long real_buf_size;
+	ssize_t rc;
+	void *buf;
+	int fh;
+
+	do {
+		fh = dg_debugfs_open(DEBUGFS_FILE);
+		if (fh < 0)
+			ERR_EXIT_ERRNO("Could not open file: %s", DEBUGFS_FILE);
+		*hdr = buf = ht_alloc(l_0c_buf_size);
+		rc = read(fh, buf, l_0c_buf_size);
+		if (rc == -1)
+			ERR_EXIT_ERRNO("Reading hypervisor data failed");
+		close(fh);
+		real_buf_size = (*hdr)->len + sizeof(struct hypfs_diag0c_hdr);
+		if (rc == real_buf_size)
+			break;
+		l_0c_buf_size = real_buf_size;
+		ht_free(buf);
+	} while (1);
+	*entry = buf + sizeof(struct hypfs_diag0c_hdr);
+}
+
+/*
+ * Fill System Data
+ */
+void dg_debugfs_vmd0c_sys_cpu_fill(struct sd_sys *sys, u64 online_time,
+				   unsigned int cpu_cnt)
+{
+	unsigned int i, cpu_online_vec[cpu_cnt];
+	struct hypfs_diag0c_entry *d0c_entry;
+	struct hypfs_diag0c_hdr *hdr;
+	u64 mgm_time;
+
+	memset(cpu_online_vec, 0, sizeof(cpu_online_vec));
+	l_read_debugfs(&hdr, &d0c_entry);
+
+	/* First fill online CPUs */
+	for (i = 0; i < hdr->count; i++) {
+		mgm_time = G0(d0c_entry[i].totalproc - d0c_entry[i].virtcpu);
+		l_sd_cpu_fill(sys, d0c_entry[i].cpu, online_time,
+			      d0c_entry[i].virtcpu, mgm_time,
+			      SD_CPU_STATE_OPERATING);
+		cpu_online_vec[d0c_entry[i].cpu] = 1;
+	}
+	/* Then fill offline CPUs */
+	for (i = 0; i < cpu_cnt; i++) {
+		if (cpu_online_vec[i])
+			continue;
+		l_sd_cpu_fill(sys, i, 0, 0, 0, SD_CPU_STATE_STOPPED);
+	}
+}
+
+/*
+ * Initialize z/VM debugfs data gatherer
+ */
+int dg_debugfs_vmd0c_init(void)
+{
+	int fh;
+
+	fh = dg_debugfs_open(DEBUGFS_FILE);
+	if (fh < 0)
+		return -1;
+	else
+		close(fh);
+	l_0c_buf_size = sizeof(struct hypfs_diag0c_hdr);
+	return 0;
+}
--- a/hyptop/helper.c
+++ b/hyptop/helper.c
@@ -113,8 +113,10 @@ char *ht_mount_point_get(const char *fs_
 	if (!mounts)
 		ERR_EXIT_ERRNO("Could not find \"%s\" mount point", fs_type);
 	while ((mntbuf = getmntent(mounts)) != NULL) {
-		if (strcmp(mntbuf->mnt_type, fs_type) == 0)
+		if (strcmp(mntbuf->mnt_type, fs_type) == 0) {
+			endmntent(mounts);
 			return ht_strdup(mntbuf->mnt_dir);
+		}
 	}
 	endmntent(mounts);
 	return NULL;
@@ -184,9 +186,9 @@ void ht_print_head(const char *sys)
 		ht_bold_off();
 		hyptop_printf(" ");
 	}
-	hyptop_printf("CPU-");
+	hyptop_printf("cpu-");
 	ht_underline_on();
-	hyptop_printf("T");
+	hyptop_printf("t");
 	ht_underline_off();
 	hyptop_printf(": ");
 
--- a/hyptop/hyptop.8
+++ b/hyptop/hyptop.8
@@ -125,20 +125,25 @@ The following fields are available under
 
   In "sys_list" and "sys" window:
   'c' - CPU time per second
+  'm' - Management time per second (*)
   'C' - Total CPU time
+  'M' - Total management time (*)
   'o' - Online time
 
   In "sys_list" window:
   '#' - Number of CPUs
+  'O' - Number of operating CPUs
   'u' - Used memory
   'a' - Maximum memory
-  'n' - Minimum weight
   'r' - Current weight
   'x' - Maximum weight
 
   In "sys" window:
   'v' - Visualization of CPU time per second
 
+  (*) Only available for the local guest virtual machine
+      Only available if the system has the required support
+
 .SH UNITS
 Depending on the field type the values can be displayed in different units.
 The following units are supported:
@@ -176,6 +181,16 @@ On LPAR the following CPU types are supp
 NOTE: It is possible that on older machines also IFLs are shown as CPs.
 On z/VM currently only the processor type 'UN' is available.
 
+.SH CPU DATA
+For Linux on z/VM, no performance data is available for individual CPUs
+on remote guests.
+Therefore, the hyptop "sys" window shows identical values for each of
+the operating CPUs. For the CPU time fields, these values represent the
+total across all operating CPUs divided by the number of operating CPUs.
+For the online time field, the value is the time during which at least
+one CPU has been operational.
+Operating CPUs are shown with CPU identifier "0" and stopped CPUs with "1".
+
 .SH EXAMPLES
 To start hyptop with the "sys_list" window in interactive mode, enter:
 .br
--- a/hyptop/sd.h
+++ b/hyptop/sd.h
@@ -21,6 +21,7 @@
  */
 struct sd_cpu_info {
 	u64	cpu_time_us;
+	u64	mgm_time_us_set;
 	u64	mgm_time_us;
 	u64	wait_time_us;
 	s64	steal_time_us;
@@ -233,6 +234,7 @@ static inline void sd_cpu_cpu_time_us_se
 
 static inline void sd_cpu_mgm_time_us_set(struct sd_cpu *cpu, u64 value)
 {
+	cpu->d_cur->mgm_time_us_set = 1;
 	cpu->d_cur->mgm_time_us = value;
 }
 
@@ -279,6 +281,7 @@ struct sd_cpu_item {
 	enum sd_item_type	type;
 	int			offset;
 	char			*desc;
+	int (*fn_set)(struct sd_cpu_item *, struct sd_cpu *);
 	u64 (*fn_u64)(struct sd_cpu_item *, struct sd_cpu *);
 	s64 (*fn_s64)(struct sd_cpu_item *, struct sd_cpu *);
 	char *(*fn_str)(struct sd_cpu_item *, struct sd_cpu *);
@@ -293,6 +296,11 @@ extern int sd_cpu_item_cnt(void);
 /*
  * Item access functions
  */
+static inline u64 sd_cpu_item_set(struct sd_cpu_item *item, struct sd_cpu *cpu)
+{
+	return (item->fn_set == NULL) ? 1 : item->fn_set(item, cpu);
+}
+
 static inline u64 sd_cpu_item_u64(struct sd_cpu_item *item,
 				  struct sd_cpu *cpu)
 {
@@ -338,6 +346,7 @@ struct sd_sys_item {
 	int			offset;
 	char			*desc;
 	int			info;
+	int (*fn_set)(struct sd_sys_item *, struct sd_sys *);
 	u64 (*fn_u64)(struct sd_sys_item *, struct sd_sys *);
 	s64 (*fn_s64)(struct sd_sys_item *, struct sd_sys *);
 };
@@ -351,6 +360,11 @@ extern int sd_sys_item_cnt(void);
 /*
  * Item access functions
  */
+static inline int sd_sys_item_set(struct sd_sys *sys, struct sd_sys_item *item)
+{
+	return (item->fn_set == NULL) ? 1 : item->fn_set(item, sys);
+}
+
 static inline u64 sd_sys_item_u64(struct sd_sys *sys,
 				  struct sd_sys_item *item)
 {
@@ -447,6 +461,11 @@ void sd_dg_register(struct sd_dg *);
 #define SD_CPU_INFO_OFFSET(x) \
 	((unsigned long)(void *)&(((struct sd_cpu_info *) NULL)->x))
 
+static inline int l_cpu_info_set(struct sd_cpu_info *info, unsigned long offset)
+{
+	return (int)*(u64 *)(((char *) info) + offset - 8);
+}
+
 static inline u64 l_cpu_info_u64(struct sd_cpu_info *info,
 				 unsigned long offset)
 {
--- a/hyptop/sd_core.c
+++ b/hyptop/sd_core.c
@@ -314,7 +314,6 @@ static void l_cpu_update_end(struct sd_s
 	/* Has system not lost any CPU? */
 	if (sys->cpu_cnt_active == sys->cpu_cnt)
 		return;
-
 	util_list_iterate_safe(&sys->cpu_list, cpu, tmp) {
 		if (!cpu->i.active) {
 			/* CPU has not been updated, remove it */
@@ -323,6 +322,7 @@ static void l_cpu_update_end(struct sd_s
 			continue;
 		}
 	}
+	sys->cpu_cnt = sys->cpu_cnt_active;
 }
 
 /*
@@ -332,9 +332,6 @@ static void l_sys_update_end(struct sd_s
 {
 	struct sd_sys *child, *tmp;
 
-	if (sys->child_cnt_active == sys->child_cnt)
-		return;
-
 	l_cpu_update_end(sys);
 
 	util_list_iterate_safe(&sys->child_list, child, tmp) {
--- a/hyptop/sd_cpu_items.c
+++ b/hyptop/sd_cpu_items.c
@@ -46,7 +46,7 @@ static double l_cpu_diff(struct sd_cpu_i
 	if (online_time_diff_us == 0)
 		return 0;
 
-	factor = ((double) online_time_diff_us) / 1000000;
+	factor = ((double) online_time_diff_us) / 1000000 * cpu->cnt;
 	if (sign)
 		diff_us = l_cpu_info_s64(cpu->d_cur, item->offset) -
 			  l_cpu_info_s64(cpu->d_prev, item->offset);
@@ -74,17 +74,26 @@ static s64 l_cpu_diff_s64(struct sd_cpu_
 }
 
 /*
+ * Return true, if CPU item has been set
+ */
+static inline int l_cpu_item_set(struct sd_cpu_item *item, struct sd_cpu *cpu)
+{
+	return l_cpu_info_set(cpu->d_cur, item->offset);
+}
+
+/*
  * Return cpu item value
  */
 static u64 l_cpu_item_64(struct sd_cpu_item *item, struct sd_cpu *cpu)
 {
-	if (sd_cpu_state(cpu) == SD_CPU_STATE_STOPPED)
-		return 0;
 	if (!cpu->d_cur)
 		return 0;
 	if (!sd_cpu_type_selected(cpu->type))
 		return 0;
-	return l_cpu_info_u64(cpu->d_cur, item->offset);
+	if (item->table_col.agg == TABLE_COL_AGG_MAX)
+		return l_cpu_info_u64(cpu->d_cur, item->offset);
+	else
+		return l_cpu_info_u64(cpu->d_cur, item->offset) / cpu->cnt;
 }
 
 /*
@@ -117,6 +126,7 @@ struct sd_cpu_item sd_cpu_item_mgm_diff
 	.type	= SD_TYPE_U64,
 	.offset = SD_CPU_INFO_OFFSET(mgm_time_us),
 	.desc	= "Management time per second",
+	.fn_set	= l_cpu_item_set,
 	.fn_u64	= l_cpu_diff_u64,
 };
 
@@ -150,6 +160,7 @@ struct sd_cpu_item sd_cpu_item_mgm = {
 	.type	= SD_TYPE_U64,
 	.offset = SD_CPU_INFO_OFFSET(mgm_time_us),
 	.desc	= "Total management time",
+	.fn_set	= l_cpu_item_set,
 	.fn_u64	= l_cpu_item_64,
 };
 
@@ -162,7 +173,7 @@ struct sd_cpu_item sd_cpu_item_wait = {
 };
 
 struct sd_cpu_item sd_cpu_item_steal = {
-	.table_col = TABLE_COL_STIME_SUM(table_col_unit_hm, 'S', "steal+"),
+	.table_col = TABLE_COL_STIME_SUM(table_col_unit_hm, 'T', "steal+"),
 	.type	= SD_TYPE_U64,
 	.offset = SD_CPU_INFO_OFFSET(steal_time_us),
 	.desc	= "Total steal time",
--- a/hyptop/sd_sys_items.c
+++ b/hyptop/sd_sys_items.c
@@ -68,6 +68,22 @@ static u64 l_sys_cpu_dc_cnt(struct sd_sy
 }
 
 /*
+ * Check if CPU info is set
+ */
+static int l_sys_cpu_info_set(struct sd_sys_item *item, struct sd_sys *sys)
+{
+	struct sd_cpu *cpu;
+
+	sd_cpu_iterate(sys, cpu) {
+		if (!sd_cpu_type_selected(cpu->type))
+			continue;
+		if (!l_cpu_info_set(cpu->d_cur, item->offset))
+			return 0;
+	}
+	return 1;
+}
+
+/*
  * Get u64 system item value from "sys"
  */
 static u64 l_sys_item_u64(struct sd_sys_item *item, struct sd_sys *sys)
@@ -191,21 +207,21 @@ struct sd_sys_item sd_sys_item_cpu_cnt =
 };
 
 struct sd_sys_item sd_sys_item_cpu_oper_cnt = {
-	.table_col = TABLE_COL_CNT_SUM('e', "#cpuope"),
+	.table_col = TABLE_COL_CNT_SUM('O', "#cpuop"),
 	.type	= SD_TYPE_U32,
 	.desc	= "Number of operating CPUs",
 	.fn_u64	= l_sys_cpu_op_cnt,
 };
 
 struct sd_sys_item sd_sys_item_cpu_stop_cnt = {
-	.table_col = TABLE_COL_CNT_SUM('p', "#cpusp"),
+	.table_col = TABLE_COL_CNT_SUM('S', "#cpust"),
 	.type	= SD_TYPE_U32,
 	.desc	= "Number of stopped CPUs",
 	.fn_u64	= l_sys_cpu_st_cnt,
 };
 
 struct sd_sys_item sd_sys_item_cpu_deconf_cnt = {
-	.table_col = TABLE_COL_CNT_SUM('d', "#cpudc"),
+	.table_col = TABLE_COL_CNT_SUM('D', "#cpudc"),
 	.type	= SD_TYPE_U32,
 	.desc	= "Number of deconfigured CPUs",
 	.fn_u64	= l_sys_cpu_dc_cnt,
@@ -224,6 +240,7 @@ struct sd_sys_item sd_sys_item_mgm_diff
 	.offset = SD_CPU_INFO_OFFSET(mgm_time_us),
 	.type	= SD_TYPE_U64,
 	.desc	= "Management time per second",
+	.fn_set	= l_sys_cpu_info_set,
 	.fn_u64	= l_sys_cpu_info_diff_u64,
 };
 
@@ -265,11 +282,12 @@ struct sd_sys_item sd_sys_item_mgm = {
 	.offset = SD_CPU_INFO_OFFSET(mgm_time_us),
 	.type	= SD_TYPE_U64,
 	.desc	= "Total management time",
+	.fn_set	= l_sys_cpu_info_set,
 	.fn_u64	= l_sys_cpu_info_sum_u64,
 };
 
 struct sd_sys_item sd_sys_item_steal = {
-	.table_col = TABLE_COL_STIME_SUM(table_col_unit_hm, 'S', "steal+"),
+	.table_col = TABLE_COL_STIME_SUM(table_col_unit_hm, 'T', "steal+"),
 	.offset = SD_CPU_INFO_OFFSET(steal_time_us),
 	.type	= SD_TYPE_U64,
 	.desc	= "Total steal time",
--- a/hyptop/table.c
+++ b/hyptop/table.c
@@ -46,7 +46,6 @@ static void l_mark_key_add(struct table
 	key = ht_zalloc(sizeof(*key));
 	strncpy(key->str, str, sizeof(key->str));
 	util_list_add_tail(&t->mark_key_list, key);
-	t->mark_keys_cnt++;
 }
 
 /*
@@ -60,7 +59,6 @@ static void l_mark_key_remove(struct tab
 		if (strcmp(str, key->str) == 0) {
 			util_list_remove(&t->mark_key_list, key);
 			ht_free(key);
-			t->mark_keys_cnt--;
 			return;
 		}
 	}
@@ -80,7 +78,6 @@ void table_row_mark_del_all(struct table
 		util_list_remove(&t->mark_key_list, key);
 		ht_free(key);
 	}
-	t->mark_keys_cnt = 0;
 }
 
 /*
@@ -358,6 +355,8 @@ static void l_row_last_agg(struct table
 		struct table_entry *e_last = &t->row_last->entries[col_nr];
 		struct table_entry *e_new = &table_row->entries[col_nr];
 
+		if (!e_new->set)
+			continue;
 		switch (col->agg) {
 		case TABLE_COL_AGG_SUM:
 			l_entry_sum(col->type, e_last, e_new);
@@ -368,6 +367,7 @@ static void l_row_last_agg(struct table
 		case TABLE_COL_AGG_NONE:
 			break;
 		}
+		e_last->set = 1;
 	}
 }
 
--- a/hyptop/table.h
+++ b/hyptop/table.h
@@ -285,6 +285,7 @@ struct table_entry {
 			s64	v2;
 		} s64;
 	} d;
+	int			set;
 	char			str[TABLE_STR_MAX];
 };
 
@@ -293,7 +294,6 @@ struct table_entry {
  */
 struct table_row {
 	struct util_list_node	list;
-	int			entry_count;
 	struct table_entry	*entries;
 	int			marked;
 };
@@ -322,7 +322,6 @@ struct table {
 	int 			row_nr_select;
 	int			ready;
 	struct util_list	mark_key_list;
-	unsigned int		mark_keys_cnt;
 	int			attr_sorted_table;
 	int			attr_first_bold;
 	int			attr_with_units;
@@ -390,6 +389,7 @@ static inline void table_row_entry_u64_a
 					   u64 value)
 {
 	table_row->entries[table_col->p->col_nr].d.u64.v1 = value;
+	table_row->entries[table_col->p->col_nr].set = 1;
 }
 
 static inline void table_row_entry_s64_add(struct table_row *table_row,
@@ -397,6 +397,7 @@ static inline void table_row_entry_s64_a
 					   s64 value)
 {
 	table_row->entries[table_col->p->col_nr].d.s64.v1 = value;
+	table_row->entries[table_col->p->col_nr].set = 1;
 }
 
 static inline void table_row_entry_u64_add_pair(struct table_row *table_row,
@@ -405,6 +406,7 @@ static inline void table_row_entry_u64_a
 {
 	table_row->entries[table_col->p->col_nr].d.u64.v1 = value1;
 	table_row->entries[table_col->p->col_nr].d.u64.v2 = value2;
+	table_row->entries[table_col->p->col_nr].set = 1;
 }
 
 static inline void table_row_entry_str_add(struct table_row *table_row,
@@ -413,6 +415,7 @@ static inline void table_row_entry_str_a
 {
 	assert(strlen(str) < TABLE_STR_MAX);
 	strcpy(table_row->entries[table_col->p->col_nr].str, str);
+	table_row->entries[table_col->p->col_nr].set = 1;
 }
 
 /*
--- a/hyptop/table_col_unit.c
+++ b/hyptop/table_col_unit.c
@@ -13,6 +13,7 @@
 #define L_VISUAL_ROW_CNT	45
 #define L_COL_FMT_STR_0		"%.0lf"
 #define L_COL_FMT_STR_2		"%.2lf"
+#define L_COL_NOT_SET_STR	"-"
 
 /*
  * Helper: Divide value and format it
@@ -22,6 +23,8 @@ static int l_unit_raw_div(struct table_c
 {
 	double v1;
 
+	if (!e->set)
+		return snprintf(e->str, sizeof(e->str), L_COL_NOT_SET_STR);
 	switch (col->type) {
 	case TABLE_COL_TYPE_U64:
 		v1 = ((double) e->d.u64.v1) / divisor;
@@ -40,6 +43,8 @@ static int l_unit_raw_div(struct table_c
  */
 static int l_unit_raw(struct table_col *col, struct table_entry *e)
 {
+	if (!e->set)
+		return snprintf(e->str, sizeof(e->str), L_COL_NOT_SET_STR);
 	switch (col->type) {
 	case TABLE_COL_TYPE_U64:
 		return snprintf(e->str, sizeof(e->str), "%llu", e->d.u64.v1);
@@ -219,6 +224,9 @@ static int l_unit_hm_u64(char *str, u64
 
 static int l_unit_hm(struct table_col *col, struct table_entry *e)
 {
+	if (!e->set)
+		return snprintf(e->str, sizeof(e->str), L_COL_NOT_SET_STR);
+
 	switch (col->type) {
 	case TABLE_COL_TYPE_U64:
 		return l_unit_hm_u64(e->str, e->d.u64.v1, 0);
@@ -262,6 +270,9 @@ static int l_unit_dhm_u64(char *str, u64
 
 static int l_unit_dhm(struct table_col *col, struct table_entry *e)
 {
+	if (!e->set)
+		return snprintf(e->str, sizeof(e->str), L_COL_NOT_SET_STR);
+
 	switch (col->type) {
 	case TABLE_COL_TYPE_U64:
 		return l_unit_dhm_u64(e->str, e->d.u64.v1, 0);
--- a/hyptop/win_sys.c
+++ b/hyptop/win_sys.c
@@ -134,9 +134,8 @@ static void l_cpu_add_visual(struct tabl
 	s64 steal_us;
 	u64 cpu_us;
 
-	cpu_us = sd_cpu_item_u64(&sd_cpu_item_cpu_diff, cpu) / sd_cpu_cnt(cpu);
-	steal_us = sd_cpu_item_s64(&sd_cpu_item_steal_diff, cpu) /
-		sd_cpu_cnt(cpu);
+	cpu_us = sd_cpu_item_u64(&sd_cpu_item_cpu_diff, cpu);
+	steal_us = sd_cpu_item_s64(&sd_cpu_item_steal_diff, cpu);
 	steal_us = MAX(steal_us, 0);
 	table_row_entry_u64_add_pair(table_row, &l_vis_col, cpu_us, steal_us);
 }
@@ -155,8 +154,11 @@ static void l_cpu_add(struct sd_cpu *cpu
 	cpu_id = atoi(sd_cpu_id(cpu));
 	table_row_entry_u64_add(table_row, &l_cpu_col, cpu_id);
 
-	sd_cpu_item_iterate(item, i)
+	sd_cpu_item_iterate(item, i) {
+		if (!sd_cpu_item_set(item, cpu))
+			continue;
 		l_cpu_item_add(table_row, cpu, item);
+	}
 	l_cpu_add_visual(table_row, cpu);
 	table_row_add(l_t, table_row);
 }
@@ -168,13 +170,16 @@ static int l_table_create(void)
 {
 	struct sd_sys *parent;
 	struct sd_cpu *cpu;
+	int i;
 
 	parent = sd_sys_get(sd_sys_root_get(), l_sys_id);
 	if (!parent)
 		return -ENODEV;
 	table_row_del_all(l_t);
-	sd_cpu_iterate(parent, cpu)
-		l_cpu_add(cpu);
+	sd_cpu_iterate(parent, cpu) {
+		for (i = 0; i < cpu->cnt; i++)
+			l_cpu_add(cpu);
+	}
 	table_finish(l_t);
 	return 0;
 }
--- a/hyptop/win_sys_list.c
+++ b/hyptop/win_sys_list.c
@@ -172,8 +172,11 @@ static void l_sys_add(struct sd_sys *sys
 	table_row = table_row_alloc(l_t);
 	table_row_entry_str_add(table_row, &l_col_sys, sd_sys_id(sys));
 
-	sd_sys_item_iterate(item, i)
+	sd_sys_item_iterate(item, i) {
+		if (!sd_sys_item_set(sys, item))
+			continue;
 		l_sys_item_add(table_row, sys, item);
+	}
 	table_row_add(l_t, table_row);
 }
 
