From c317fa53650557ea1310036d96747e74c0af809c Mon Sep 17 00:00:00 2001
From: Firo Yang <firo.yang@suse.com>
Date: Thu, 12 Sep 2024 13:14:17 +0200
Subject: [PATCH 1/1] makedumpfile: dump per_cpu memory unconditionally

We are seeing many un-openable dumps due to missing of per_cpu memory.
The root cause for the missing dump is that the struct page
was corrutped sometimes. For this reason, makedumpfile can't recognize
and dump neccesary per_cpu pages.

Solution:
Parse __per_cpu_offset from /proc/kallsyms, pass it to makedumpfile
through /proc/cmdline. Then, force the makedumpfile to dump per_cpu
pages.
---
 makedumpfile.c | 271 ++++++++++++++++++++++++++++++++++++++++++++-----
 makedumpfile.h |  22 ++++
 print_info.c   |   1 +
 3 files changed, 268 insertions(+), 26 deletions(-)

diff --git a/makedumpfile.c b/makedumpfile.c
index 78c5b45..c65598d 100644
--- a/makedumpfile.c
+++ b/makedumpfile.c
@@ -52,7 +52,7 @@ static unsigned long long	write_bytes;
 
 static void first_cycle(mdf_pfn_t start, mdf_pfn_t max, struct cycle *cycle)
 {
-	cycle->start_pfn = round(start, info->pfn_cyclic);
+	cycle->start_pfn = MAX(start, round(start, info->pfn_cyclic));
 	cycle->end_pfn = cycle->start_pfn + info->pfn_cyclic;
 
 	if (cycle->end_pfn > max)
@@ -6164,7 +6164,8 @@ __exclude_unnecessary_pages(unsigned long mem_map,
 		/*
 		 * If this pfn doesn't belong to target region, skip this pfn.
 		 */
-		if (info->flag_cyclic && !is_cyclic_region(pfn, cycle))
+		if ((info->flag_cyclic || (info->nr_ranges>1))&& !is_cyclic_region(pfn, cycle))
+		//if (info->flag_cyclic && !is_cyclic_region(pfn, cycle))
 			continue;
 
 		/*
@@ -6383,6 +6384,9 @@ exclude_unnecessary_pages(struct cycle *cycle)
 		if (mmd->mem_map == NOT_MEMMAP_ADDR)
 			continue;
 
+//		#TODO not sure why enable this print will cause the dump to grow huge from 200MB to 1.6GB
+//		ERRMSG(" exclude  %d to 2nd-bitmap %d %d %d  %d %d.\n", is_dumpable(info->bitmap2, 0x7c408, cycle), 
+//				mm, mmd->pfn_start, mmd->pfn_end, cycle->start_pfn, cycle->end_pfn);
 		if (mmd->pfn_end >= cycle->start_pfn &&
 		    mmd->pfn_start <= cycle->end_pfn) {
 			if (!__exclude_unnecessary_pages(mmd->mem_map,
@@ -6794,10 +6798,14 @@ create_2nd_bitmap(struct cycle *cycle)
 		/* Have to do it from scratch. */
 		initialize_2nd_bitmap_cyclic(cycle);
 	} else {
-		/* Can copy 1st-bitmap to 2nd-bitmap. */
-		if (!copy_bitmap()) {
-			ERRMSG("Can't copy 1st-bitmap to 2nd-bitmap.\n");
-			return FALSE;
+		if (info->non_cyclic_bitmp2 == 0)
+		{
+			/* Can copy 1st-bitmap to 2nd-bitmap. */
+			if (!copy_bitmap()) {
+				ERRMSG("Can't copy 1st-bitmap to 2nd-bitmap.\n");
+				return FALSE;
+			}
+			info->non_cyclic_bitmp2 = 1;
 		}
 	}
 
@@ -6807,6 +6815,10 @@ create_2nd_bitmap(struct cycle *cycle)
 	 */
 	exclude_nodata_pages(cycle);
 
+	if (cycle->flags & MRF_MANDATORY) {
+		goto skip_and_sync;
+	}
+
 	/*
 	 * Exclude cache pages, cache private pages, user data pages,
 	 * and hwpoison pages.
@@ -6866,6 +6878,7 @@ create_2nd_bitmap(struct cycle *cycle)
 		}
 	}
 
+skip_and_sync:
 	if (!sync_2nd_bitmap())
 		return FALSE;
 
@@ -7659,6 +7672,7 @@ create_dump_bitmap(void)
 
 	} else {
 		struct cycle cycle = {0};
+		struct cycle sub_cycle;
 		first_cycle(0, info->max_mapnr, &cycle);
 		if (!prepare_bitmap_buffer())
 			goto out;
@@ -7667,8 +7681,25 @@ create_dump_bitmap(void)
 		if (!create_1st_bitmap(&cycle))
 			goto out;
 
-		if (!create_2nd_bitmap(&cycle))
-			goto out;
+		if (info->nr_ranges == 1) {
+			if (!create_2nd_bitmap(&cycle))
+				goto out;
+		} else {
+			for (int i = 0; i < info->nr_ranges; i++) {
+				mdf_pfn_t start = info->mr[i].start_pfn;
+				mdf_pfn_t end = info->mr[i].end_pfn;
+				memset(&sub_cycle, 0, sizeof(struct cycle));
+				sub_cycle.flags = info->mr[i].flags;	
+				ERRMSG("sub cycle %llx %llx %llx.\n", start, end, sub_cycle.flags);
+				for_each_cycle(start, end, &sub_cycle)
+				{
+					ERRMSG("per sub cycle %llx %llx %llx %d\n", sub_cycle.start_pfn, sub_cycle.end_pfn, sub_cycle.flags, 
+							info->flag_cyclic);
+					if (!create_2nd_bitmap(&sub_cycle))
+						goto out;
+				}
+			}
+		}
 
 		if (!(info->num_dumpable = get_num_dumpable_cyclic()))
 			goto out;
@@ -9143,6 +9174,171 @@ write_kdump_bitmap2(struct cycle *cycle) {
 	}
 }
 
+#define COMMAND_LINE_SIZE 2048
+#define proccmdline "/proc/cmdline"
+static ulong get_per_cpu_offset()
+{
+        FILE *fp;
+        ulong per_cpu_offset_addr = 0;
+        char cmdline[COMMAND_LINE_SIZE], *ptr, *end;
+        memset(cmdline, 0, COMMAND_LINE_SIZE);
+
+
+        fp = fopen(proccmdline, "r"); 
+        if (!fp) {
+                ERRMSG("Cannot open %s\n", proccmdline);
+                return 0;
+        }
+        if (fgets(cmdline, sizeof(cmdline), fp) != 0)
+        {
+                ptr = strstr(cmdline, "core_pcpu_off=");
+                if (!ptr)
+                        return 0;
+
+                ptr += strlen("core_pcpu_off=");
+                per_cpu_offset_addr = strtoull(ptr, &end, 16);
+        }
+
+        return per_cpu_offset_addr;
+}
+/* 
+ * x86_64 implemention. #TODO: parse the percpu address from /proc/cmdline then replace 0xffffffffb863cba0
+ */
+int get_per_cpu_size_offset(int *pcpu_size, unsigned long *pcpu_offset)
+{
+	ulong addr = 0;
+	unsigned long pcpu_offsets[2]; // CPU0 and CPU1
+
+	addr = get_per_cpu_offset();
+	//addr = 0xffffffffa2c22b60;
+	if (0 == addr)
+		return FALSE;
+	if (!readmem(VADDR, addr, pcpu_offsets, sizeof(pcpu_offsets))) {
+			ERRMSG("Can't read __per_cpu_offset from vmcore memory.\n");
+			return FALSE;
+	} else {
+
+	//	ERRMSG("read __per_cpu_offset from vmcore memory %lx %lx %lx.\n", addr, pcpu_offsets[1], pcpu_offsets[0]);
+		*pcpu_size = pcpu_offsets[1] - pcpu_offsets[0];
+		*pcpu_offset = pcpu_offsets[0];
+		return TRUE;
+	}
+
+/*
+	if (SYMBOL(__per_cpu_offset) != NOT_FOUND_SYMBOL) {
+		addr = SYMBOL(__per_cpu_offset);
+
+	} else {
+		addr = get_per_cpu_offset();
+		ERRMSG("Symbol __per_cpu_offset isn't found.\n");
+	}
+	if (addr) {
+		if (!readmem(VADDR, addr, pcpu_offsets, sizeof(pcpu_offsets))) {
+			ERRMSG("Can't read __per_cpu_offset from vmcore memory.\n");
+			return FALSE;
+		}
+		*pcpu_size = pcpu_offsets[1] - pcpu_offsets[0];
+		*pcpu_offset = pcpu_offsets[0];
+		return TRUE;
+	}
+*/
+	return FALSE;
+}
+
+#ifdef __x86_64__
+/*
+ * x86_64
+ * x = y + ((x > y) ? phys_base : (__START_KERNEL_map - PAGE_OFFSET));
+ */
+unsigned long __phys_addr(unsigned long va)
+{
+        unsigned long pa;
+
+	if (va > __START_KERNEL_map) {
+		pa = va - __START_KERNEL_map + info->phys_base;
+	} else {
+		pa = va - info->page_offset;
+	}
+
+        return pa;
+}
+
+int get_per_cpu_memory_range(struct mem_range *mr)
+{
+	int rv;
+	int pcpu_size;
+	unsigned long pcpu_offset;
+	int cpus = get_nr_cpus();
+
+	rv = get_per_cpu_size_offset(&pcpu_size, &pcpu_offset);
+	if (rv != TRUE) {
+		ERRMSG("Can't get __per_cpu_offset info.\n");
+		return FALSE;
+	}
+
+	mr->start = __phys_addr(pcpu_offset);
+	mr->end = __phys_addr(pcpu_offset + cpus * pcpu_size);
+
+	return TRUE;
+}
+#else
+unsigned long __phys_addr(unsigned long va)
+{
+	return va;
+/*
+        unsigned long pa;
+
+        if (va > __START_KERNEL_map) {
+                pa = va - __START_KERNEL_map + info->phys_base;
+        } else {
+                pa = va - info->page_offset;
+        }
+
+        return pa;
+*/
+}
+
+int get_per_cpu_memory_range(struct mem_range *mr)
+{
+	return FALSE;
+}
+#endif
+
+
+void init_memory_range(struct mem_range *mr, unsigned long s, 
+			unsigned long e, unsigned flags)
+{
+	mr->start = s;
+	mr->end = e;
+	mr->start_pfn = paddr_to_pfn(s);
+	mr->end_pfn = paddr_to_pfn(roundup(e, PAGESIZE()));
+	mr->flags = flags;
+}
+/*
+ * [0]: 0 - per_cpu_start;
+ * [1]: per_cpu_start - per_cpu_end;
+ * [2]: per_cpu_end - maxpfn;
+ */
+int initialize_memory_ranges(struct mem_range *mr)
+{
+	int rv;
+	struct mem_range pcpu_mr;
+
+	rv = get_per_cpu_memory_range(&pcpu_mr);
+
+	if (rv != TRUE) {
+		init_memory_range(&mr[0], 0, info->max_mapnr * PAGESIZE(), 0);
+		return FALSE;
+	}
+
+	init_memory_range(&mr[0], 0, pcpu_mr.start, 0);
+	init_memory_range(&mr[1], pcpu_mr.start, pcpu_mr.end, MRF_MANDATORY);
+	init_memory_range(&mr[2], pcpu_mr.end, info->max_mapnr * PAGESIZE(), 0);
+	ERRMSG("read __per_cpu_offset from vmcore memory %llx %llx %llx.\n", pcpu_mr.start, pcpu_mr.end, info->max_mapnr);
+
+	return TRUE;
+}
+
 int
 write_kdump_pages_and_bitmap_cyclic(struct cache_data *cd_header, struct cache_data *cd_page)
 {
@@ -9204,30 +9400,48 @@ write_kdump_pages_and_bitmap_cyclic(struct cache_data *cd_header, struct cache_d
 			return FALSE;
 	}
 
+	/*
+	 * Initialize physcial memory ranges.
+	int nr_ranges= 3;
+	struct mem_range mr[nr_ranges];
+	if (!initialize_memory_ranges(mr)) {
+		nr_ranges = 1;
+	}
+	 */
+
 	/*
 	 * Write pages and bitmap cyclically.
 	 */
-	//cycle = {0, 0};
-	memset(&cycle, 0, sizeof(struct cycle));
-	for_each_cycle(0, info->max_mapnr, &cycle)
+	for (int i = 0; i < info->nr_ranges; i++) 
 	{
-		if (info->flag_cyclic) {
-			if (!create_2nd_bitmap(&cycle))
-				return FALSE;
-		}
-
-		if (!write_kdump_bitmap2(&cycle))
-			return FALSE;
+		mdf_pfn_t start = info->mr[i].start_pfn;
+		mdf_pfn_t end = info->mr[i].end_pfn;
+		//cycle = {0, 0};
+		memset(&cycle, 0, sizeof(struct cycle));
+		cycle.flags = info->mr[i].flags;
+		ERRMSG("cycle %llx %llx %llx.\n", start, end, cycle.flags);
+		for_each_cycle(start, end, &cycle)
+		{
+			ERRMSG("per cycle %llx %llx %llx %d .\n", cycle.start_pfn, cycle.end_pfn, cycle.flags
+					,info->flag_cyclic);
+			if (info->flag_cyclic) {
+				if (!create_2nd_bitmap(&cycle))
+					return FALSE;
+			}
 
-		if (info->num_threads) {
-			if (!write_kdump_pages_parallel_cyclic(cd_header,
-							cd_page, &pd_zero,
-							&offset_data, &cycle))
-				return FALSE;
-		} else {
-			if (!write_kdump_pages_cyclic(cd_header, cd_page, &pd_zero,
-					&offset_data, &cycle))
+			if (!write_kdump_bitmap2(&cycle))
 				return FALSE;
+
+			if (info->num_threads) {
+				if (!write_kdump_pages_parallel_cyclic(cd_header,
+								cd_page, &pd_zero,
+								&offset_data, &cycle))
+					return FALSE;
+			} else {
+				if (!write_kdump_pages_cyclic(cd_header, cd_page, &pd_zero,
+						&offset_data, &cycle))
+					return FALSE;
+			}
 		}
 	}
 	free_bitmap2_buffer();
@@ -10469,6 +10683,11 @@ create_dumpfile(void)
 	print_vtop();
 
 	num_retry = 0;
+	info->nr_ranges = 3;
+	info->non_cyclic_bitmp2 = 0;
+	if (!initialize_memory_ranges(info->mr)) {
+		info->nr_ranges = 1;
+	}
 retry:
 	if (info->flag_refiltering)
 		update_dump_level();
diff --git a/makedumpfile.h b/makedumpfile.h
index 5125c5e..f1b0959 100644
--- a/makedumpfile.h
+++ b/makedumpfile.h
@@ -1340,6 +1340,19 @@ struct ppc64_vmemmap {
 	unsigned long		virt;
 };
 
+#define MRF_MANDATORY		(0x1 << 0)
+#define MRF_EXCLUDED		(0x1 << 1)
+
+#define MRT_PERCPU		0x1
+struct mem_range {
+	unsigned long long start;
+	unsigned long long end;
+	unsigned long start_pfn;
+	unsigned long end_pfn;
+	unsigned long long flags;
+	int type;
+};
+
 struct DumpInfo {
 	int32_t		kernel_version;      /* version of first kernel*/
 	struct timeval	timestamp;
@@ -1493,6 +1506,13 @@ struct DumpInfo {
 	int			fd_bitmap;
 	char			*name_bitmap;
 
+	/*
+	 * Dump range info 
+	 */
+	int 			nr_ranges;
+	struct mem_range 	mr[3];
+	int			non_cyclic_bitmp2;
+
 	/*
 	 * vmcoreinfo file info:
 	 */
@@ -2137,6 +2157,7 @@ struct memory_range {
 	unsigned long long start, end;
 };
 
+
 #define CRASH_RESERVED_MEM_NR   8
 extern struct memory_range crash_reserved_mem[CRASH_RESERVED_MEM_NR];
 extern int crash_reserved_mem_nr;
@@ -2312,6 +2333,7 @@ struct cycle {
 	mdf_pfn_t exclude_pfn_start;
 	mdf_pfn_t exclude_pfn_end;
 	mdf_pfn_t *exclude_pfn_counter;
+	unsigned long long flags;
 };
 
 static inline int
diff --git a/print_info.c b/print_info.c
index 8d3b1cb..e6a17e3 100644
--- a/print_info.c
+++ b/print_info.c
@@ -69,6 +69,7 @@ print_usage(void)
 #else
 	MSG("  disabled ('-z' option will be ignored.)\n");
 #endif
+	MSG("SUSE debug version.\n");
 	MSG("\n");
 	MSG("Usage:\n");
 	MSG("  Creating DUMPFILE:\n");
-- 
2.35.3

