From 0d88f46661c0907e00916d578da3602e790cbbe0 Mon Sep 17 00:00:00 2001
From: Bruce Rogers <brogers@suse.com>
Date: Wed, 24 Jan 2018 15:40:49 -0700
Subject: [PATCH] S390-qemu-kvm

---
 hw/msix.c               |   2 +
 hw/msix.h               |  34 +++++++++++++++
 hw/s390-virtio.c        |   4 +-
 kvm-tpr-opt.c           |   3 +-
 kvm/include/linux/kvm.h |   4 ++
 qemu-kvm-helper.c       |   2 +
 qemu-kvm.c              |  46 ++++++++++++++++----
 qemu-kvm.h              |   2 +
 target-s390x/kvm.c      | 110 +++++++++++++++++++++++++++++++++++++++++++++++-
 target-s390x/libkvm.h   |  26 ++++++++++++
 10 files changed, 221 insertions(+), 12 deletions(-)
 create mode 100644 target-s390x/libkvm.h

diff --git a/hw/msix.c b/hw/msix.c
index d117bcfaa0..7a2095ac75 100644
--- a/hw/msix.c
+++ b/hw/msix.c
@@ -61,6 +61,7 @@
 #define DEBUG(fmt, ...) do { } while(0)
 #endif
 
+#if ! defined(TARGET_S390X)
 /* Flag for interrupt controller to declare MSI-X support */
 int msix_supported;
 
@@ -587,3 +588,4 @@ void msix_unuse_all_vectors(PCIDevice *dev)
         return;
     msix_free_irq_entries(dev);
 }
+#endif
diff --git a/hw/msix.h b/hw/msix.h
index a9f7993c39..a43b627b81 100644
--- a/hw/msix.h
+++ b/hw/msix.h
@@ -4,6 +4,38 @@
 #include "qemu-common.h"
 #include "pci.h"
 
+#if defined(TARGET_S390X)
+static int msix_init(PCIDevice *pdev, unsigned short nentries,
+              unsigned bar_nr, unsigned bar_size) { return 0; }
+
+static void msix_write_config(PCIDevice *pci_dev, uint32_t address,
+                       uint32_t val, int len) { }
+
+static void msix_mmio_map(PCIDevice *pci_dev, int region_num,
+                   pcibus_t addr, pcibus_t size, int type) { }
+
+static int msix_uninit(PCIDevice *d) { return 0; }
+
+static void msix_save(PCIDevice *dev, QEMUFile *f) { }
+static void msix_load(PCIDevice *dev, QEMUFile *f) { }
+
+static int msix_enabled(PCIDevice *dev) { return 0; }
+static int msix_present(PCIDevice *dev) { return 0; }
+
+static uint32_t msix_bar_size(PCIDevice *dev) { return 0; }
+
+static int msix_vector_use(PCIDevice *dev, unsigned vector) { return 0; }
+static void msix_vector_unuse(PCIDevice *dev, unsigned vector) { }
+static void msix_unuse_all_vectors(PCIDevice *dev) { }
+
+static void msix_notify(PCIDevice *dev, unsigned vector) { }
+
+static void msix_reset(PCIDevice *dev) { }
+
+//static int msix_supported = 0;
+
+#else
+
 int msix_init(PCIDevice *pdev, unsigned short nentries,
               unsigned bar_nr, unsigned bar_size);
 
@@ -34,3 +66,5 @@ void msix_reset(PCIDevice *dev);
 extern int msix_supported;
 
 #endif
+
+#endif
diff --git a/hw/s390-virtio.c b/hw/s390-virtio.c
index 0fa6ba68c9..ff104273f9 100644
--- a/hw/s390-virtio.c
+++ b/hw/s390-virtio.c
@@ -186,7 +186,7 @@ static void s390_init(ram_addr_t ram_size,
             exit(1);
         }
 
-        cpu_synchronize_state(env);
+        //cpu_synchronize_state(env);
         env->psw.addr = KERN_IMAGE_START;
         env->psw.mask = 0x0000000180000000ULL;
     }
@@ -247,6 +247,8 @@ static void s390_init(ram_addr_t ram_size,
         qdev_prop_set_drive(dev, "drive", dinfo);
         qdev_init_nofail(dev);
     }
+
+    kvm_arch_load_regs(env);
 }
 
 static QEMUMachine s390_machine = {
diff --git a/kvm-tpr-opt.c b/kvm-tpr-opt.c
index bf9c9a06e7..04097d5a7c 100644
--- a/kvm-tpr-opt.c
+++ b/kvm-tpr-opt.c
@@ -19,6 +19,7 @@
 
 #include <stdio.h>
 
+#if ! defined(TARGET_S390X)
 static uint64_t map_addr(struct kvm_sregs *sregs, target_ulong virt, unsigned *perms)
 {
     uint64_t mask = ((1ull << 48) - 1) & ~4095ull;
@@ -407,4 +408,4 @@ void kvm_tpr_opt_setup(void)
     register_ioport_write(0x7e, 1, 1, vtpr_ioport_write, NULL);
     register_ioport_write(0x7e, 2, 2, vtpr_ioport_write16, NULL);
 }
-
+#endif
diff --git a/kvm/include/linux/kvm.h b/kvm/include/linux/kvm.h
index 3fd3371a1a..15c79eb309 100644
--- a/kvm/include/linux/kvm.h
+++ b/kvm/include/linux/kvm.h
@@ -211,7 +211,9 @@ struct kvm_run {
 			__u64 data_offset; /* relative to kvm_run start */
 		} io;
 		struct {
+#if ! defined(TARGET_S390X)
 			struct kvm_debug_exit_arch arch;
+#endif
 		} debug;
 		/* KVM_EXIT_MMIO */
 		struct {
@@ -375,7 +377,9 @@ struct kvm_s390_interrupt {
 struct kvm_guest_debug {
 	__u32 control;
 	__u32 pad;
+#if ! defined(TARGET_S390X)
 	struct kvm_guest_debug_arch arch;
+#endif
 };
 
 enum {
diff --git a/qemu-kvm-helper.c b/qemu-kvm-helper.c
index 9420eb1763..e984adbfaf 100644
--- a/qemu-kvm-helper.c
+++ b/qemu-kvm-helper.c
@@ -30,7 +30,9 @@ void qemu_kvm_call_with_env(void (*func)(void *), void *data, CPUState *newenv)
 
 static void call_helper_cpuid(void *junk)
 {
+#if ! defined(TARGET_S390X)
     helper_cpuid();
+#endif
 }
 
 void qemu_kvm_cpuid_on_env(CPUState *env)
diff --git a/qemu-kvm.c b/qemu-kvm.c
index 8cc85bc4af..cd096a20ab 100644
--- a/qemu-kvm.c
+++ b/qemu-kvm.c
@@ -66,7 +66,11 @@ pthread_cond_t qemu_pause_cond = PTHREAD_COND_INITIALIZER;
 pthread_cond_t qemu_work_cond = PTHREAD_COND_INITIALIZER;
 __thread CPUState *current_env;
 
+#if defined(TARGET_S390X)
+int qemu_system_ready;
+#else
 static int qemu_system_ready;
+#endif
 
 #define SIG_IPI (SIGRTMIN+4)
 
@@ -157,7 +161,7 @@ static void init_slots(void)
 
 static int get_free_slot(kvm_context_t kvm)
 {
-    int i;
+    int i = 0;
     int tss_ext;
 
 #if defined(KVM_CAP_SET_TSS_ADDR) && !defined(__s390__)
@@ -171,14 +175,23 @@ static int get_free_slot(kvm_context_t kvm)
      * slot 0 to hold the extended memory, as the vmx will use the last 3
      * pages of this slot.
      */
-    if (tss_ext > 0)
-        i = 0;
-    else
+#if ! defined(TARGET_S390X)
+    if (tss_ext <= 0)
         i = 1;
 
     for (; i < KVM_MAX_NUM_MEM_REGIONS; ++i)
-        if (!slots[i].len)
+       if (!slots[i].len)
+            return i;
+#else
+    for (; i < KVM_MAX_NUM_MEM_REGIONS; ++i) {
+        DPRINTF("slot[%d].len = %#lx\n", i, slots[i].len);
+        if (!slots[i].len) {
             return i;
+        }
+        DPRINTF("slot[%d] skipped\n", i, slots[i].len);
+    }
+#endif
+
     return -1;
 }
 
@@ -451,6 +464,14 @@ static void kvm_create_vcpu(CPUState *env, int id)
     env->kvm_fd = r;
     env->kvm_state = kvm_state;
 
+#if defined(TARGET_S390X)
+    r = kvm_vcpu_ioctl(env, KVM_S390_INITIAL_RESET, 0);
+    if (r < 0) {
+        fprintf(stderr, "kvm_s390_initial_reset: %m\n");
+        exit(1);
+    }
+#endif
+
     mmap_size = kvm_ioctl(kvm_state, KVM_GET_VCPU_MMAP_SIZE, 0);
     if (mmap_size < 0) {
         fprintf(stderr, "get vcpu mmap size: %m\n");
@@ -943,12 +964,11 @@ int kvm_run(CPUState *env)
     }
 #endif
 
-#if !defined(__s390__)
     if (r == -1) {
         r = handle_io_window(kvm);
         goto more;
     }
-#endif
+
     if (1) {
         switch (run->exit_reason) {
         case KVM_EXIT_UNKNOWN:
@@ -986,7 +1006,7 @@ int kvm_run(CPUState *env)
         case KVM_EXIT_SHUTDOWN:
             r = handle_shutdown(kvm, env);
             break;
-#if defined(__s390__)
+#if 0
         case KVM_EXIT_S390_SIEIC:
             r = kvm_s390_handle_intercept(kvm, env, run);
             break;
@@ -1132,7 +1152,11 @@ int kvm_destroy_memory_region_works(kvm_context_t kvm)
     return ret;
 }
 
+#if defined(TARGET_S390X)
+static int kvm_reinject_control(kvm_context_t kvm, int pit_reinject)
+#else
 int kvm_reinject_control(kvm_context_t kvm, int pit_reinject)
+#endif
 {
 #ifdef KVM_CAP_REINJECT_CONTROL
     int r;
@@ -1632,10 +1656,12 @@ static void kvm_do_save_mpstate(void *_env)
     CPUState *env = _env;
 
     kvm_arch_save_mpstate(env);
-#ifdef KVM_CAP_MP_STATE
+#if KVM_CAP_MP_STATE
+#if ! defined(TARGET_S390X)
     if (kvm_irqchip_in_kernel())
         env->halted = (env->mp_state == KVM_MP_STATE_HALTED);
 #endif
+#endif
 }
 
 void kvm_save_mpstate(CPUState *env)
@@ -2303,11 +2329,13 @@ void kvm_set_phys_mem(target_phys_addr_t start_addr, ram_addr_t size,
             return;
 #endif
         while (size > 0) {
+#if ! defined(TARGET_S390X) 
             p = find_mapping(start_addr);
             if (p) {
                 kvm_unregister_memory_area(kvm_context, p->phys, p->len);
                 drop_mapping(p->phys);
             }
+#endif
             start_addr += TARGET_PAGE_SIZE;
             if (size > TARGET_PAGE_SIZE) {
                 size -= TARGET_PAGE_SIZE;
diff --git a/qemu-kvm.h b/qemu-kvm.h
index 6b3e5a1c46..3559a82c6f 100644
--- a/qemu-kvm.h
+++ b/qemu-kvm.h
@@ -670,6 +670,7 @@ int kvm_enable_vapic(CPUState *env, uint64_t vapic);
 
 #endif
 
+#if ! defined(TARGET_S390X)
 #if defined(__s390__)
 int kvm_s390_initial_reset(kvm_context_t kvm, int slot);
 int kvm_s390_interrupt(kvm_context_t kvm, int slot,
@@ -677,6 +678,7 @@ int kvm_s390_interrupt(kvm_context_t kvm, int slot,
 int kvm_s390_set_initial_psw(kvm_context_t kvm, int slot, psw_t psw);
 int kvm_s390_store_status(kvm_context_t kvm, int slot, unsigned long addr);
 #endif
+#endif
 
 #ifdef KVM_CAP_DEVICE_ASSIGNMENT
 /*!
diff --git a/target-s390x/kvm.c b/target-s390x/kvm.c
index 0992563924..50daca892d 100644
--- a/target-s390x/kvm.c
+++ b/target-s390x/kvm.c
@@ -70,10 +70,12 @@
 #define SCLP_CMDW_READ_SCP_INFO         0x00020001
 #define SCLP_CMDW_READ_SCP_INFO_FORCED  0x00120001
 
+#ifdef KVM_UPSTREAM
 int kvm_arch_init(KVMState *s, int smp_cpus)
 {
     return 0;
 }
+#endif
 
 int kvm_arch_init_vcpu(CPUState *env)
 {
@@ -86,17 +88,33 @@ int kvm_arch_init_vcpu(CPUState *env)
     return ret;
 }
 
+#ifdef KVM_UPSTREAM
 void kvm_arch_reset_vcpu(CPUState *env)
+#else
+void kvm_arch_cpu_reset(CPUState *env)
+#endif
 {
     /* FIXME: add code to reset vcpu. */
 }
 
+#ifdef KVM_UPSTREAM
 int kvm_arch_put_registers(CPUState *env)
+#else
+int _kvm_arch_load_regs(CPUState *env);
+
+void kvm_arch_load_regs(CPUState *env)
+{
+    _kvm_arch_load_regs(env);
+}
+
+int _kvm_arch_load_regs(CPUState *env)
+#endif
 {
     struct kvm_regs regs;
     int ret;
     int i;
 
+    dprintf("put regs!\n");
     ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
     if (ret < 0) {
         return ret;
@@ -117,14 +135,27 @@ int kvm_arch_put_registers(CPUState *env)
     return ret;
 }
 
+#ifdef KVM_UPSTREAM
 int kvm_arch_get_registers(CPUState *env)
+#else
+int _kvm_arch_save_regs(CPUState *env);
+
+void kvm_arch_save_regs(CPUState *env)
+{
+    _kvm_arch_save_regs(env);
+}
+
+int _kvm_arch_save_regs(CPUState *env)
+#endif
 {
     uint32_t ret;
     struct kvm_regs regs;
     int i;
 
+    dprintf("get regs!\n");
     ret = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
     if (ret < 0) {
+        dprintf("OH NOEZ!\n");
         return ret;
     }
 
@@ -180,6 +211,10 @@ static void kvm_s390_interrupt_internal(CPUState *env, int type, uint32_t parm,
 {
     struct kvm_s390_interrupt kvmint;
     int r;
+    extern int qemu_system_ready;
+
+    if (!qemu_system_ready)
+        return;
 
     if (!env->kvm_state) {
         return;
@@ -459,10 +494,14 @@ static int handle_intercept(CPUState *env)
             break;
     }
 
-    return r;
+    return 0; //r;
 }
 
+#ifdef KVM_UPSTREAM
 int kvm_arch_handle_exit(CPUState *env, struct kvm_run *run)
+#else
+static int kvm_arch_handle_exit(CPUState *env, struct kvm_run *run)
+#endif
 {
     int ret = 0;
 
@@ -476,8 +515,77 @@ int kvm_arch_handle_exit(CPUState *env, struct kvm_run *run)
             break;
         default:
             fprintf(stderr, "Unknown KVM exit: %d\n", run->exit_reason);
+            ret = -1;
             break;
     }
 
     return ret;
 }
+
+int kvm_arch_run(CPUState *env)
+{
+    return kvm_arch_handle_exit(env, env->kvm_run);
+}
+
+#ifndef KVM_UPSTREAM
+void kvm_arch_save_mpstate(CPUState *env)
+{
+}
+
+void kvm_arch_load_mpstate(CPUState *env)
+{
+}
+
+int kvm_arch_create(kvm_context_t kvm, unsigned long phys_mem_bytes,
+			void **vm_mem)
+{
+    return 0;
+}
+
+int kvm_arch_qemu_create_context(void)
+{
+    return 0;
+}
+
+void kvm_show_regs(CPUState *env)
+{
+    struct kvm_regs regs;
+    int i, r;
+
+    r = kvm_vcpu_ioctl(env, KVM_GET_REGS, &regs);
+    if (r < 0) {
+        perror("KVM_GET_REGS");
+        return;
+    }
+
+    for (i = 0; i < 16; i++) {
+        fprintf(stderr, "R%02d=%016lx", i, regs.gprs[i]);
+        if ((i % 4) == 3) {
+            fprintf(stderr, "\n");
+        } else {
+            fprintf(stderr, " ");
+        }
+    }
+
+    fprintf(stderr, "PSW=mask %016lx addr %016lx\n", env->kvm_run->psw_addr, env->kvm_run->psw_mask);
+    fprintf(stderr, "ENV PSW=mask %016lx addr %016lx\n", env->psw.addr, env->psw.mask);
+}
+
+int kvm_arch_halt(CPUState *env)
+{
+    return 1;
+}
+
+void kvm_show_code(CPUState *env)
+{
+}
+
+int kvm_arch_has_work(CPUState *env)
+{
+    return 1;
+}
+
+void kvm_arch_process_irqchip_events(CPUState *env)
+{
+}
+#endif
diff --git a/target-s390x/libkvm.h b/target-s390x/libkvm.h
new file mode 100644
index 0000000000..9a6bcd6a82
--- /dev/null
+++ b/target-s390x/libkvm.h
@@ -0,0 +1,26 @@
+/*
+ * This header is for functions & variables that will ONLY be
+ * used inside libkvm for x86.
+ * THESE ARE NOT EXPOSED TO THE USER AND ARE ONLY FOR USE
+ * WITHIN LIBKVM.
+ *
+ * derived from libkvm.c
+ *
+ * Copyright (C) 2006 Qumranet, Inc.
+ *
+ * Authors:
+ *	Avi Kivity   <avi@qumranet.com>
+ *	Yaniv Kamay  <yaniv@qumranet.com>
+ *
+ * This work is licensed under the GNU LGPL license, version 2.
+ */
+
+#ifndef KVM_X86_H
+#define KVM_X86_H
+
+#define PAGE_SIZE 4096ul
+#define PAGE_MASK (~(PAGE_SIZE - 1))
+
+#define smp_wmb()   asm volatile("" ::: "memory")
+
+#endif
