From f0ed194236b1fe55199ee82c014b70119ee3f227 Mon Sep 17 00:00:00 2001
From: Michael Roth <michael.roth@amd.com>
Date: Thu, 2 May 2024 13:49:26 +0200
Subject: [PATCH] OvmfPkg: Don't make APIC MMIO accesses with encryption bit
 set

For the most part, OVMF will clear the encryption bit for MMIO regions,
but there is currently one known exception during SEC when the APIC
base address is accessed via MMIO with the encryption bit set for
SEV-ES/SEV-SNP guests. In the case of SEV-SNP, this requires special
handling on the hypervisor side which may not be available in the
future[1], so make the necessary changes in the SEC-configured page
table to clear the encryption bit for 4K region containing the APIC
base address.

[1] https://lore.kernel.org/lkml/20240208002420.34mvemnzrwwsaesw@amd.com/#t

Suggested-by: Tom Lendacky <thomas.lendacky@amd.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Gerd Hoffmann <kraxel@redhat.com>
Cc: Erdem Aktas <erdemaktas@google.com>
Cc: Jiewen Yao <jiewen.yao@intel.com>
Cc: Min Xu <min.m.xu@intel.com>
Cc: Tom Lendacky <thomas.lendacky@amd.com>
Cc: Jianyong Wu <jianyong.wu@arm.com>
Cc: Anatol Belski <anbelski@linux.microsoft.com>
Signed-off-by: Michael Roth <michael.roth@amd.com>
Reviewed-by: Gerd Hoffmann <kraxel@redhat.com>
---
 OvmfPkg/AmdSev/AmdSevX64.fdf   |  5 ++-
 OvmfPkg/Bhyve/BhyveX64.dsc     |  1 +
 OvmfPkg/CloudHv/CloudHvX64.fdf |  5 ++-
 OvmfPkg/Microvm/MicrovmX64.fdf |  3 ++
 OvmfPkg/OvmfPkg.dec            |  5 +++
 OvmfPkg/OvmfPkgX64.fdf         |  5 ++-
 OvmfPkg/Sec/AmdSev.c           | 58 ++++++++++++++++++++++++++++++++++
 OvmfPkg/Sec/AmdSev.h           | 14 ++++++++
 OvmfPkg/Sec/SecMain.c          |  1 +
 OvmfPkg/Sec/SecMain.inf        |  3 ++
 10 files changed, 97 insertions(+), 3 deletions(-)

Index: edk2-edk2-stable202308/OvmfPkg/AmdSev/AmdSevX64.fdf
===================================================================
--- edk2-edk2-stable202308.orig/OvmfPkg/AmdSev/AmdSevX64.fdf
+++ edk2-edk2-stable202308/OvmfPkg/AmdSev/AmdSevX64.fdf
@@ -74,7 +74,10 @@ gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSnpSec
 0x00F000|0x001000
 gUefiOvmfPkgTokenSpaceGuid.PcdOvmfCpuidBase|gUefiOvmfPkgTokenSpaceGuid.PcdOvmfCpuidSize
 
-0x010000|0x010000
+0x010000|0x001000
+gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecApicPageTableBase|gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecApicPageTableSize
+
+0x011000|0x00F000
 gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecPeiTempRamBase|gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecPeiTempRamSize
 
 0x020000|0x0E0000
Index: edk2-edk2-stable202308/OvmfPkg/CloudHv/CloudHvX64.fdf
===================================================================
--- edk2-edk2-stable202308.orig/OvmfPkg/CloudHv/CloudHvX64.fdf
+++ edk2-edk2-stable202308/OvmfPkg/CloudHv/CloudHvX64.fdf
@@ -76,7 +76,10 @@ gUefiOvmfPkgTokenSpaceGuid.PcdOvmfCpuidB
 0x00F000|0x001000
 gUefiOvmfPkgTokenSpaceGuid.PcdXenPvhStartOfDayStructPtr|gUefiOvmfPkgTokenSpaceGuid.PcdXenPvhStartOfDayStructPtrSize
 
-0x010000|0x010000
+0x010000|0x001000
+gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecApicPageTableBase|gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecApicPageTableSize
+
+0x011000|0x00F000
 gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecPeiTempRamBase|gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecPeiTempRamSize
 
 0x020000|0x0E0000
Index: edk2-edk2-stable202308/OvmfPkg/Microvm/MicrovmX64.fdf
===================================================================
--- edk2-edk2-stable202308.orig/OvmfPkg/Microvm/MicrovmX64.fdf
+++ edk2-edk2-stable202308/OvmfPkg/Microvm/MicrovmX64.fdf
@@ -62,6 +62,9 @@ gUefiOvmfPkgTokenSpaceGuid.PcdOvmfWorkAr
 0x00C000|0x001000
 gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecGhcbBackupBase|gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecGhcbBackupSize
 
+0x00D000|0x001000
+gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecApicPageTableBase|gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecApicPageTableSize
+
 0x010000|0x010000
 gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecPeiTempRamBase|gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecPeiTempRamSize
 
Index: edk2-edk2-stable202308/OvmfPkg/OvmfPkg.dec
===================================================================
--- edk2-edk2-stable202308.orig/OvmfPkg/OvmfPkg.dec
+++ edk2-edk2-stable202308/OvmfPkg/OvmfPkg.dec
@@ -368,6 +368,11 @@
   gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecGhcbBackupBase|0|UINT32|0x44
   gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecGhcbBackupSize|0|UINT32|0x45
 
+  ## Specify the extra page table needed to mark the APIC MMIO range as unencrypted.
+  #  The value should be a multiple of 4KB for each.
+  gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecApicPageTableBase|0x0|UINT32|0x72
+  gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecApicPageTableSize|0x0|UINT32|0x73
+
   ## The base address and size of the SEV Launch Secret Area provisioned
   #  after remote attestation.  If this is set in the .fdf, the platform
   #  is responsible for protecting the area from DXE phase overwrites.
Index: edk2-edk2-stable202308/OvmfPkg/Sec/AmdSev.c
===================================================================
--- edk2-edk2-stable202308.orig/OvmfPkg/Sec/AmdSev.c
+++ edk2-edk2-stable202308/OvmfPkg/Sec/AmdSev.c
@@ -8,7 +8,10 @@
 **/
 
 #include <Library/BaseLib.h>
+#include <Library/CpuLib.h>
+#include <Library/CpuPageTableLib.h>
 #include <Library/DebugLib.h>
+#include <Library/LocalApicLib.h>
 #include <Library/MemEncryptSevLib.h>
 #include <Library/BaseMemoryLib.h>
 #include <Register/Amd/Ghcb.h>
@@ -301,3 +304,58 @@ SecValidateSystemRam (
     MemEncryptSevSnpPreValidateSystemRam (Start, EFI_SIZE_TO_PAGES ((UINTN)(End - Start)));
   }
 }
+
+/**
+  Map known MMIO regions unencrypted if SEV-ES is active.
+
+  During early booting, page table entries default to having the encryption bit
+  set for SEV-ES/SEV-SNP guests. In cases where there is MMIO to an address, the
+  encryption bit should be cleared. Clear it here for any known MMIO accesses
+  during SEC, which is currently just the APIC base address.
+
+**/
+VOID
+SecMapApicBaseUnencrypted (
+  VOID
+  )
+{
+  PHYSICAL_ADDRESS    Cr3;
+  UINT64              ApicAddress;
+  VOID                *Buffer;
+  UINTN               BufferSize;
+  IA32_MAP_ATTRIBUTE  MapAttribute;
+  IA32_MAP_ATTRIBUTE  MapMask;
+  RETURN_STATUS       Status;
+
+  if (!SevEsIsEnabled ()) {
+    return;
+  }
+
+  ApicAddress = (UINT64)GetLocalApicBaseAddress ();
+  Buffer      = (VOID *)(UINTN)FixedPcdGet32 (PcdOvmfSecApicPageTableBase);
+  Cr3         = AsmReadCr3 ();
+
+  MapAttribute.Uint64         = ApicAddress;
+  MapAttribute.Bits.Present   = 1;
+  MapAttribute.Bits.ReadWrite = 1;
+  MapMask.Uint64              = MAX_UINT64;
+  BufferSize                  = SIZE_4KB;
+
+  Status = PageTableMap (
+             (UINTN *)&Cr3,
+             Paging4Level,
+             Buffer,
+             &BufferSize,
+             ApicAddress,
+             SIZE_4KB,
+             &MapAttribute,
+             &MapMask,
+             NULL
+             );
+  if (RETURN_ERROR (Status)) {
+    DEBUG ((DEBUG_ERROR, "Failed to map APIC MMIO region as unencrypted: %d\n", Status));
+    ASSERT (FALSE);
+  }
+
+  CpuFlushTlb ();
+}
Index: edk2-edk2-stable202308/OvmfPkg/Sec/AmdSev.h
===================================================================
--- edk2-edk2-stable202308.orig/OvmfPkg/Sec/AmdSev.h
+++ edk2-edk2-stable202308/OvmfPkg/Sec/AmdSev.h
@@ -91,4 +91,18 @@ SevSnpIsEnabled (
   VOID
   );
 
+/**
+  Map MMIO regions unencrypted if SEV-ES is active.
+
+  During early booting, page table entries default to having the encryption bit
+  set for SEV-ES/SEV-SNP guests. In cases where there is MMIO to an address, the
+  encryption bit should be cleared. Clear it here for any known MMIO accesses
+  during SEC, which is currently just the APIC base address.
+
+**/
+VOID
+SecMapApicBaseUnencrypted (
+  VOID
+  );
+
 #endif
Index: edk2-edk2-stable202308/OvmfPkg/Sec/SecMain.c
===================================================================
--- edk2-edk2-stable202308.orig/OvmfPkg/Sec/SecMain.c
+++ edk2-edk2-stable202308/OvmfPkg/Sec/SecMain.c
@@ -939,6 +939,7 @@ SecCoreStartupWithStack (
   // interrupts before initializing the Debug Agent and the debug timer is
   // enabled.
   //
+  SecMapApicBaseUnencrypted ();
   InitializeApicTimer (0, MAX_UINT32, TRUE, 5);
   DisableApicTimerInterrupt ();
 
Index: edk2-edk2-stable202308/OvmfPkg/Sec/SecMain.inf
===================================================================
--- edk2-edk2-stable202308.orig/OvmfPkg/Sec/SecMain.inf
+++ edk2-edk2-stable202308/OvmfPkg/Sec/SecMain.inf
@@ -55,6 +55,7 @@
   MemEncryptSevLib
   CpuExceptionHandlerLib
   CcProbeLib
+  CpuPageTableLib
 
 [Ppis]
   gEfiTemporaryRamSupportPpiGuid                # PPI ALWAYS_PRODUCED
@@ -83,6 +84,8 @@
   gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecGhcbBackupBase
   gUefiOvmfPkgTokenSpaceGuid.PcdTdxAcceptPageSize
   gUefiOvmfPkgTokenSpaceGuid.PcdOvmfWorkAreaBase
+  gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecApicPageTableBase
+  gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecApicPageTableSize
 
 [FeaturePcd]
   gUefiOvmfPkgTokenSpaceGuid.PcdSmmSmramRequire
Index: edk2-edk2-stable202308/OvmfPkg/Bhyve/BhyveX64.dsc
===================================================================
--- edk2-edk2-stable202308.orig/OvmfPkg/Bhyve/BhyveX64.dsc
+++ edk2-edk2-stable202308/OvmfPkg/Bhyve/BhyveX64.dsc
@@ -173,6 +173,7 @@
   MemEncryptTdxLib|OvmfPkg/Library/BaseMemEncryptTdxLib/BaseMemEncryptTdxLib.inf
   PeiHardwareInfoLib|OvmfPkg/Library/HardwareInfoLib/PeiHardwareInfoLib.inf
   DxeHardwareInfoLib|OvmfPkg/Library/HardwareInfoLib/DxeHardwareInfoLib.inf
+  CpuPageTableLib|UefiCpuPkg/Library/CpuPageTableLib/CpuPageTableLib.inf
 
   CustomizedDisplayLib|MdeModulePkg/Library/CustomizedDisplayLib/CustomizedDisplayLib.inf
   FrameBufferBltLib|MdeModulePkg/Library/FrameBufferBltLib/FrameBufferBltLib.inf
Index: edk2-edk2-stable202308/OvmfPkg/OvmfPkgX64.fdf
===================================================================
--- edk2-edk2-stable202308.orig/OvmfPkg/OvmfPkgX64.fdf
+++ edk2-edk2-stable202308/OvmfPkg/OvmfPkgX64.fdf
@@ -94,6 +94,9 @@ gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSnpSec
 0x00E000|0x001000
 gUefiOvmfPkgTokenSpaceGuid.PcdOvmfCpuidBase|gUefiOvmfPkgTokenSpaceGuid.PcdOvmfCpuidSize
 
+0x00F000|0x001000
+gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecApicPageTableBase|gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecApicPageTableSize
+ 
 0x010000|0x010000
 gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecPeiTempRamBase|gUefiOvmfPkgTokenSpaceGuid.PcdOvmfSecPeiTempRamSize
 
