/* SPDX-License-Identifier: GPL-2.0-only */
#include <linux/export.h>
#include <asm/asm-offsets.h>
#include <asm/cache.h>
#include <asm/code-patching-asm.h>
#include <asm/exception-64s.h>
#include <asm/kvm_asm.h>
#include <asm/kvm_book3s_asm.h>
#include <asm/mmu.h>
#include <asm/ppc_asm.h>
#include <asm/ptrace.h>
#include <asm/reg.h>
#include <asm/ultravisor-api.h>
/*
* These are branched to from interrupt handlers in exception-64s.S which set
* IKVM_REAL or IKVM_VIRT, if HSTATE_IN_GUEST was found to be non-zero.
*/
/*
* This is a hcall, so register convention is as
* Documentation/arch/powerpc/papr_hcalls.rst.
*
* This may also be a syscall from PR-KVM userspace that is to be
* reflected to the PR guest kernel, so registers may be set up for
* a system call rather than hcall. We don't currently clobber
* anything here, but the 0xc00 handler has already clobbered CTR
* and CR0, so PR-KVM can not support a guest kernel that preserves
* those registers across its system calls.
*
* The state of registers is as kvmppc_interrupt, except CFAR is not
* saved, R13 is not in SCRATCH0, and R10 does not contain the trap.
*/
.global kvmppc_hcall
.balign IFETCH_ALIGN_BYTES
kvmppc_hcall:
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
lbz r10,HSTATE_IN_GUEST(r13)
cmpwi r10,KVM_GUEST_MODE_HV_P9
beq kvmppc_p9_exit_hcall
#endif
ld r10,PACA_EXGEN+EX_R13(r13)
SET_SCRATCH0(r10)
li r10,0xc00
/* Now we look like kvmppc_interrupt */
li r11,PACA_EXGEN
b .Lgot_save_area
/*
* KVM interrupt entry occurs after GEN_INT_ENTRY runs, and follows that
* call convention:
*
* guest R9-R13, CTR, CFAR, PPR saved in PACA EX_xxx save area
* guest (H)DAR, (H)DSISR are also in the save area for relevant interrupts
* guest R13 also saved in SCRATCH0
* R13 = PACA
* R11 = (H)SRR0
* R12 = (H)SRR1
* R9 = guest CR
* PPR is set to medium
*
* With the addition for KVM:
* R10 = trap vector
*/
.global kvmppc_interrupt
.balign IFETCH_ALIGN_BYTES
kvmppc_interrupt:
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
std r10,HSTATE_SCRATCH0(r13)
lbz r10,HSTATE_IN_GUEST(r13)
cmpwi r10,KVM_GUEST_MODE_HV_P9
beq kvmppc_p9_exit_interrupt
ld r10,HSTATE_SCRATCH0(r13)
#endif
li r11,PACA_EXGEN
cmpdi r10,0x200
bgt+ .Lgot_save_area
li r11,PACA_EXMC
beq .Lgot_save_area
li r11,PACA_EXNMI
.Lgot_save_area:
add r11,r11,r13
BEGIN_FTR_SECTION
ld r12,EX_CFAR(r11)
std r12,HSTATE_CFAR(r13)
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
ld r12,EX_CTR(r11)
mtctr r12
BEGIN_FTR_SECTION
ld r12,EX_PPR(r11)
std r12,HSTATE_PPR(r13)
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
ld r12,EX_R12(r11)
std r12,HSTATE_SCRATCH0(r13)
sldi r12,r9,32
or r12,r12,r10
ld r9,EX_R9(r11)
ld r10,EX_R10(r11)
ld r11,EX_R11(r11)
/*
* Hcalls and other interrupts come here after normalising register
* contents and save locations:
*
* R12 = (guest CR << 32) | interrupt vector
* R13 = PACA
* guest R12 saved in shadow HSTATE_SCRATCH0
* guest R13 saved in SPRN_SCRATCH0
*/
std r9,HSTATE_SCRATCH2(r13)
lbz r9,HSTATE_IN_GUEST(r13)
cmpwi r9,KVM_GUEST_MODE_SKIP
beq- .Lmaybe_skip
.Lno_skip:
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
#ifdef CONFIG_KVM_BOOK3S_PR_POSSIBLE
cmpwi r9,KVM_GUEST_MODE_GUEST
beq kvmppc_interrupt_pr
#endif
b kvmppc_interrupt_hv
#else
b kvmppc_interrupt_pr
#endif
/*
* "Skip" interrupts are part of a trick KVM uses a with hash guests to load
* the faulting instruction in guest memory from the hypervisor without
* walking page tables.
*
* When the guest takes a fault that requires the hypervisor to load the
* instruction (e.g., MMIO emulation), KVM is running in real-mode with HV=1
* and the guest MMU context loaded. It sets KVM_GUEST_MODE_SKIP, and sets
* MSR[DR]=1 while leaving MSR[IR]=0, so it continues to fetch HV instructions
* but loads and stores will access the guest context. This is used to load
* the faulting instruction using the faulting guest effective address.
*
* However the guest context may not be able to translate, or it may cause a
* machine check or other issue, which results in a fault in the host
* (even with KVM-HV).
*
* These faults come here because KVM_GUEST_MODE_SKIP was set, so if they
* are (or are likely) caused by that load, the instruction is skipped by
* just returning with the PC advanced +4, where it is noticed the load did
* not execute and it goes to the slow path which walks the page tables to
* read guest memory.
*/
.Lmaybe_skip:
cmpwi r12,BOOK3S_INTERRUPT_MACHINE_CHECK
beq 1f
cmpwi r12,BOOK3S_INTERRUPT_DATA_STORAGE
beq 1f
cmpwi r12,BOOK3S_INTERRUPT_DATA_SEGMENT
beq 1f
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
/* HSRR interrupts get 2 added to interrupt number */
cmpwi r12,BOOK3S_INTERRUPT_H_DATA_STORAGE | 0x2
beq 2f
#endif
b .Lno_skip
1: mfspr r9,SPRN_SRR0
addi r9,r9,4
mtspr SPRN_SRR0,r9
ld r12,HSTATE_SCRATCH0(r13)
ld r9,HSTATE_SCRATCH2(r13)
GET_SCRATCH0(r13)
RFI_TO_KERNEL
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
2: mfspr r9,SPRN_HSRR0
addi r9,r9,4
mtspr SPRN_HSRR0,r9
ld r12,HSTATE_SCRATCH0(r13)
ld r9,HSTATE_SCRATCH2(r13)
GET_SCRATCH0(r13)
HRFI_TO_KERNEL
#endif
#ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE
/* Stack frame offsets for kvmppc_p9_enter_guest */
#define SFS (144 + STACK_FRAME_MIN_SIZE)
#define STACK_SLOT_NVGPRS (SFS - 144) /* 18 gprs */
/*
* void kvmppc_p9_enter_guest(struct vcpu *vcpu);
*
* Enter the guest on a ISAv3.0 or later system.
*/
.balign IFETCH_ALIGN_BYTES
_GLOBAL(kvmppc_p9_enter_guest)
EXPORT_SYMBOL_GPL(kvmppc_p9_enter_guest)
mflr r0
std r0,PPC_LR_STKOFF(r1)
stdu r1,-SFS(r1)
std r1,HSTATE_HOST_R1(r13)
mfcr r4
stw r4,SFS+8(r1)
reg = 14
.rept 18
std reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1)
reg = reg + 1
.endr
ld r4,VCPU_LR(r3)
mtlr r4
ld r4,VCPU_CTR(r3)
mtctr r4
ld r4,VCPU_XER(r3)
mtspr SPRN_XER,r4
ld r1,VCPU_CR(r3)
BEGIN_FTR_SECTION
ld r4,VCPU_CFAR(r3)
mtspr SPRN_CFAR,r4
END_FTR_SECTION_IFSET(CPU_FTR_CFAR)
BEGIN_FTR_SECTION
ld r4,VCPU_PPR(r3)
mtspr SPRN_PPR,r4
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
reg = 4
.rept 28
ld reg,__VCPU_GPR(reg)(r3)
reg = reg + 1
.endr
ld r4,VCPU_KVM(r3)
lbz r4,KVM_SECURE_GUEST(r4)
cmpdi r4,0
ld r4,VCPU_GPR(R4)(r3)
bne .Lret_to_ultra
mtcr r1
ld r0,VCPU_GPR(R0)(r3)
ld r1,VCPU_GPR(R1)(r3)
ld r2,VCPU_GPR(R2)(r3)
ld r3,VCPU_GPR(R3)(r3)
HRFI_TO_GUEST
b .
/*
* Use UV_RETURN ultracall to return control back to the Ultravisor
* after processing an hypercall or interrupt that was forwarded
* (a.k.a. reflected) to the Hypervisor.
*
* All registers have already been reloaded except the ucall requires:
* R0 = hcall result
* R2 = SRR1, so UV can detect a synthesized interrupt (if any)
* R3 = UV_RETURN
*/
.Lret_to_ultra:
mtcr r1
ld r1,VCPU_GPR(R1)(r3)
ld r0,VCPU_GPR(R3)(r3)
mfspr r2,SPRN_SRR1
LOAD_REG_IMMEDIATE(r3, UV_RETURN)
sc 2
/*
* kvmppc_p9_exit_hcall and kvmppc_p9_exit_interrupt are branched to from
* above if the interrupt was taken for a guest that was entered via
* kvmppc_p9_enter_guest().
*
* The exit code recovers the host stack and vcpu pointer, saves all guest GPRs
* and CR, LR, XER as well as guest MSR and NIA into the VCPU, then re-
* establishes the host stack and registers to return from the
* kvmppc_p9_enter_guest() function, which saves CTR and other guest registers
* (SPRs and FP, VEC, etc).
*/
.balign IFETCH_ALIGN_BYTES
kvmppc_p9_exit_hcall:
mfspr r11,SPRN_SRR0
mfspr r12,SPRN_SRR1
li r10,0xc00
std r10,HSTATE_SCRATCH0(r13)
.balign IFETCH_ALIGN_BYTES
kvmppc_p9_exit_interrupt:
/*
* If set to KVM_GUEST_MODE_HV_P9 but we're still in the
* hypervisor, that means we can't return from the entry stack.
*/
rldicl. r10,r12,64-MSR_HV_LG,63
bne- kvmppc_p9_bad_interrupt
std r1,HSTATE_SCRATCH1(r13)
std r3,HSTATE_SCRATCH2(r13)
ld r1,HSTATE_HOST_R1(r13)
ld r3,HSTATE_KVM_VCPU(r13)
std r9,VCPU_CR(r3)
1:
std r11,VCPU_PC(r3)
std r12,VCPU_MSR(r3)
reg = 14
.rept 18
std reg,__VCPU_GPR(reg)(r3)
reg = reg + 1
.endr
/* r1, r3, r9-r13 are saved to vcpu by C code */
std r0,VCPU_GPR(R0)(r3)
std r2,VCPU_GPR(R2)(r3)
reg = 4
.rept 5
std reg,__VCPU_GPR(reg)(r3)
reg = reg + 1
.endr
LOAD_PACA_TOC()
mflr r4
std r4,VCPU_LR(r3)
mfspr r4,SPRN_XER
std r4,VCPU_XER(r3)
reg = 14
.rept 18
ld reg,STACK_SLOT_NVGPRS + ((reg - 14) * 8)(r1)
reg = reg + 1
.endr
lwz r4,SFS+8(r1)
mtcr r4
/*
* Flush the link stack here, before executing the first blr on the
* way out of the guest.
*
* The link stack won't match coming out of the guest anyway so the
* only cost is the flush itself. The call clobbers r0.
*/
1: nop
patch_site 1b patch__call_kvm_flush_link_stack_p9
addi r1,r1,SFS
ld r0,PPC_LR_STKOFF(r1)
mtlr r0
blr
/*
* Took an interrupt somewhere right before HRFID to guest, so registers are
* in a bad way. Return things hopefully enough to run host virtual code and
* run the Linux interrupt handler (SRESET or MCE) to print something useful.
*
* We could be really clever and save all host registers in known locations
* before setting HSTATE_IN_GUEST, then restoring them all here, and setting
* return address to a fixup that sets them up again. But that's a lot of
* effort for a small bit of code. Lots of other things to do first.
*/
kvmppc_p9_bad_interrupt:
BEGIN_MMU_FTR_SECTION
/*
* Hash host doesn't try to recover MMU (requires host SLB reload)
*/
b .
END_MMU_FTR_SECTION_IFCLR(MMU_FTR_TYPE_RADIX)
/*
* Clean up guest registers to give host a chance to run.
*/
li r10,0
mtspr SPRN_AMR,r10
mtspr SPRN_IAMR,r10
mtspr SPRN_CIABR,r10
mtspr SPRN_DAWRX0,r10
BEGIN_FTR_SECTION
mtspr SPRN_DAWRX1,r10
END_FTR_SECTION_IFSET(CPU_FTR_DAWR1)
/*
* Switch to host MMU mode (don't have the real host PID but we aren't
* going back to userspace).
*/
hwsync
isync
mtspr SPRN_PID,r10
ld r10, HSTATE_KVM_VCPU(r13)
ld r10, VCPU_KVM(r10)
lwz r10, KVM_HOST_LPID(r10)
mtspr SPRN_LPID,r10
ld r10, HSTATE_KVM_VCPU(r13)
ld r10, VCPU_KVM(r10)
ld r10, KVM_HOST_LPCR(r10)
mtspr SPRN_LPCR,r10
isync
/*
* Set GUEST_MODE_NONE so the handler won't branch to KVM, and clear
* MSR_RI in r12 ([H]SRR1) so the handler won't try to return.
*/
li r10,KVM_GUEST_MODE_NONE
stb r10,HSTATE_IN_GUEST(r13)
li r10,MSR_RI
andc r12,r12,r10
/*
* Go back to interrupt handler. MCE and SRESET have their specific
* PACA save area so they should be used directly. They set up their
* own stack. The other handlers all use EXGEN. They will use the
* guest r1 if it looks like a kernel stack, so just load the
* emergency stack and go to program check for all other interrupts.
*/
ld r10,HSTATE_SCRATCH0(r13)
cmpwi r10,BOOK3S_INTERRUPT_MACHINE_CHECK
beq .Lcall_machine_check_common
cmpwi r10,BOOK3S_INTERRUPT_SYSTEM_RESET
beq .Lcall_system_reset_common
b .
.Lcall_machine_check_common:
b machine_check_common
.Lcall_system_reset_common:
b system_reset_common
#endif
Codebase Browser
linux