#include <linux/types.h>
#include <linux/string.h>
#include <linux/kvm.h>
#include <linux/kvm_host.h>
#include <linux/highmem.h>
#include <asm/tlbflush.h>
#include <asm/kvm_ppc.h>
#include <asm/kvm_book3s.h>
#include <asm/mmu-hash64.h>
/* #define DEBUG_MMU */
#ifdef DEBUG_MMU
#define dprintk(X...) printk(KERN_INFO X)
#else
#define dprintk(X...) do { } while(0)
#endif
static void kvmppc_mmu_book3s_64_reset_msr(struct kvm_vcpu *vcpu)
{
kvmppc_set_msr(vcpu, MSR_SF);
}
static struct kvmppc_slb *kvmppc_mmu_book3s_64_find_slbe(
struct kvm_vcpu *vcpu,
gva_t eaddr)
{
int i;
u64 esid = GET_ESID(eaddr);
u64 esid_1t = GET_ESID_1T(eaddr);
for (i = 0; i < vcpu->arch.slb_nr; i++) {
u64 cmp_esid = esid;
if (!vcpu->arch.slb[i].valid)
continue;
if (vcpu->arch.slb[i].tb)
cmp_esid = esid_1t;
if (vcpu->arch.slb[i].esid == cmp_esid)
return &vcpu->arch.slb[i];
}
dprintk("KVM: No SLB entry found for 0x%lx [%llx | %llx]\n",
eaddr, esid, esid_1t);
for (i = 0; i < vcpu->arch.slb_nr; i++) {
if (vcpu->arch.slb[i].vsid)
dprintk(" %d: %c%c%c %llx %llx\n", i,
vcpu->arch.slb[i].valid ? 'v' : ' ',
vcpu->arch.slb[i].large ? 'l' : ' ',
vcpu->arch.slb[i].tb ? 't' : ' ',
vcpu->arch.slb[i].esid,
vcpu->arch.slb[i].vsid);
}
return NULL;
}
static int kvmppc_slb_sid_shift(struct kvmppc_slb *slbe)
{
return slbe->tb ? SID_SHIFT_1T : SID_SHIFT;
}
static u64 kvmppc_slb_offset_mask(struct kvmppc_slb *slbe)
{
return (1ul << kvmppc_slb_sid_shift(slbe)) - 1;
}
static u64 kvmppc_slb_calc_vpn(struct kvmppc_slb *slb, gva_t eaddr)
{
eaddr &= kvmppc_slb_offset_mask(slb);
return (eaddr >> VPN_SHIFT) |
((slb->vsid) << (kvmppc_slb_sid_shift(slb) - VPN_SHIFT));
}
static u64 kvmppc_mmu_book3s_64_ea_to_vp(struct kvm_vcpu *vcpu, gva_t eaddr,
bool data)
{
struct kvmppc_slb *slb;
slb = kvmppc_mmu_book3s_64_find_slbe(vcpu, eaddr);
if (!slb)
return 0;
return kvmppc_slb_calc_vpn(slb, eaddr);
}
static int mmu_pagesize(int mmu_pg)
{
switch (mmu_pg) {
case MMU_PAGE_64K:
return 16;
case MMU_PAGE_16M:
return 24;
}
return 12;
}
static int kvmppc_mmu_book3s_64_get_pagesize(struct kvmppc_slb *slbe)
{
return mmu_pagesize(slbe->base_page_size);
}
static u32 kvmppc_mmu_book3s_64_get_page(struct kvmppc_slb *slbe, gva_t eaddr)
{
int p = kvmppc_mmu_book3s_64_get_pagesize(slbe);
return ((eaddr & kvmppc_slb_offset_mask(slbe)) >> p);
}
static hva_t kvmppc_mmu_book3s_64_get_pteg(struct kvm_vcpu *vcpu,
struct kvmppc_slb *slbe, gva_t eaddr,
bool second)
{
struct kvmppc_vcpu_book3s *vcpu_book3s = to_book3s(vcpu);
u64 hash, pteg, htabsize;
u32 ssize;
hva_t r;
u64 vpn;
htabsize = ((1 << ((vcpu_book3s->sdr1 & 0x1f) + 11)) - 1);
vpn = kvmppc_slb_calc_vpn(slbe, eaddr);
ssize = slbe->tb ? MMU_SEGSIZE_1T : MMU_SEGSIZE_256M;
hash = hpt_hash(vpn, kvmppc_mmu_book3s_64_get_pagesize(slbe), ssize);
if (second)
hash = ~hash;
hash &= ((1ULL << 39ULL) - 1ULL);
hash &= htabsize;
hash <<= 7ULL;
pteg = vcpu_book3s->sdr1 & 0xfffffffffffc0000ULL;
pteg |= hash;
dprintk("MMU: page=0x%x sdr1=0x%llx pteg=0x%llx vsid=0x%llx\n",
page, vcpu_book3s->sdr1, pteg, slbe->vsid);
/* When running a PAPR guest, SDR1 contains a HVA address instead
of a GPA */
if (vcpu->arch.papr_enabled)
r = pteg;
else
r = gfn_to_hva(vcpu->kvm, pteg >> PAGE_SHIFT);
if (kvm_is_error_hva(r))
return r;
return r | (pteg & ~PAGE_MASK);
}
static u64 kvmppc_mmu_book3s_64_get_avpn(struct kvmppc_slb *slbe, gva_t eaddr)
{
int p = kvmppc_mmu_book3s_64_get_pagesize(slbe);
u64 avpn;
avpn = kvmppc_mmu_book3s_64_get_page(slbe, eaddr);
avpn |= slbe->vsid << (kvmppc_slb_sid_shift(slbe) - p);
if (p < 16)
avpn >>= ((80 - p) - 56) - 8; /* 16 - p */
else
avpn <<= p - 16;
return avpn;
}
/*
* Return page size encoded in the second word of a HPTE, or
* -1 for an invalid encoding for the base page size indicated by
* the SLB entry. This doesn't handle mixed pagesize segments yet.
*/
static int decode_pagesize(struct kvmppc_slb *slbe, u64 r)
{
switch (slbe->base_page_size) {
case MMU_PAGE_64K:
if ((r & 0xf000) == 0x1000)
return MMU_PAGE_64K;
break;
case MMU_PAGE_16M:
if ((r & 0xff000) == 0)
return MMU_PAGE_16M;
break;
}
return -1;
}
static int kvmppc_mmu_book3s_64_xlate(struct kvm_vcpu *vcpu, gva_t eaddr,
struct kvmppc_pte *gpte, bool data,
bool iswrite)
{
struct kvmppc_slb *slbe;
hva_t ptegp;
u64 pteg[16];
u64 avpn = 0;
u64 v, r;
u64 v_val, v_mask;
u64 eaddr_mask;
int i;
u8 pp, key = 0;
bool found = false;
bool second = false;
int pgsize;
ulong mp_ea = vcpu->arch.magic_page_ea;
/* Magic page override */
if (unlikely(mp_ea) &&
unlikely((eaddr & ~0xfffULL) == (mp_ea & ~0xfffULL)) &&
!(vcpu->arch.shared->msr & MSR_PR)) {
gpte->eaddr = eaddr;
gpte->vpage = kvmppc_mmu_book3s_64_ea_to_vp(vcpu, eaddr, data);
gpte->raddr = vcpu->arch.magic_page_pa | (gpte->raddr & 0xfff);
gpte->raddr &= KVM_PAM;
gpte->may_execute = true;
gpte->may_read = true;
gpte->may_write = true;
gpte->page_size = MMU_PAGE_4K;
return 0;
}
slbe = kvmppc_mmu_book3s_64_find_slbe(vcpu, eaddr);
if (!slbe)
goto no_seg_found;
avpn = kvmppc_mmu_book3s_64_get_avpn(slbe, eaddr);
v_val = avpn & HPTE_V_AVPN;
if (slbe->tb)
v_val |= SLB_VSID_B_1T;
if (slbe->large)
v_val |= HPTE_V_LARGE;
v_val |= HPTE_V_VALID;
v_mask = SLB_VSID_B | HPTE_V_AVPN | HPTE_V_LARGE | HPTE_V_VALID |
HPTE_V_SECONDARY;
pgsize = slbe->large ? MMU_PAGE_16M : MMU_PAGE_4K;
mutex_lock(&vcpu->kvm->arch.hpt_mutex);
do_second:
ptegp = kvmppc_mmu_book3s_64_get_pteg(vcpu, slbe, eaddr, second);
if (kvm_is_error_hva(ptegp))
goto no_page_found;
if(copy_from_user(pteg, (void __user *)ptegp, sizeof(pteg))) {
printk(KERN_ERR "KVM can't copy data from 0x%lx!\n", ptegp);
goto no_page_found;
}
if ((vcpu->arch.shared->msr & MSR_PR) && slbe->Kp)
key = 4;
else if (!(vcpu->arch.shared->msr & MSR_PR) && slbe->Ks)
key = 4;
for (i=0; i<16; i+=2) {
/* Check all relevant fields of 1st dword */
if ((pteg[i] & v_mask) == v_val) {
/* If large page bit is set, check pgsize encoding */
if (slbe->large &&
(vcpu->arch.hflags & BOOK3S_HFLAG_MULTI_PGSIZE)) {
pgsize = decode_pagesize(slbe, pteg[i+1]);
if (pgsize < 0)
continue;
}
found = true;
break;
}
}
if (!found) {
if (second)
goto no_page_found;
v_val |= HPTE_V_SECONDARY;
second = true;
goto do_second;
}
v = pteg[i];
r = pteg[i+1];
pp = (r & HPTE_R_PP) | key;
if (r & HPTE_R_PP0)
pp |= 8;
gpte->eaddr = eaddr;
gpte->vpage = kvmppc_mmu_book3s_64_ea_to_vp(vcpu, eaddr, data);
eaddr_mask = (1ull << mmu_pagesize(pgsize)) - 1;
gpte->raddr = (r & HPTE_R_RPN & ~eaddr_mask) | (eaddr & eaddr_mask);
gpte->page_size = pgsize;
gpte->may_execute = ((r & HPTE_R_N) ? false : true);
gpte->may_read = false;
gpte->may_write = false;
switch (pp) {
case 0:
case 1:
case 2:
case 6:
gpte->may_write = true;
/* fall through */
case 3:
case 5:
case 7:
case 10:
gpte->may_read = true;
break;
}
dprintk("KVM MMU: Translated 0x%lx [0x%llx] -> 0x%llx "
"-> 0x%lx\n",
eaddr, avpn, gpte->vpage, gpte->raddr);
/* Update PTE R and C bits, so the guest's swapper knows we used the
* page */
if (gpte->may_read && !(r & HPTE_R_R)) {
/*
* Set the accessed flag.
* We have to write this back with a single byte write
* because another vcpu may be accessing this on
* non-PAPR platforms such as mac99, and this is
* what real hardware does.
*/
char __user *addr = (char __user *) &pteg[i+1];
r |= HPTE_R_R;
put_user(r >> 8, addr + 6);
}
if (iswrite && gpte->may_write && !(r & HPTE_R_C)) {
/* Set the dirty flag */
/* Use a single byte write */
char __user *addr = (char __user *) &pteg[i+1];
r |= HPTE_R_C;
put_user(r, addr + 7);
}
mutex_unlock(&vcpu->kvm->arch.hpt_mutex);
if (!gpte->may_read || (iswrite && !gpte->may_write))
return -EPERM;
return 0;
no_page_found: