ref: 220dfb87f561f73ec1d154e08511332a660d1119
dir: /sys/src/9/pc/mtrr.c/
#include "u.h" #include "../port/lib.h" #include "mem.h" #include "dat.h" #include "fns.h" #include "io.h" enum { /* * MTRR Physical base/mask are indexed by * MTRRPhys{Base|Mask}N = MTRRPhys{Base|Mask}0 + 2*N */ MTRRPhysBase0 = 0x200, MTRRPhysMask0 = 0x201, MTRRDefaultType = 0x2FF, Deftype = 0xFF, /* default MTRR type */ Deffixena = 1<<10, /* fixed-range MTRR enable */ Defena = 1<<11, /* MTRR enable */ MTRRCap = 0xFE, Capvcnt = 0xFF, /* mask: # of variable-range MTRRs we have */ Capwc = 1<<8, /* flag: have write combining? */ Capfix = 1<<10, /* flag: have fixed MTRRs? */ AMDK8SysCfg = 0xC0010010, Tom2Enabled = 1<<21, Tom2ForceMemTypeWB = 1<<22, AMDK8TopMem2 = 0xC001001D, }; enum { Nvarreg = 8, Nfixreg = 11*8, Nranges = Nfixreg+Nvarreg*2+1, }; typedef struct Varreg Varreg; struct Varreg { vlong base; vlong mask; }; typedef struct Fixreg Fixreg; struct Fixreg { int msr; ulong base; ulong size; }; typedef struct State State; struct State { uvlong mask; vlong cap; vlong def; vlong tom2; int nvarreg; Varreg varreg[Nvarreg]; vlong fixreg[Nfixreg/8]; }; typedef struct Range Range; struct Range { uvlong base; uvlong size; int type; }; enum { Uncacheable = 0, Writecomb = 1, Unknown1 = 2, Unknown2 = 3, Writethru = 4, Writeprot = 5, Writeback = 6, }; static char *types[] = { [Uncacheable] "uc", [Writecomb] "wc", [Unknown1] "uk1", [Unknown2] "uk2", [Writethru] "wt", [Writeprot] "wp", [Writeback] "wb", }; static char * type2str(int type) { if(type < 0 || type >= nelem(types)) return nil; return types[type]; } static int str2type(char *str) { int type; for(type = 0; type < nelem(types); type++){ if(strcmp(str, types[type]) == 0) return type; } return -1; } static int getvarreg(State *s, Range *rp, int index) { Varreg *reg = &s->varreg[index]; if((reg->mask & (1<<11)) == 0) return 0; rp->base = reg->base & ~0xFFFULL; rp->type = reg->base & 0xFF; rp->size = (s->mask ^ (reg->mask & ~0xFFFULL)) + 1; return 1; } static void setvarreg(State *s, Range *rp, int index) { Varreg *reg = &s->varreg[index]; if(rp == nil || rp->size == 0){ reg->base = 0; reg->mask = 0; return; } reg->base = rp->base | (rp->type & 0xFF); reg->mask = (s->mask & ~(rp->size-1)) | 1<<11; } static Fixreg fixreg[Nfixreg/8] = { 0x250, 0x00000, 0x10000, 0x258, 0x80000, 0x04000, 0x259, 0xA0000, 0x04000, 0x268, 0xC0000, 0x01000, 0x269, 0xC8000, 0x01000, 0x26A, 0xD0000, 0x01000, 0x26B, 0xD8000, 0x01000, 0x26C, 0xE0000, 0x01000, 0x26D, 0xE8000, 0x01000, 0x26E, 0xF0000, 0x01000, 0x26F, 0xF8000, 0x01000, }; static int getfixreg(State *s, Range *rp, int index) { Fixreg *reg = &fixreg[index >> 3]; index &= 7; rp->base = reg->base + reg->size * index; rp->size = reg->size; rp->type = ((uvlong)s->fixreg[reg - fixreg] >> 8*index) & 0xFF; return 1; } static void setfixreg(State *s, Range *rp, int index) { Fixreg *reg = &fixreg[index >> 3]; int type; index &= 7; if(rp == nil || rp->size == 0) type = Uncacheable; else type = rp->type & 0xFF; s->fixreg[reg - fixreg] &= ~(0xFFULL << 8*index); s->fixreg[reg - fixreg] |= (uvlong)type << 8*index; } static int preftype(int a, int b) { if(a == b) return a; if(a == Uncacheable || b == Uncacheable) return Uncacheable; if(a == Writethru && b == Writeback || a == Writeback && b == Writethru) return Writethru; return -1; } static int gettype(State *s, uvlong pa, Range *new) { int i, type; Range r; if(new != nil && pa >= new->base && pa < new->base + new->size) return new->type; if((s->def & Defena) == 0) return Uncacheable; if(pa < 0x100000 && (s->def & Deffixena) != 0){ for(i = 0; i < Nfixreg; i++){ if(getfixreg(s, &r, i) && pa < r.base + r.size && pa >= r.base) return r.type; } } if(pa >= 0x100000000ULL && pa < s->tom2) return Writeback; type = -1; for(i = 0; i < s->nvarreg; i++){ if(!getvarreg(s, &r, i)) continue; if((pa & -r.size) == r.base) type = (type == -1) ? r.type : preftype(r.type, type); } if(type == -1) type = s->def & Deftype; return type; } static uvlong getnext(State *s, uvlong pa, Range *new) { uvlong end; Range r; int i; if(new != nil){ end = getnext(s, pa, nil); if(pa < new->base && end > new->base) return new->base; if(pa < new->base + new->size && end > new->base + new->size) return new->base + new->size; return end; } end = s->mask+1; if((s->def & Defena) == 0) return end; if(pa < 0x100000 && (s->def & Deffixena) != 0){ for(i = 0; i < Nfixreg; i++){ if(getfixreg(s, &r, i) && pa < r.base + r.size && pa >= r.base) return r.base + r.size; } } if(pa >= 0x100000000ULL && pa < s->tom2) return s->tom2; for(i = 0; i < s->nvarreg; i++){ if(!getvarreg(s, &r, i)) continue; if((pa & -r.size) == r.base) r.base += r.size; else if(r.base <= pa) continue; if(r.base < end) end = r.base; } if(pa < 0x100000000ULL && end > 0x100000000ULL) end = 0x100000000ULL; return end; } enum { Exthighfunc = 1ul << 31, Extprocsigamd, Extprocname0, Extprocname1, Extprocname2, Exttlbl1, Extl2, Extapm, Extaddrsz, }; static uvlong physmask(void) { ulong regs[4]; uvlong mask; cpuid(Exthighfunc, 0, regs); if(regs[0] >= Extaddrsz) { /* ax */ cpuid(Extaddrsz, 0, regs); mask = (1ULL << (regs[0] & 0xFF)) - 1; /* ax */ } else { mask = (1ULL << 36) - 1; } return mask; } static int getstate(State *s) { vlong v; int i; if(rdmsr(MTRRCap, &s->cap) < 0) return -1; if((s->cap & (Capfix|Capvcnt)) == 0) return -1; if(rdmsr(MTRRDefaultType, &s->def) < 0) return -1; if(s->cap & Capfix){ for(i = 0; i < nelem(fixreg); i++){ if(rdmsr(fixreg[i].msr, &s->fixreg[i]) < 0) return -1; } } else { s->def &= ~(vlong)Deffixena; } s->nvarreg = s->cap & Capvcnt; if(s->nvarreg > Nvarreg) s->nvarreg = Nvarreg; for(i = 0; i < s->nvarreg; i++){ if(rdmsr(MTRRPhysBase0 + 2*i, &s->varreg[i].base) < 0) return -1; if(rdmsr(MTRRPhysMask0 + 2*i, &s->varreg[i].mask) < 0) return -1; } s->mask = physmask(); if(strcmp(m->cpuidid, "AuthenticAMD") != 0 || m->cpuidfamily < 15 || rdmsr(AMDK8SysCfg, &v) < 0 || (v & (Tom2Enabled|Tom2ForceMemTypeWB)) != (Tom2Enabled|Tom2ForceMemTypeWB) || rdmsr(AMDK8TopMem2, &s->tom2) < 0) s->tom2 = 0; else { s->tom2 &= s->mask; s->tom2 &= -0x800000LL; } return 0; } enum { CR4PageGlobalEnable = 1 << 7, CR0CacheDisable = 1 << 30, }; static void putstate(State *s) { uintptr cr0, cr4; int i, x; x = splhi(); /* disable cache */ cr0 = getcr0(); putcr0(cr0 | CR0CacheDisable); wbinvd(); /* disable PGE */ cr4 = getcr4(); putcr4(cr4 & ~CR4PageGlobalEnable); /* flush tlb */ putcr3(getcr3()); /* disable MTRRs */ wrmsr(MTRRDefaultType, s->def & ~(vlong)(Defena|Deffixena|Deftype)); wbinvd(); /* write all registers */ if(s->cap & Capfix){ for(i = 0; i < nelem(fixreg); i++) wrmsr(fixreg[i].msr, s->fixreg[i]); } for(i = 0; i < s->nvarreg; i++){ wrmsr(MTRRPhysBase0 + 2*i, s->varreg[i].base); wrmsr(MTRRPhysMask0 + 2*i, s->varreg[i].mask); } /* flush tlb */ putcr3(getcr3()); /* enable MTRRs */ wrmsr(MTRRDefaultType, s->def); /* reenable cache */ putcr0(cr0); /* reenable PGE */ putcr4(cr4); splx(x); } static int fls64(uvlong x) { int i; for(i = 0; i < 64; i++) if(x & (1ULL<<i)) break; return i; } static int fms64(uvlong x) { int i; if(x == 0) return 0; for(i = 63; i >= 0; i--) if(x & (1ULL<<i)) break; return i; } static int range2varreg(State *s, Range r, int index, int doit) { uvlong len; if(index < 0) return -1; if(r.base <= 0x100000 && (s->def & Deffixena) != 0){ r.size += r.base; r.base = 0; } if(r.base >= 0x100000000ULL && r.base <= s->tom2){ if(r.base + r.size <= s->tom2){ if(r.type != Writeback) return -1; return index; } } len = r.size; while(len){ if(index >= s->nvarreg) return -1; if(fls64(r.base) > fms64(len)) r.size = 1ULL << fms64(len); else r.size = 1ULL << fls64(r.base); if(doit) setvarreg(s, &r, index); index++; len -= r.size; r.base += r.size; } return index; } static int ranges2varregs(State*, Range*, int, int, int); /* * try to combine same type ranges that are split by * higher precedence ranges. */ static int ranges2varregscomb(State *s, Range *rp, int nr, int index, int doit) { Range rr; int i, j; if(nr < 2 || rp[0].type == rp[1].type) return -1; rr = rp[0]; if(preftype(rr.type, rp[1].type) == rr.type) rr.type = rp[1].type; for(j = 1; j < nr; j++){ if(rp[j].type != rr.type && preftype(rp[j].type, rr.type) != rp[j].type) return -1; rr.size += rp[j].size; } i = ranges2varregs(s, &rr, 1, index, doit); for(j = 0; j < nr && i >= index; j++){ if(rp[j].type != rr.type) i = range2varreg(s, rp[j], i, doit); } return i; } static int ranges2varregs(State *s, Range *rp, int nr, int index, int doit) { int i, j, k; if(nr == 1){ if(rp->type == (s->def & Deftype)) return index; return range2varreg(s, *rp, index, doit); } /* try combining */ i = ranges2varregscomb(s, rp, nr, index, doit); /* * now see if we can find a better solution using * different splittings. */ for(k = 1; k < nr; k++){ j = ranges2varregs(s, rp+k, nr-k, ranges2varregs(s, rp, k, index, 0), 0); if(j < 0) continue; if(i < 0 || j < i) i = doit ? ranges2varregs(s, rp+k, nr-k, ranges2varregs(s, rp, k, index, 1), 1) : j; } return i; } static int range2fixreg(State *s, Range r) { Range rr; int i; for(i = 0; i < Nfixreg; i++){ if(!getfixreg(s, &rr, i) || rr.base + rr.size <= r.base) continue; if(rr.base >= r.base + r.size) break; if(r.base > rr.base || r.base + r.size < rr.base + rr.size) return -1; rr.type = r.type; setfixreg(s, &rr, i); } return 0; } static int setranges(State *s, Range *rp, int nr) { int i, j; if(nr < 1 || nr > Nranges) return -1; s->def &= ~(vlong)(Defena|Deffixena|Deftype); i = 0; if(rp[0].size != s->mask+1 || rp[0].type != Uncacheable){ s->def |= Defena; /* first handle ranges below 1MB using fixed registers */ if(rp[0].size < 0x100000 && (s->cap & Capfix) != 0){ s->def |= Deffixena; for(i = 0; i < Nfixreg; i++) setfixreg(s, nil, i); while(nr > 0 && rp->base < 0x100000){ if(range2fixreg(s, *rp) < 0) return -1; if(rp->base + rp->size > 0x100000) break; rp++; nr--; } } /* remaining ranges to to variable registers */ if(nr > 0){ /* make sure the algorithm doesnt explode */ if(nr > Nvarreg+1) return -1; /* try with UC default type */ s->def = (s->def & ~(vlong)Deftype) | Uncacheable; i = ranges2varregs(s, rp, nr, 0, 1); /* try with WB default type, dont do it yet */ s->def = (s->def & ~(vlong)Deftype) | Writeback; j = ranges2varregs(s, rp, nr, 0, 0); if(j < 0 || (i >= 0 && i <= j)){ /* WB not better or worse, use UC solution */ s->def = (s->def & ~(vlong)Deftype) | Uncacheable; } else { /* WB default is better, doit! */ i = ranges2varregs(s, rp, nr, 0, 1); } if(i < 0) return -1; } } /* clear unused variable registers */ for(; i < s->nvarreg; i++) setvarreg(s, nil, i); return 0; } static int checkranges(State *s, Range *rp, int nr) { uvlong base, next; int i; for(i = 0; i < nr; i++){ next = rp[i].base + rp[i].size; for(base = rp[i].base; base < next; base = getnext(s, base, nil)){ if(gettype(s, base, nil) != rp[i].type) return -1; } } return 0; } static int getranges(State *s, Range *rp, int nr, Range *new) { uvlong base, next; Range *rs, *re; int type; rs = rp; re = rp + nr; for(base = 0; base <= s->mask; base = next) { if(rp >= re) return -1; type = gettype(s, base, new); next = getnext(s, base, new); while(next <= s->mask && (gettype(s, next, new) == type)) next = getnext(s, next, new); rp->base = base; rp->size = next - base; rp->type = type; rp++; } return rp - rs; } static int dosync; static QLock mtrrlk; static State cpu0state; static Range ranges[Nranges]; char* mtrr(uvlong base, uvlong size, char *tstr) { static State newstate; Range new; int nr; if(cpu0state.mask == 0) return "mtrr not supported"; if(size < 0x1000) return "size too small"; if((base | size) & 0xFFF) return "base or size not page aligned"; if(base & ~cpu0state.mask) return "base out of range"; if(base + size > cpu0state.mask+1) return "size out of range"; new.base = base; new.size = size; if((new.type = str2type(tstr)) < 0) return "bad cache type"; if(new.type == Writecomb && (cpu0state.cap & Capwc) == 0) return "write combining not supported"; qlock(&mtrrlk); newstate = cpu0state; nr = getranges(&newstate, ranges, Nranges, &new); if(setranges(&newstate, ranges, nr) < 0 || checkranges(&newstate, ranges, nr) < 0){ qunlock(&mtrrlk); return "cache range not satisfiable"; } cpu0state = newstate; coherence(); dosync = 1; mtrrclock(); qunlock(&mtrrlk); return nil; } char* mtrrattr(uvlong pa, uvlong *pnext) { if(cpu0state.mask == 0) return nil; if(pnext != nil) *pnext = getnext(&cpu0state, pa, nil); return type2str(gettype(&cpu0state, pa, nil)); } int mtrrprint(char *buf, long bufsize) { char *cp, *ep; int i, nr; if(cpu0state.mask == 0) return 0; cp = buf; ep = buf + bufsize; qlock(&mtrrlk); nr = getranges(&cpu0state, ranges, Nranges, nil); for(i = 0; i < nr; i++){ cp = seprint(cp, ep, "cache %#.16llux %15llud %s\n", ranges[i].base, ranges[i].size, type2str(ranges[i].type)); } qunlock(&mtrrlk); return cp - buf; } /* called from clock interrupt */ void mtrrclock(void) { static Ref bar1, bar2; int x; if(dosync == 0 || cpu0state.mask == 0) return; x = splhi(); /* * wait for all CPUs to sync here, so that the MTRR setup gets * done at roughly the same time on all processors. */ incref(&bar1); while(bar1.ref < conf.nmach) microdelay(10); putstate(&cpu0state); /* * wait for all CPUs to sync up again, so that we don't continue * executing while the MTRRs are still being set up. */ incref(&bar2); while(bar2.ref < conf.nmach) microdelay(10); decref(&bar1); while(bar1.ref > 0) microdelay(10); decref(&bar2); dosync = 0; splx(x); } /* called from cpuidentify() */ void mtrrsync(void) { State s; if(getstate(&s) < 0) return; if(cpu0state.mask == 0){ cpu0state = s; coherence(); return; } putstate(&cpu0state); }