shithub: riscv

Info  •  Files  •  Log  •  Branches

imx8: port new FPU code, making FPU available to interrupts

--- a/sys/src/9/imx8/dat.h

+++ b/sys/src/9/imx8/dat.h

@@ -74,11 +74,14 @@

 	ulong	status;

 };

+#define KFPSTATE

+

 struct PFPU

 {

-	FPsave	fpsave[1];

-

 	int	fpstate;

+	int	kfpstate;

+	FPsave	*fpsave;

+	FPsave	*kfpsave;

 };

 enum

@@ -86,6 +89,7 @@

 	FPinit,

 	FPactive,

 	FPinactive,

+	FPprotected,

 	/* bits or'd with the state */

 	FPillegal= 0x100,

@@ -154,6 +158,9 @@

 	MMMU;

 	PMach;

+

+	int	fpstate;

+	FPsave	*fpsave;

 	int	cputype;

 	ulong	delayloop;

--- a/sys/src/9/imx8/fns.h

+++ b/sys/src/9/imx8/fns.h

@@ -90,11 +90,12 @@

 /* fpu */

 extern void fpuinit(void);

-extern void fpoff(void);

-extern void fpinit(void);

-extern void fpclear(void);

-extern void fpsave(FPsave*);

-extern void fprestore(FPsave*);

+extern void fpuprocsetup(Proc*);

+extern void fpuprocfork(Proc*);

+extern void fpuprocsave(Proc*);

+extern void fpuprocrestore(Proc*);

+extern FPsave* fpukenter(Ureg*);

+extern void fpukexit(Ureg*, FPsave*);

 extern void mathtrap(Ureg*);

 /* trap */

--- a/sys/src/9/imx8/fpu.c

+++ b/sys/src/9/imx8/fpu.c

@@ -13,65 +13,274 @@

 extern ulong getfsr(void);

 extern void setfsr(ulong fsr);

-void

-fpuinit(void)

+static FPsave fpsave0;

+

+static void

+fpsave(FPsave *p)

 {

+	p->control = getfcr();

+	p->status = getfsr();

+	fpsaveregs(p->regs);

 	fpoff();

 }

-void

-fpinit(void)

+static void

+fprestore(FPsave *p)

 {

 	fpon();

-	setfcr(0);

-	setfsr(0);

+	setfcr(p->control);

+	setfsr(p->status);

+	fploadregs(p->regs);

 }

+static void

+fpinit(void)

+{

+	fprestore(&fpsave0);

+}

+

 void

-fpclear(void)

+fpuinit(void)

 {

+	m->fpstate = FPinit;

+	m->fpsave = nil;

 	fpoff();

 }

+static FPsave*

+fpalloc(void)

+{

+	FPsave *save;

+

+	while((save = mallocalign(sizeof(FPsave), 16, 0, 0)) == nil){

+		spllo();

+		resrcwait("no memory for FPsave");

+		splhi();

+	}

+	return save;

+}

+

+static void

+fpfree(FPsave *save)

+{

+	free(save);

+}

+

+

+/*

+ *  Protect or save FPU state and setup new state

+ *  (lazily in the case of user process) for the kernel.

+ *  All syscalls, traps and interrupts (except mathtrap()!)

+ *  are handled between fpukenter() and fpukexit(),

+ *  so they can use floating point and vector instructions.

+ */

+FPsave*

+fpukenter(Ureg*)

+{

+	if(up == nil){

+		switch(m->fpstate){

+		case FPactive:

+			fpsave(m->fpsave);

+			/* wet floor */

+		case FPinactive:

+			m->fpstate = FPinit;

+			return m->fpsave;

+		}

+		return nil;

+	}

+

+	switch(up->fpstate){

+	case FPactive:

+		up->fpstate = FPprotected;

+		fpoff();

+		/* wet floor */

+	case FPprotected:

+		return nil;

+	}

+

+	switch(up->kfpstate){

+	case FPactive:

+		fpsave(up->kfpsave);

+		/* wet floor */

+	case FPinactive:

+		up->kfpstate = FPinit;

+		return up->kfpsave;

+	}

+	return nil;

+}

+

 void

-fpsave(FPsave *p)

+fpukexit(Ureg *ureg, FPsave *save)

 {

-	p->control = getfcr();

-	p->status = getfsr();

-	fpsaveregs(p->regs);

-	fpoff();

+	if(up == nil){

+		switch(m->fpstate){

+		case FPactive:

+			fpoff();

+			/* wet floor */

+		case FPinactive:

+			fpfree(m->fpsave);

+			m->fpstate = FPinit;

+		}

+		m->fpsave = save;

+		if(save != nil)

+			m->fpstate = FPinactive;

+		return;

+	}

+

+	if(up->fpstate == FPprotected){

+		if(userureg(ureg)){

+			up->fpstate = FPactive;

+			fpon();

+		}

+		return;

+	}

+

+	switch(up->kfpstate){

+	case FPactive:

+		fpoff();

+		/* wet floor */

+	case FPinactive:

+		fpfree(up->kfpsave);

+		up->kfpstate = FPinit;

+	}

+	up->kfpsave = save;

+	if(save != nil)

+		up->kfpstate = FPinactive;

 }

 void

-fprestore(FPsave *p)

+fpuprocsetup(Proc *p)

 {

-	fpon();

-	setfcr(p->control);

-	setfsr(p->status);

-	fploadregs(p->regs);

+	p->fpstate = FPinit;

 }

 void

-mathtrap(Ureg*)

+fpuprocfork(Proc *p)

 {

 	int s;

-	if((up->fpstate & FPillegal) != 0){

+	s = splhi();

+	switch(up->fpstate & ~FPillegal){

+	case FPprotected:

+		fpon();

+		/* wet floor */

+	case FPactive:

+		fpsave(up->fpsave);

+		up->fpstate = FPinactive;

+		/* wet floor */

+	case FPinactive:

+		if(p->fpsave == nil)

+			p->fpsave = fpalloc();

+		memmove(p->fpsave, up->fpsave, sizeof(FPsave));

+		p->fpstate = FPinactive;

+	}

+	splx(s);

+}

+

+void

+fpuprocsave(Proc *p)

+{

+	if(p->state == Moribund){

+		if(p->fpstate == FPactive || p->kfpstate == FPactive)

+			fpoff();

+		fpfree(p->fpsave);

+		fpfree(p->kfpsave);

+		p->fpsave = p->kfpsave = nil;

+		p->fpstate = p->kfpstate = FPinit;

+		return;

+	}

+	if(p->kfpstate == FPactive){

+		fpsave(p->kfpsave);

+		p->kfpstate = FPinactive;

+		return;

+	}

+	if(p->fpstate == FPprotected)

+		fpon();

+	else if(p->fpstate != FPactive)

+		return;

+	fpsave(p->fpsave);

+	p->fpstate = FPinactive;

+}

+

+void

+fpuprocrestore(Proc*)

+{

+	/*

+	 * when the scheduler switches,

+	 * we can discard its fp state.

+	 */

+	switch(m->fpstate){

+	case FPactive:

+		fpoff();

+		/* wet floor */

+	case FPinactive:

+		fpfree(m->fpsave);

+		m->fpsave = nil;

+		m->fpstate = FPinit;

+	}

+}

+

+void

+mathtrap(Ureg *ureg)

+{

+	if(!userureg(ureg)){

+		if(up == nil){

+			switch(m->fpstate){

+			case FPinit:

+				m->fpsave = fpalloc();

+				m->fpstate = FPactive;

+				fpinit();

+				break;

+			case FPinactive:

+				fprestore(m->fpsave);

+				m->fpstate = FPactive;

+				break;

+			default:

+				panic("floating point error in irq");

+			}

+			return;

+		}

+

+		if(up->fpstate == FPprotected){

+			fpon();

+			fpsave(up->fpsave);

+			up->fpstate = FPinactive;

+		}

+

+		switch(up->kfpstate){

+		case FPinit:

+			up->kfpsave = fpalloc();

+			up->kfpstate = FPactive;

+			fpinit();

+			break;

+		case FPinactive:

+			fprestore(up->kfpsave);

+			up->kfpstate = FPactive;

+			break;

+		default:

+			panic("floating point error in trap");

+		}

+		return;

+	}

+

+	if(up->fpstate & FPillegal){

 		postnote(up, 1, "sys: floating point in note handler", NDebug);

 		return;

 	}

 	switch(up->fpstate){

 	case FPinit:

-		s = splhi();

-		fpinit();

+		if(up->fpsave == nil)

+			up->fpsave = fpalloc();

 		up->fpstate = FPactive;

-		splx(s);

+		fpinit();

 		break;

 	case FPinactive:

-		s = splhi();

 		fprestore(up->fpsave);

 		up->fpstate = FPactive;

-		splx(s);

+		break;

+	case FPprotected:

+		up->fpstate = FPactive;

+		fpon();

 		break;

 	case FPactive:

 		postnote(up, 1, "sys: floating point error", NDebug);

--- a/sys/src/9/imx8/main.c

+++ b/sys/src/9/imx8/main.c

@@ -161,6 +161,9 @@

 	sp[3] = sp[2] = sp[1] = nil;

 	strcpy(sp[1] = (char*)&sp[4], "boot");

 	sp[0] = (void*)&sp[1];

+

+	splhi();

+	fpukexit(nil, nil);

 	touser((uintptr)sp);

 }

--- a/sys/src/9/imx8/trap.c

+++ b/sys/src/9/imx8/trap.c

@@ -97,6 +97,7 @@

 void

 trap(Ureg *ureg)

 {

+	FPsave *f = nil;

 	u32int type, intr;

 	int user;

@@ -113,6 +114,7 @@

 	case 0x21:	// instruction abort from same level

 	case 0x24:	// data abort from lower level

 	case 0x25:	// data abort from same level

+		f = fpukenter(ureg);

 		faultarm64(ureg);

 		break;

 	case 0x07:	// SIMD/FP

@@ -121,6 +123,7 @@

 		break;

 	case 0x00:	// unknown

 		if(intr == 1){

+			f = fpukenter(ureg);

 			if(irq(ureg) && up != nil && up->delaysched)

 				sched();

 			break;

@@ -127,6 +130,7 @@

 		}

 		if(intr == 3){

 	case 0x2F:	// SError interrupt

+			f = fpukenter(ureg);

 			if(buserror != nil && (*buserror)(ureg))

 				break;

 			dumpregs(ureg);

@@ -162,6 +166,7 @@

 	case 0x3A:	// vector catch exception (A32 only)

 	case 0x3C:	// BRK instruction (A64 only)

 	default:

+		f = fpukenter(ureg);

 		if(!userureg(ureg)){

 			dumpregs(ureg);

 			panic("unhandled trap");

@@ -170,6 +175,7 @@

 		postnote(up, 1, traps[type], NDebug);

 		break;

 	}

+

 	splhi();

 	if(user){

 		if(up->procctl || up->nnote)

@@ -176,6 +182,8 @@

 			notify(ureg);

 		kexit(ureg);

 	}

+	if(type != 0x07 && type != 0x2C)

+		fpukexit(ureg, f);

 }

 void

@@ -189,6 +197,7 @@

 	if(!kenter(ureg))

 		panic("syscall from  kernel");

+	fpukenter(ureg);

 	m->syscall++;

 	up->insyscall = 1;

@@ -196,7 +205,6 @@

 	sp = ureg->sp;

 	up->scallnr = scallnr = ureg->r0;

-

 	spllo();

 	up->nerrlab = 0;

@@ -246,9 +254,9 @@

 		procctl();

 		splx(s);

 	}

-	

 	up->insyscall = 0;

 	up->psstate = 0;

+

 	if(scallnr == NOTED){

 		noted(ureg, *((ulong*) up->s.args));

 		/*

@@ -260,21 +268,27 @@

 		 * to it when returning form syscall()

 		 */

 		returnto(noteret);

-	}

-	if(scallnr != RFORK && (up->procctl || up->nnote)){

 		splhi();

-		notify(ureg);

+		up->fpstate &= ~FPillegal;

 	}

+	else

+		splhi();

+

+	if(scallnr != RFORK && (up->procctl || up->nnote))

+		notify(ureg);

+

 	if(up->delaysched)

 		sched();

+

 	kexit(ureg);

+	fpukexit(ureg, nil);

 }

 int

 notify(Ureg *ureg)

 {

-	uintptr s, sp;

+	uintptr sp;

 	char *msg;

 	if(up->procctl)

@@ -281,13 +295,8 @@

 		procctl();

 	if(up->nnote == 0)

 		return 0;

-	if(up->fpstate == FPactive){

-		fpsave(up->fpsave);

-		up->fpstate = FPinactive;

-	}

-	up->fpstate |= FPillegal;

-	s = spllo();

+	spllo();

 	qlock(&up->debug);

 	msg = popnote(ureg);

 	if(msg == nil){

@@ -324,7 +333,10 @@

 	ureg->pc = (uintptr) up->notify;

 	ureg->link = 0;

 	qunlock(&up->debug);

-	splx(s);

+

+	splhi();

+	fpuprocsave(up);

+	up->fpstate |= FPillegal;

 	return 1;

 }

@@ -333,7 +345,7 @@

 {

 	Ureg *nureg;

 	uintptr oureg, sp;

-	

+

 	qlock(&up->debug);

 	if(arg0 != NRSTR && !up->notified){

 		qunlock(&up->debug);

@@ -343,7 +355,6 @@

 	up->notified = 0;

 	nureg = up->ureg;

-	up->fpstate &= ~FPillegal;

 	oureg = (uintptr) nureg;

 	if(!okaddr(oureg - BY2WD, BY2WD + sizeof(Ureg), 0) || (oureg & 7) != 0){

@@ -376,7 +387,6 @@

 		}

 		qunlock(&up->debug);

 		sp = oureg - 4 * BY2WD - ERRMAX;

-		splhi();

 		ureg->sp = sp;

 		ureg->r0 = (uintptr) oureg;

 		((uintptr *) sp)[1] = oureg;

@@ -399,26 +409,24 @@

 {

 	extern void checkpages(void);

 	char buf[ERRMAX];

-	int read, insyscall;

+	int user, read;

 	uintptr addr;

-	insyscall = up->insyscall;

-	up->insyscall = 1;

-

-	if(!userureg(ureg)){

+	user = userureg(ureg);

+	if(user)

+		up->insyscall = 1;

+	else {

 		extern void _peekinst(void);

 		if(ureg->pc == (uintptr)_peekinst){

 			ureg->pc = ureg->link;

-			goto out;

+			return;

 		}

-

 		if(waserror()){

 			if(up->nerrlab == 0){

 				pprint("suicide: sys: %s\n", up->errstr);

 				pexit(up->errstr, 1);

 			}

-			up->insyscall = insyscall;

 			nexterror();

 		}

 	}

@@ -446,7 +454,7 @@

 	case 61:				// first level domain fault

 	case 62:				// second level domain fault

 	default:

-		if(!userureg(ureg)){

+		if(!user){

 			dumpregs(ureg);

 			panic("fault: %s addr=%#p", read ? "read" : "write", addr);

 		}

@@ -455,11 +463,10 @@

 		postnote(up, 1, buf, NDebug);

 	}

-	if(!userureg(ureg))

+	if(user)

+		up->insyscall = 0;

+	else

 		poperror();

-

-out:

-	up->insyscall = insyscall;

 }

 int

@@ -487,19 +494,7 @@

 void

 procfork(Proc *p)

 {

-	int s;

-

-	s = splhi();

-	switch(up->fpstate & ~FPillegal){

-	case FPactive:

-		fpsave(up->fpsave);

-		up->fpstate = FPinactive;

-	case FPinactive:

-		memmove(p->fpsave, up->fpsave, sizeof(FPsave));

-		p->fpstate = FPinactive;

-	}

-	splx(s);

-

+	fpuprocfork(p);

 	p->tpidr = up->tpidr;

 }

@@ -506,9 +501,7 @@

 void

 procsetup(Proc *p)

 {

-	p->fpstate = FPinit;

-	fpoff();

-

+	fpuprocsetup(p);

 	p->tpidr = 0;

 	syswr(TPIDR_EL0, p->tpidr);

 }

@@ -516,17 +509,9 @@

 void

 procsave(Proc *p)

 {

-	if(p->fpstate == FPactive){

-		if(p->state == Moribund)

-			fpclear();

-		else

-			fpsave(p->fpsave);

-		p->fpstate = FPinactive;

-	}

-

+	fpuprocsave(p);

 	if(p->kp == 0)

 		p->tpidr = sysrd(TPIDR_EL0);

-

 	putasid(p);	// release asid

 }

@@ -533,6 +518,7 @@

 void

 procrestore(Proc *p)

 {

+	fpuprocrestore(p);

 	if(p->kp == 0)

 		syswr(TPIDR_EL0, p->tpidr);

 }