shithub: riscv

Download patch

ref: 29f60cace1a71edd730c60ddac8dfbd962741038
parent: 2fb5fbbd73a1c861bdc7326ece8035ffb35f7093
author: cinap_lenrek <cinap_lenrek@felloff.net>
date: Tue Dec 22 11:29:55 EST 2020 

kernel: avoid palloc lock during mmurelease()

Previously, mmurelease() was always called with
palloc spinlock held.

This is unneccesary for some mmurelease()
implementations as they wont release pages
to the palloc pool.

This change removes pagechainhead() and
pagechaindone() and replaces them with just
freepages() call, which aquires the palloc
lock internally as needed.

freepages() avoids holding the palloc lock
while walking the linked list of pages,
avoding some lock contention.


--- a/sys/src/9/bcm/mmu.c
+++ b/sys/src/9/bcm/mmu.c
@@ -173,17 +173,9 @@
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %lud", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
+
+	freepages(proc->mmul2cache, nil, 0);
 	proc->mmul2cache = nil;
 
 	mmul1empty();
--- a/sys/src/9/bcm64/mmu.c
+++ b/sys/src/9/bcm64/mmu.c
@@ -518,20 +518,10 @@
 void
 mmurelease(Proc *p)
 {
-	Page *t;
-
 	mmuswitch(nil);
 	mmufree(p);
-
-	if((t = p->mmufree) != nil){
-		do {
-			p->mmufree = t->next;
-			if(--t->ref != 0)
-				panic("mmurelease: bad page ref");
-			pagechainhead(t);
-		} while((t = p->mmufree) != nil);
-		pagechaindone();
-	}
+	freepages(p->mmufree, nil, 0);
+	p->mmufree = nil;
 }
 
 void
--- a/sys/src/9/cycv/mmu.c
+++ b/sys/src/9/cycv/mmu.c
@@ -197,23 +197,21 @@
 void
 mmurelease(Proc *proc)
 {
-	Page *p, *n;
+	Page *p;
 
-	if(islo())
-		panic("mmurelease: islo");
-	
 	l1switch(&m->l1, 0);
-	if(proc->kmaptable != nil){
+	if((p = proc->kmaptable) != nil){
+		if(p->ref != 1)
+			panic("mmurelease: kmap ref %ld", p->ref);
 		if(proc->l1 == nil)
 			panic("mmurelease: no l1");
-		if(decref(proc->kmaptable) != 0)
-			panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
 		if(proc->nkmap)
 			panic("mmurelease: nkmap %d", proc->nkmap);
-		if(PPN(proc->l1->va[L1X(KMAP)]) != proc->kmaptable->pa)
-			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], proc->kmaptable->pa);
+		if(PPN(proc->l1->va[L1X(KMAP)]) != p->pa)
+			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], p->pa);
 		proc->l1->va[L1X(KMAP)] = 0;
-		pagechainhead(proc->kmaptable);
+		p->next = proc->mmufree;
+		proc->mmufree = p;
 		proc->kmaptable = nil;
 	}
 	if(proc->l1 != nil){
@@ -221,14 +219,7 @@
 		l1free(proc->l1);
 		proc->l1 = nil;
 	}
-	for(p = proc->mmufree; p != nil; p = n){
-		n = p->next;
-		if(decref(p) != 0)
-			panic("mmurelease: p->ref %ld", p->ref);
-		pagechainhead(p);
-	}
-	if(proc->mmufree != nil)
-		pagechaindone();
+	freepages(proc->mmufree, nil, 0);
 	proc->mmufree = nil;
 }
 
--- a/sys/src/9/kw/mmu.c
+++ b/sys/src/9/kw/mmu.c
@@ -252,20 +252,12 @@
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	/* write back dirty and invalidate l1 caches */
 	cacheuwbinv();
 
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %lud", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
+
+	freepages(proc->mmul2cache, nil, 0);
 	proc->mmul2cache = nil;
 
 	mmul1empty();
--- a/sys/src/9/omap/mmu.c
+++ b/sys/src/9/omap/mmu.c
@@ -234,20 +234,12 @@
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	/* write back dirty and invalidate l1 caches */
 	cacheuwbinv();
 
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %ld", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
+
+	freepages(proc->mmul2cache, nil, 0);
 	proc->mmul2cache = nil;
 
 	mmul1empty();
--- a/sys/src/9/pc/mmu.c
+++ b/sys/src/9/pc/mmu.c
@@ -320,23 +320,20 @@
  *   cleaning any user entries in the pdb (proc->mmupdb);
  *   if there's a pdb put it in the cache of pre-initialised pdb's
  *   for this processor (m->pdbpool) or on the process' free list;
- *   finally, place any pages freed back into the free pool (palloc).
- * This routine is only called from schedinit() with palloc locked.
+ *   finally, place any pages freed back into the free pool (freepages).
  */
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
 	ulong *pdb;
+	Page *page;
 
-	if(islo())
-		panic("mmurelease: islo");
 	taskswitch(PADDR(m->pdb), (ulong)m + BY2PG);
-	if(proc->kmaptable != nil){
+	if((page = proc->kmaptable) != nil){
+		if(page->ref != 1)
+			panic("mmurelease: kmap ref %ld", page->ref);
 		if(proc->mmupdb == nil)
 			panic("mmurelease: no mmupdb");
-		if(--proc->kmaptable->ref != 0)
-			panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
 		if(proc->nkmap)
 			panic("mmurelease: nkmap %d", proc->nkmap);
 		/*
@@ -343,31 +340,28 @@
 		 * remove kmaptable from pdb before putting pdb up for reuse.
 		 */
 		pdb = tmpmap(proc->mmupdb);
-		if(PPN(pdb[PDX(KMAP)]) != proc->kmaptable->pa)
+		if(PPN(pdb[PDX(KMAP)]) != page->pa)
 			panic("mmurelease: bad kmap pde %#.8lux kmap %#.8lux",
-				pdb[PDX(KMAP)], proc->kmaptable->pa);
+				pdb[PDX(KMAP)], page->pa);
 		pdb[PDX(KMAP)] = 0;
 		tmpunmap(pdb);
+
 		/*
 		 * move kmaptable to free list.
 		 */
-		pagechainhead(proc->kmaptable);
+		page->next = proc->mmufree;
+		proc->mmufree = page;
 		proc->kmaptable = nil;
 	}
-	if(proc->mmupdb != nil){
+	if((page = proc->mmupdb) != nil){
 		mmuptefree(proc);
-		mmupdbfree(proc, proc->mmupdb);
+		mmupdbfree(proc, page);
 		proc->mmupdb = nil;
 	}
-	for(page = proc->mmufree; page != nil; page = next){
-		next = page->next;
-		if(--page->ref != 0)
-			panic("mmurelease: page->ref %ld", page->ref);
-		pagechainhead(page);
+	if((page = proc->mmufree) != nil){
+		freepages(page, nil, 0);
+		proc->mmufree = nil;
 	}
-	if(proc->mmufree != nil)
-		pagechaindone();
-	proc->mmufree = nil;
 	if(proc->ldt != nil){
 		free(proc->ldt);
 		proc->ldt = nil;
--- a/sys/src/9/port/devsegment.c
+++ b/sys/src/9/port/devsegment.c
@@ -469,7 +469,7 @@
 {
 	KMap *k;
 	Segment *s;
-	Page **f, *p, *l, *h;
+	Page **f, *p, *l, *h, *t;
 	ulong n, i;
 	int color;
 
@@ -492,12 +492,13 @@
 			continue;
 
 		i = 0;
-		h = nil;
+		h = t = nil;
 		f = &palloc.head;
 		while((p = *f) != nil){
 			if(p > &l[-len] && p <= l){
 				*f = p->next;
-				p->next = h;
+				if((p->next = h) == nil)
+					t = p;
 				h = p;
 				if(++i < len)
 					continue;
@@ -505,15 +506,15 @@
 			}
 			f = &p->next;
 		}
-		palloc.freecount -= i;
 
 		if(i != len){
-			while((p = h) != nil){
-				h = h->next;
-				pagechainhead(p);
+			if(h != nil){
+				t->next = palloc.head;
+				palloc.head = h;
 			}
 			goto Retry;
 		}
+		palloc.freecount -= i;
 		unlock(&palloc);
 
 		p = &l[-len];
--- a/sys/src/9/port/page.c
+++ b/sys/src/9/port/page.c
@@ -11,7 +11,7 @@
 pageinit(void)
 {
 	int color, i, j;
-	Page *p;
+	Page *p, **t;
 	Pallocmem *pm;
 	vlong m, v, u;
 
@@ -29,8 +29,12 @@
 	}
 
 	color = 0;
+	palloc.freecount = 0;
 	palloc.head = nil;
+
+	t = &palloc.head;
 	p = palloc.pages;
+
 	for(i=0; i<nelem(palloc.mem); i++){
 		pm = &palloc.mem[i];
 		for(j=0; j<pm->npage; j++){
@@ -40,7 +44,8 @@
 				continue;
 			p->color = color;
 			color = (color+1)%NCOLOR;
-			pagechainhead(p);
+			*t = p, t = &p->next;
+			palloc.freecount++;
 			p++;
 		}
 	}
@@ -65,15 +70,7 @@
 	print("%lldM swap\n", v/(1024*1024));
 }
 
-void
-pagechainhead(Page *p)
-{
-	p->next = palloc.head;
-	palloc.head = p;
-	palloc.freecount++;
-}
-
-void
+static void
 pagechaindone(void)
 {
 	if(palloc.pwait[0].p != nil && wakeup(&palloc.pwait[0]) != nil)
@@ -85,11 +82,23 @@
 void
 freepages(Page *head, Page *tail, ulong np)
 {
-	assert(palloc.Lock.p == up);
+	if(head == nil)
+		return;
+	if(tail == nil){
+		tail = head;
+		for(np = 1;; np++){
+			tail->ref = 0;
+			if(tail->next == nil)
+				break;
+			tail = tail->next;
+		}
+	}
+	lock(&palloc);
 	tail->next = palloc.head;
 	palloc.head = head;
 	palloc.freecount += np;
 	pagechaindone();
+	unlock(&palloc);
 }
 
 ulong
@@ -138,11 +147,8 @@
 	}
 	putimage(i);
 
-	if(np > 0){
-		lock(&palloc);
+	if(np > 0)
 		freepages(fh, ft, np);
-		unlock(&palloc);
-	}
 
 	return np;
 }
@@ -237,11 +243,8 @@
 		decref(p);
 		return;
 	}
-	if(decref(p) == 0){
-		lock(&palloc);
+	if(decref(p) == 0)
 		freepages(p, p, 1);
-		unlock(&palloc);
-	}
 }
 
 void
--- a/sys/src/9/port/portfns.h
+++ b/sys/src/9/port/portfns.h
@@ -218,8 +218,6 @@
 int		openmode(ulong);
 Block*		packblock(Block*);
 Block*		padblock(Block*, int);
-void		pagechaindone(void);
-void		pagechainhead(Page*);
 void		pageinit(void);
 ulong		pagereclaim(Image*);
 void		panic(char*, ...);
--- a/sys/src/9/port/proc.c
+++ b/sys/src/9/port/proc.c
@@ -81,27 +81,21 @@
 		case Moribund:
 			up->state = Dead;
 			edfstop(up);
-			if(up->edf != nil)
+			if(up->edf != nil){
 				free(up->edf);
-			up->edf = nil;
+				up->edf = nil;
+			}
 
-			/*
-			 * Holding locks from pexit:
-			 * 	procalloc
-			 *	palloc
-			 */
 			mmurelease(up);
-			unlock(&palloc);
 
-			updatecpu(up);
+			lock(&procalloc);
 			up->mach = nil;
-
 			up->qnext = procalloc.free;
 			procalloc.free = up;
-
 			/* proc is free now, make sure unlock() wont touch it */
 			up = procalloc.Lock.p = nil;
 			unlock(&procalloc);
+
 			sched();
 		}
 		coherence();
@@ -1222,10 +1216,6 @@
 		}
 	}
 	qunlock(&up->seglock);
-
-	/* Sched must not loop for these locks */
-	lock(&procalloc);
-	lock(&palloc);
 
 	edfstop(up);
 	up->state = Moribund;
--- a/sys/src/9/teg2/mmu.c
+++ b/sys/src/9/teg2/mmu.c
@@ -475,20 +475,12 @@
 void
 mmurelease(Proc* proc)
 {
-	Page *page, *next;
-
 	/* write back dirty and invalidate caches */
 	l1cache->wbinv();
 
 	mmul2empty(proc, 0);
-	for(page = proc->mmul2cache; page != nil; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %ld", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmul2cache != nil)
-		pagechaindone();
+
+	freepages(proc->mmul2cache, nil, 0);
 	proc->mmul2cache = nil;
 
 	mmul1empty();
--- a/sys/src/9/xen/mmu.c
+++ b/sys/src/9/xen/mmu.c
@@ -282,15 +282,8 @@
 		}
 	}
 
-	for(page = proc->mmufree; page; page = next){
-		next = page->next;
-		if(--page->ref)
-			panic("mmurelease: page->ref %ld\n", page->ref);
-		pagechainhead(page);
-	}
-	if(proc->mmufree)
-		pagechaindone();
-	proc->mmufree = 0;
+	freepages(proc->mmufree, nil, 0);
+	proc->mmufree = nil;
 }
 
 static Page*
--- a/sys/src/9/zynq/mmu.c
+++ b/sys/src/9/zynq/mmu.c
@@ -205,23 +205,22 @@
 void
 mmurelease(Proc *proc)
 {
-	Page *p, *n;
+	Page *p;
 
-	if(islo())
-		panic("mmurelease: islo");
-	
 	l1switch(&m->l1, 0);
-	if(proc->kmaptable != nil){
+	if((p = proc->kmaptable) != nil){
+		if(p->ref != 1)
+			panic("mmurelease: kmap ref %ld", p->ref);
 		if(proc->l1 == nil)
 			panic("mmurelease: no l1");
-		if(decref(proc->kmaptable) != 0)
-			panic("mmurelease: kmap ref %ld", proc->kmaptable->ref);
 		if(proc->nkmap)
 			panic("mmurelease: nkmap %d", proc->nkmap);
-		if(PPN(proc->l1->va[L1X(KMAP)]) != proc->kmaptable->pa)
-			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], proc->kmaptable->pa);
+		if(PPN(proc->l1->va[L1X(KMAP)]) != p->pa)
+			panic("mmurelease: bad kmap l2 %#.8lux kmap %#.8lux", proc->l1->va[L1X(KMAP)], p->pa);
 		proc->l1->va[L1X(KMAP)] = 0;
-		pagechainhead(proc->kmaptable);
+
+		p->next = proc->mmufree;
+		proc->mmufree = p;
 		proc->kmaptable = nil;
 	}
 	if(proc->l1 != nil){
@@ -229,14 +228,7 @@
 		l1free(proc->l1);
 		proc->l1 = nil;
 	}
-	for(p = proc->mmufree; p != nil; p = n){
-		n = p->next;
-		if(decref(p) != 0)
-			panic("mmurelease: p->ref %ld", p->ref);
-		pagechainhead(p);
-	}
-	if(proc->mmufree != nil)
-		pagechaindone();
+	freepages(proc->mmufree, nil, 0);
 	proc->mmufree = nil;
 }