shithub: clone

Download patch

ref: 94d6eed5cfd4c6950d66dca15f6424fcafcc7abc
parent: 4745a86fa929cf656db1125b482bda2ade12a60c
author: kvik <kvik@a-b.xyz>
date: Tue Jan 28 08:14:02 EST 2020

Fix directory read serialization bottleneck

The directory walker used dirreadall call for loading directory
entries into memory before assigning them to workers -- causing
severe and unwanted serialization that was especially noticeable
on long directories.

The solution is simple: chunk up the directory read and assign
work in between the chunks.

--- a/clone.c	Sun Jan 26 17:03:33 2020
+++ b/clone.c	Tue Jan 28 08:14:02 2020
@@ -263,32 +263,29 @@
 		error("can't open: %r");
 		return;
 	}
-	n = dirreadall(fd, &dirs);
-	if(n < 0){
-		error("can't read directory: %r");
-		close(fd);
-		return;
-	}
-	close(fd);
-
-	for(d = dirs; n; n--, d++){
-		if(d->mode & DMDIR && same(skipdir, d))
-			continue;
-
-		sn = smprint("%s/%s", src, d->name);
-		dn = smprint("%s/%s", dst, d->name);
-		if(d->mode & DMDIR){
-			if(mkdir(sn, dn, d, nil) < 0)
+	while((n = dirread(fd, &dirs)) > 0){
+		for(d = dirs; n; n--, d++){
+			if(d->mode & DMDIR && same(skipdir, d))
 				continue;
-			clonedir(sn, dn);
-		}else{
-			f = filenew(sn, dn, d);
-			sendp(filechan, f);
+
+			sn = smprint("%s/%s", src, d->name);
+			dn = smprint("%s/%s", dst, d->name);
+			if(d->mode & DMDIR){
+				if(mkdir(sn, dn, d, nil) < 0)
+					continue;
+				clonedir(sn, dn);
+			}else{
+				f = filenew(sn, dn, d);
+				sendp(filechan, f);
+			}
+			free(sn);
+			free(dn);
 		}
-		free(sn);
-		free(dn);
+		free(dirs);
 	}
-	free(dirs);
+	if(n < 0)
+		error("can't read directory: %r");
+	close(fd);
 }
 
 void