ref: a3b95d3e567c0ef60f9f1ead26124fb706dec20d
dir: /sys/src/cmd/vac/vac.c/
#include "stdinc.h" #include "vac.h" #include "dat.h" #include "fns.h" // TODO: qids void usage(void) { fprint(2, "vac [-imqstv] [-a archive.vac] [-b bsize] [-d old.vac] [-e exclude] [-f new.vac] [-i name] [-h host] [-x excludefile] file...\n"); threadexitsall("usage"); } enum { BlockSize = 8*1024, }; struct { int nfile; int ndir; vlong data; vlong skipdata; int skipfiles; } stats; int qdiff; int merge; int verbose; int notmp; char *host; VtConn *z; VacFs *fs; char *archivefile; char *vacfile; int vacmerge(VacFile*, char*); void vac(VacFile*, VacFile*, char*, Dir*); void vacstdin(VacFile*, char*); VacFile *recentarchive(VacFs*, char*); static u64int unittoull(char*); static void warn(char *fmt, ...); static void removevacfile(void); void threadmain(int argc, char **argv) { int i, j, fd, n, printstats; Dir *d; char *s; uvlong u; VacFile *f, *fdiff; VacFs *fsdiff; int blocksize; int outfd; char *stdinname; char *diffvac; uvlong qid; fmtinstall('F', vtfcallfmt); fmtinstall('H', encodefmt); fmtinstall('V', vtscorefmt); blocksize = BlockSize; stdinname = nil; printstats = 0; fsdiff = nil; diffvac = nil; ARGBEGIN{ case 'V': chattyventi++; break; case 'a': archivefile = EARGF(usage()); break; case 'b': u = unittoull(EARGF(usage())); if(u < 512) u = 512; if(u > VtMaxLumpSize) u = VtMaxLumpSize; blocksize = u; break; case 'd': diffvac = EARGF(usage()); break; case 'e': excludepattern(EARGF(usage())); break; case 'f': vacfile = EARGF(usage()); break; case 'h': host = EARGF(usage()); break; case 'i': stdinname = EARGF(usage()); break; case 'm': merge++; break; case 'q': qdiff++; break; case 's': printstats++; break; case 't': notmp++; break; case 'v': verbose++; break; case 'x': loadexcludefile(EARGF(usage())); break; default: usage(); }ARGEND if(argc == 0 && !stdinname) usage(); if(archivefile && (vacfile || diffvac)){ fprint(2, "cannot use -a with -f, -d\n"); usage(); } z = vtdial(host); if(z == nil) sysfatal("could not connect to server: %r"); if(vtconnect(z) < 0) sysfatal("vtconnect: %r"); // Setup: // fs is the output vac file system // f is directory in output vac to write new files // fdiff is corresponding directory in existing vac if(archivefile){ VacFile *fp; char yyyy[5]; char mmdd[10]; char oldpath[40]; Tm tm; fdiff = nil; if((outfd = open(archivefile, ORDWR)) < 0){ if(access(archivefile, 0) >= 0) sysfatal("open %s: %r", archivefile); if((outfd = create(archivefile, OWRITE, 0666)) < 0) sysfatal("create %s: %r", archivefile); atexit(removevacfile); // because it is new if((fs = vacfscreate(z, blocksize, 512)) == nil) sysfatal("vacfscreate: %r"); }else{ if((fs = vacfsopen(z, archivefile, VtORDWR, 512)) == nil) sysfatal("vacfsopen %s: %r", archivefile); if((fdiff = recentarchive(fs, oldpath)) != nil){ if(verbose) fprint(2, "diff %s\n", oldpath); }else if(verbose) fprint(2, "no recent archive to diff against\n"); } // Create yyyy/mmdd. tm = *localtime(time(0)); snprint(yyyy, sizeof yyyy, "%04d", tm.year+1900); fp = vacfsgetroot(fs); if((f = vacfilewalk(fp, yyyy)) == nil && (f = vacfilecreate(fp, yyyy, ModeDir|0555)) == nil) sysfatal("vacfscreate %s: %r", yyyy); vacfiledecref(fp); fp = f; snprint(mmdd, sizeof mmdd, "%02d%02d", tm.mon+1, tm.mday); n = 0; while((f = vacfilewalk(fp, mmdd)) != nil){ vacfiledecref(f); n++; snprint(mmdd+4, sizeof mmdd-4, ".%d", n); } f = vacfilecreate(fp, mmdd, ModeDir|0555); if(f == nil) sysfatal("vacfscreate %s/%s: %r", yyyy, mmdd); vacfiledecref(fp); if(verbose) fprint(2, "archive %s/%s\n", yyyy, mmdd); }else{ if(vacfile == nil) outfd = 1; else if((outfd = create(vacfile, OWRITE, 0666)) < 0) sysfatal("create %s: %r", vacfile); atexit(removevacfile); if((fs = vacfscreate(z, blocksize, 512)) == nil) sysfatal("vacfscreate: %r"); f = vacfsgetroot(fs); fdiff = nil; if(diffvac){ if((fsdiff = vacfsopen(z, diffvac, VtOREAD, 128)) == nil) warn("vacfsopen %s: %r", diffvac); else fdiff = vacfsgetroot(fsdiff); } } if(stdinname) vacstdin(f, stdinname); for(i=0; i<argc; i++){ // We can't use / and . and .. and ../.. as valid archive // names, so expand to the list of files in the directory. if(argv[i][0] == 0){ warn("empty string given as command-line argument"); continue; } cleanname(argv[i]); if(strcmp(argv[i], "/") == 0 || strcmp(argv[i], ".") == 0 || strcmp(argv[i], "..") == 0 || (strlen(argv[i]) > 3 && strcmp(argv[i]+strlen(argv[i])-3, "/..") == 0)){ if((fd = open(argv[i], OREAD)) < 0){ warn("open %s: %r", argv[i]); continue; } while((n = dirread(fd, &d)) > 0){ for(j=0; j<n; j++){ s = vtmalloc(strlen(argv[i])+1+strlen(d[j].name)+1); strcpy(s, argv[i]); strcat(s, "/"); strcat(s, d[j].name); cleanname(s); vac(f, fdiff, s, &d[j]); } free(d); } close(fd); continue; } if((d = dirstat(argv[i])) == nil){ warn("stat %s: %r", argv[i]); continue; } vac(f, fdiff, argv[i], d); free(d); } if(fdiff) vacfiledecref(fdiff); /* * Record the maximum qid so that vacs can be merged * without introducing overlapping qids. Older versions * of vac arranged that the root would have the largest * qid in the file system, but we can't do that anymore * (the root gets created first!). */ if(_vacfsnextqid(fs, &qid) >= 0) vacfilesetqidspace(f, 0, qid); vacfiledecref(f); /* * Copy fsdiff's root block score into fs's slot for that, * so that vacfssync will copy it into root.prev for us. * Just nice documentation, no effect. */ if(fsdiff) memmove(fs->score, fsdiff->score, VtScoreSize); if(vacfssync(fs) < 0) fprint(2, "vacfssync: %r\n"); fprint(outfd, "vac:%V\n", fs->score); atexitdont(removevacfile); vacfsclose(fs); vthangup(z); if(printstats){ fprint(2, "%d files, %d files skipped, %d directories\n" "%lld data bytes written, %lld data bytes skipped\n", stats.nfile, stats.skipfiles, stats.ndir, stats.data, stats.skipdata); dup(2, 1); packetstats(); } threadexitsall(0); } VacFile* recentarchive(VacFs *fs, char *path) { VacFile *fp, *f; VacDirEnum *de; VacDir vd; char buf[10]; int year, mmdd, nn, n, n1; char *p; fp = vacfsgetroot(fs); de = vdeopen(fp); year = 0; if(de){ for(; vderead(de, &vd) > 0; vdcleanup(&vd)){ if(strlen(vd.elem) != 4) continue; if((n = strtol(vd.elem, &p, 10)) < 1900 || *p != 0) continue; if(year < n) year = n; } } vdeclose(de); if(year == 0){ vacfiledecref(fp); return nil; } snprint(buf, sizeof buf, "%04d", year); if((f = vacfilewalk(fp, buf)) == nil){ fprint(2, "warning: dirread %s but cannot walk", buf); vacfiledecref(fp); return nil; } fp = f; de = vdeopen(fp); mmdd = 0; nn = 0; if(de){ for(; vderead(de, &vd) > 0; vdcleanup(&vd)){ if(strlen(vd.elem) < 4) continue; if((n = strtol(vd.elem, &p, 10)) < 100 || n > 1231 || p != vd.elem+4) continue; if(*p == '.'){ if(p[1] == '0' || (n1 = strtol(p+1, &p, 10)) == 0 || *p != 0) continue; }else{ if(*p != 0) continue; n1 = 0; } if(n < mmdd || (n == mmdd && n1 < nn)) continue; mmdd = n; nn = n1; } } vdeclose(de); if(mmdd == 0){ vacfiledecref(fp); return nil; } if(nn == 0) snprint(buf, sizeof buf, "%04d", mmdd); else snprint(buf, sizeof buf, "%04d.%d", mmdd, nn); if((f = vacfilewalk(fp, buf)) == nil){ fprint(2, "warning: dirread %s but cannot walk", buf); vacfiledecref(fp); return nil; } vacfiledecref(fp); sprint(path, "%04d/%s", year, buf); return f; } static void removevacfile(void) { if(vacfile) remove(vacfile); } void plan9tovacdir(VacDir *vd, Dir *dir) { memset(vd, 0, sizeof *vd); vd->elem = dir->name; vd->uid = dir->uid; vd->gid = dir->gid; vd->mid = dir->muid; if(vd->mid == nil) vd->mid = ""; vd->mtime = dir->mtime; vd->mcount = 0; vd->ctime = dir->mtime; /* ctime: not available on plan 9 */ vd->atime = dir->atime; vd->size = dir->length; vd->mode = dir->mode & 0777; if(dir->mode & DMDIR) vd->mode |= ModeDir; if(dir->mode & DMAPPEND) vd->mode |= ModeAppend; if(dir->mode & DMEXCL) vd->mode |= ModeExclusive; vd->plan9 = 1; vd->p9path = dir->qid.path; vd->p9version = dir->qid.vers; } /* * Archive the file named name, which has stat info d, * into the vac directory fp (p = parent). * * If we're doing a vac -d against another archive, the * equivalent directory to fp in that archive is diffp. */ void vac(VacFile *fp, VacFile *diffp, char *name, Dir *d) { char *elem, *s; static char buf[65536]; int fd, i, n, bsize; vlong off; Dir *dk; // kids VacDir vd, vddiff; VacFile *f, *fdiff; VtEntry e; if(!includefile(name) || (notmp && (d->qid.type&QTTMP))) { warn("excluding %s%s", name, (d->mode&DMDIR) ? "/" : ""); return; } if(d->mode&DMDIR) stats.ndir++; else stats.nfile++; if(merge && vacmerge(fp, name) >= 0) return; if(verbose) fprint(2, "%s%s\n", name, (d->mode&DMDIR) ? "/" : ""); if((fd = open(name, OREAD)) < 0){ warn("open %s: %r", name); return; } elem = strrchr(name, '/'); if(elem) elem++; else elem = name; plan9tovacdir(&vd, d); if((f = vacfilecreate(fp, elem, vd.mode)) == nil){ warn("vacfilecreate %s: %r", name); return; } if(diffp) fdiff = vacfilewalk(diffp, elem); else fdiff = nil; if(vacfilesetdir(f, &vd) < 0) warn("vacfilesetdir %s: %r", name); if(d->mode&DMDIR){ while((n = dirread(fd, &dk)) > 0){ for(i=0; i<n; i++){ s = vtmalloc(strlen(name)+1+strlen(dk[i].name)+1); strcpy(s, name); strcat(s, "/"); strcat(s, dk[i].name); vac(f, fdiff, s, &dk[i]); free(s); } free(dk); } }else{ off = 0; bsize = fs->bsize; if(fdiff){ /* * Copy fdiff's contents into f by moving the score. * We'll diff and update below. */ if(vacfilegetentries(fdiff, &e, nil) >= 0) if(vacfilesetentries(f, &e, nil) >= 0){ bsize = e.dsize; /* * Or if -q is set, and the metadata looks the same, * don't even bother reading the file. */ if(qdiff && vacfilegetdir(fdiff, &vddiff) >= 0){ if(vddiff.mtime == vd.mtime) if(vddiff.size == vd.size) if(!vddiff.plan9 || (/* vddiff.p9path == vd.p9path && */ vddiff.p9version == vd.p9version)){ stats.skipfiles++; stats.nfile--; vdcleanup(&vddiff); goto Out; } /* * Skip over presumably-unchanged prefix * of an append-only file. */ if(vd.mode&ModeAppend) if(vddiff.size < vd.size) if(vddiff.plan9 && vd.plan9) if(vddiff.p9path == vd.p9path){ off = vd.size/bsize*bsize; if(seek(fd, off, 0) >= 0) stats.skipdata += off; else{ seek(fd, 0, 0); // paranoia off = 0; } } vdcleanup(&vddiff); // XXX different verbose chatty prints for kaminsky? } } } if(qdiff && verbose) fprint(2, "+%s\n", name); while((n = readn(fd, buf, bsize)) > 0){ if(fdiff && sha1matches(f, off/bsize, (uchar*)buf, n)){ off += n; stats.skipdata += n; continue; } if(vacfilewrite(f, buf, n, off) < 0){ warn("venti write %s: %r", name); goto Out; } stats.data += n; off += n; } /* * Since we started with fdiff's contents, * set the size in case fdiff was bigger. */ if(fdiff && vacfilesetsize(f, off) < 0) warn("vtfilesetsize %s: %r", name); } Out: vacfileflush(f, 1); vacfiledecref(f); if(fdiff) vacfiledecref(fdiff); close(fd); } void vacstdin(VacFile *fp, char *name) { vlong off; VacFile *f; static char buf[8192]; int n; if((f = vacfilecreate(fp, name, 0666)) == nil){ warn("vacfilecreate %s: %r", name); return; } off = 0; while((n = read(0, buf, sizeof buf)) > 0){ if(vacfilewrite(f, buf, n, off) < 0){ warn("venti write %s: %r", name); vacfiledecref(f); return; } off += n; } vacfileflush(f, 1); vacfiledecref(f); } /* * fp is the directory we're writing. * mp is the directory whose contents we're merging in. * d is the directory entry of the file from mp that we want to add to fp. * vacfile is the name of the .vac file, for error messages. * offset is the qid that qid==0 in mp should correspond to. * max is the maximum qid we expect to see (not really needed). */ int vacmergefile(VacFile *fp, VacFile *mp, VacDir *d, char *vacfile, vlong offset, vlong max) { VtEntry ed, em; VacFile *mf; VacFile *f; mf = vacfilewalk(mp, d->elem); if(mf == nil){ warn("could not walk %s in %s", d->elem, vacfile); return -1; } if(vacfilegetentries(mf, &ed, &em) < 0){ warn("could not get entries for %s in %s", d->elem, vacfile); vacfiledecref(mf); return -1; } if((f = vacfilecreate(fp, d->elem, d->mode)) == nil){ warn("vacfilecreate %s: %r", d->elem); vacfiledecref(mf); return -1; } if(d->qidspace){ d->qidoffset += offset; d->qidmax += offset; }else{ d->qidspace = 1; d->qidoffset = offset; d->qidmax = max; } if(vacfilesetdir(f, d) < 0 || vacfilesetentries(f, &ed, &em) < 0 || vacfilesetqidspace(f, d->qidoffset, d->qidmax) < 0){ warn("vacmergefile %s: %r", d->elem); vacfiledecref(mf); vacfiledecref(f); return -1; } vacfiledecref(mf); vacfiledecref(f); return 0; } int vacmerge(VacFile *fp, char *name) { VacFs *mfs; VacDir vd; VacDirEnum *de; VacFile *mp; uvlong maxqid, offset; if(strlen(name) < 4 || strcmp(name+strlen(name)-4, ".vac") != 0) return -1; if((mfs = vacfsopen(z, name, VtOREAD, 100)) == nil) return -1; if(verbose) fprint(2, "merging %s\n", name); mp = vacfsgetroot(mfs); de = vdeopen(mp); if(de){ offset = 0; if(vacfsgetmaxqid(mfs, &maxqid) >= 0){ _vacfsnextqid(fs, &offset); vacfsjumpqid(fs, maxqid+1); } while(vderead(de, &vd) > 0){ if(vd.qid > maxqid){ warn("vacmerge %s: maxqid=%lld but %s has %lld", name, maxqid, vd.elem, vd.qid); vacfsjumpqid(fs, vd.qid - maxqid); maxqid = vd.qid; } vacmergefile(fp, mp, &vd, name, offset, maxqid); vdcleanup(&vd); } vdeclose(de); } vacfiledecref(mp); vacfsclose(mfs); return 0; } #define TWID64 ((u64int)~(u64int)0) static u64int unittoull(char *s) { char *es; u64int n; if(s == nil) return TWID64; n = strtoul(s, &es, 0); if(*es == 'k' || *es == 'K'){ n *= 1024; es++; }else if(*es == 'm' || *es == 'M'){ n *= 1024*1024; es++; }else if(*es == 'g' || *es == 'G'){ n *= 1024*1024*1024; es++; } if(*es != '\0') return TWID64; return n; } static void warn(char *fmt, ...) { va_list arg; va_start(arg, fmt); fprint(2, "vac: "); vfprint(2, fmt, arg); fprint(2, "\n"); va_end(arg); }