shithub: hugo

Download patch

ref: ff6253bc7cf745e9c0127ddc9006da3c2c00c738
parent: aa4ccb8a1e9b8aa17397acf34049a2aa16b0b6cb
author: Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
date: Mon Dec 30 05:50:00 EST 2019 

Support files in content mounts

This commit is a general improvement of handling if single file mounts.

Fixes #6684
Fixes #6696


--- a/commands/hugo.go
+++ b/commands/hugo.go
@@ -667,7 +667,7 @@
 
 // getDirList provides NewWatcher() with a list of directories to watch for changes.
 func (c *commandeer) getDirList() ([]string, error) {
-	var dirnames []string
+	var filenames []string
 
 	walkFn := func(path string, fi hugofs.FileMetaInfo, err error) error {
 		if err != nil {
@@ -681,7 +681,7 @@
 				return filepath.SkipDir
 			}
 
-			dirnames = append(dirnames, fi.Meta().Filename())
+			filenames = append(filenames, fi.Meta().Filename())
 		}
 
 		return nil
@@ -688,18 +688,22 @@
 
 	}
 
-	watchDirs := c.hugo().PathSpec.BaseFs.WatchDirs()
-	for _, watchDir := range watchDirs {
+	watchFiles := c.hugo().PathSpec.BaseFs.WatchDirs()
+	for _, fi := range watchFiles {
+		if !fi.IsDir() {
+			filenames = append(filenames, fi.Meta().Filename())
+			continue
+		}
 
-		w := hugofs.NewWalkway(hugofs.WalkwayConfig{Logger: c.logger, Info: watchDir, WalkFn: walkFn})
+		w := hugofs.NewWalkway(hugofs.WalkwayConfig{Logger: c.logger, Info: fi, WalkFn: walkFn})
 		if err := w.Walk(); err != nil {
 			c.logger.ERROR.Println("walker: ", err)
 		}
 	}
 
-	dirnames = helpers.UniqueStringsSorted(dirnames)
+	filenames = helpers.UniqueStringsSorted(filenames)
 
-	return dirnames, nil
+	return filenames, nil
 }
 
 func (c *commandeer) buildSites() (err error) {
--- a/hugofs/fileinfo.go
+++ b/hugofs/fileinfo.go
@@ -35,6 +35,9 @@
 
 const (
 	metaKeyFilename                   = "filename"
+	metaKeyPathFile                   = "pathFile"    // Path of filename relative to a root.
+	metaKeyIsFileMount                = "isFileMount" // Whether the source mount was a file.
+	metaKeyMountRoot                  = "mountRoot"
 	metaKeyOriginalFilename           = "originalFilename"
 	metaKeyName                       = "name"
 	metaKeyPath                       = "path"
@@ -108,10 +111,34 @@
 	return f.stringV(metaKeyLang)
 }
 
+// Path returns the relative file path to where this file is mounted.
 func (f FileMeta) Path() string {
 	return f.stringV(metaKeyPath)
 }
 
+// PathFile returns the relative file path for the file source. This
+// will in most cases be the same as Path.
+func (f FileMeta) PathFile() string {
+	pf := f.stringV(metaKeyPathFile)
+	if f.isFileMount() {
+		return pf
+	}
+	mountRoot := f.mountRoot()
+	if mountRoot == pf {
+		return f.Path()
+	}
+
+	return pf + (strings.TrimPrefix(f.Path(), mountRoot))
+}
+
+func (f FileMeta) mountRoot() string {
+	return f.stringV(metaKeyMountRoot)
+}
+
+func (f FileMeta) isFileMount() bool {
+	return f.GetBool(metaKeyIsFileMount)
+}
+
 func (f FileMeta) Weight() int {
 	return f.GetInt(metaKeyWeight)
 }
@@ -129,10 +156,6 @@
 	return f.GetBool(metaKeyIsSymlink)
 }
 
-func (f FileMeta) String() string {
-	return f.Filename()
-}
-
 func (f FileMeta) Watch() bool {
 	if v, found := f["watch"]; found {
 		return v.(bool)
@@ -208,6 +231,14 @@
 		mergeFileMeta(fim.Meta(), m)
 	}
 	return &fileInfoMeta{FileInfo: fi, m: m}
+}
+
+func copyFileMeta(m FileMeta) FileMeta {
+	c := make(FileMeta)
+	for k, v := range m {
+		c[k] = v
+	}
+	return c
 }
 
 // Merge metadata, last entry wins.
--- a/hugofs/rootmapping_fs.go
+++ b/hugofs/rootmapping_fs.go
@@ -35,7 +35,7 @@
 func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
 	rootMapToReal := radix.New()
 
-	for _, rm := range rms {
+	for i, rm := range rms {
 		(&rm).clean()
 
 		fromBase := files.ResolveComponentFolder(rm.From)
@@ -47,7 +47,7 @@
 			panic(fmt.Sprintf("invalid root mapping; from/to: %s/%s", rm.From, rm.To))
 		}
 
-		_, err := fs.Stat(rm.To)
+		fi, err := fs.Stat(rm.To)
 		if err != nil {
 			if os.IsNotExist(err) {
 				continue
@@ -54,10 +54,26 @@
 			}
 			return nil, err
 		}
-
 		// Extract "blog" from "content/blog"
 		rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator)
+		if rm.Meta != nil {
+			rm.Meta[metaKeyIsFileMount] = !fi.IsDir()
+			rm.Meta[metaKeyMountRoot] = rm.path
+			if rm.ToBasedir != "" {
+				pathFile := strings.TrimPrefix(strings.TrimPrefix(rm.To, rm.ToBasedir), filepathSeparator)
+				rm.Meta[metaKeyPathFile] = pathFile
+			}
+		}
 
+		meta := copyFileMeta(rm.Meta)
+
+		if !fi.IsDir() {
+			_, name := filepath.Split(rm.From)
+			meta[metaKeyName] = name
+		}
+
+		rm.fi = NewFileMetaInfo(fi, meta)
+
 		key := rm.rootKey()
 		var mappings []RootMapping
 		v, found := rootMapToReal.Get(key)
@@ -67,6 +83,8 @@
 		}
 		mappings = append(mappings, rm)
 		rootMapToReal.Insert(key, mappings)
+
+		rms[i] = rm
 	}
 
 	rfs := &RootMappingFs{Fs: fs,
@@ -91,11 +109,14 @@
 }
 
 type RootMapping struct {
-	From string
-	To   string
+	From      string   // The virtual mount.
+	To        string   // The source directory or file.
+	ToBasedir string   // The base of To. May be empty if an absolute path was provided.
+	Meta      FileMeta // File metadata (lang etc.)
 
-	path string   // The virtual mount point, e.g. "blog".
-	Meta FileMeta // File metadata (lang etc.)
+	fi   FileMetaInfo
+	path string // The virtual mount point, e.g. "blog".
+
 }
 
 func (rm *RootMapping) clean() {
@@ -148,6 +169,11 @@
 		if err != nil {
 			return nil, errors.Wrap(err, "RootMappingFs.Dirs")
 		}
+
+		if !fi.IsDir() {
+			mergeFileMeta(r.Meta, fi.(FileMetaInfo).Meta())
+		}
+
 		fss[i] = fi.(FileMetaInfo)
 	}
 
@@ -168,7 +194,6 @@
 }
 
 func (fs *RootMappingFs) doLstat(name string, allowMultiple bool) ([]FileMetaInfo, []FileMetaInfo, bool, error) {
-
 	if fs.isRoot(name) {
 		return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, true))}, nil, false, nil
 	}
@@ -210,10 +235,12 @@
 			return nil, nil, false, err
 		}
 		fim := fi.(FileMetaInfo)
+
 		fis = append(fis, fim)
 	}
 
 	for _, root = range rootsInDir {
+
 		fi, _, err := fs.statRoot(root, "")
 		if err != nil {
 			if os.IsNotExist(err) {
@@ -500,9 +527,9 @@
 
 func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) {
 	if f.File == nil {
-		dirsn := make([]os.FileInfo, 0)
+		filesn := make([]os.FileInfo, 0)
 		roots := f.fs.getRootsWithPrefix(f.name)
-		seen := make(map[string]bool)
+		seen := make(map[string]bool) // Do not return duplicate directories
 
 		j := 0
 		for _, rm := range roots {
@@ -510,13 +537,16 @@
 				break
 			}
 
-			opener := func() (afero.File, error) {
-				return f.fs.Open(rm.From)
+			if !rm.fi.IsDir() {
+				// A single file mount
+				filesn = append(filesn, rm.fi)
+				continue
 			}
 
-			name := rm.From
+			from := rm.From
+			name := from
 			if !f.isRoot {
-				_, name = filepath.Split(rm.From)
+				_, name = filepath.Split(from)
 			}
 
 			if seen[name] {
@@ -524,16 +554,21 @@
 			}
 			seen[name] = true
 
+			opener := func() (afero.File, error) {
+				return f.fs.Open(from)
+			}
+
 			j++
 
 			fi := newDirNameOnlyFileInfo(name, false, opener)
+
 			if rm.Meta != nil {
 				mergeFileMeta(rm.Meta, fi.Meta())
 			}
 
-			dirsn = append(dirsn, fi)
+			filesn = append(filesn, fi)
 		}
-		return dirsn, nil
+		return filesn, nil
 	}
 
 	if f.File == nil {
--- a/hugofs/rootmapping_fs_test.go
+++ b/hugofs/rootmapping_fs_test.go
@@ -186,21 +186,40 @@
 	c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", testfile), []byte("some en content"), 0755), qt.IsNil)
 	c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", testfile), []byte("some sv content"), 0755), qt.IsNil)
 	c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "other.txt"), []byte("some sv content"), 0755), qt.IsNil)
+	c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/singlefiles", "no.txt"), []byte("no text"), 0755), qt.IsNil)
+	c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/singlefiles", "sv.txt"), []byte("sv text"), 0755), qt.IsNil)
 
 	bfs := afero.NewBasePathFs(fs, "themes/a").(*afero.BasePathFs)
 	rm := []RootMapping{
-		RootMapping{From: "content/blog",
+		// Directories
+		RootMapping{
+			From: "content/blog",
 			To:   "mynoblogcontent",
 			Meta: FileMeta{"lang": "no"},
 		},
-		RootMapping{From: "content/blog",
+		RootMapping{
+			From: "content/blog",
 			To:   "myenblogcontent",
 			Meta: FileMeta{"lang": "en"},
 		},
-		RootMapping{From: "content/blog",
+		RootMapping{
+			From: "content/blog",
 			To:   "mysvblogcontent",
 			Meta: FileMeta{"lang": "sv"},
 		},
+		// Files
+		RootMapping{
+			From:      "content/singles/p1.md",
+			To:        "singlefiles/no.txt",
+			ToBasedir: "singlefiles",
+			Meta:      FileMeta{"lang": "no"},
+		},
+		RootMapping{
+			From:      "content/singles/p1.md",
+			To:        "singlefiles/sv.txt",
+			ToBasedir: "singlefiles",
+			Meta:      FileMeta{"lang": "sv"},
+		},
 	}
 
 	rfs, err := NewRootMappingFs(bfs, rm...)
@@ -208,6 +227,7 @@
 
 	blog, err := rfs.Stat(filepath.FromSlash("content/blog"))
 	c.Assert(err, qt.IsNil)
+	c.Assert(blog.IsDir(), qt.Equals, true)
 	blogm := blog.(FileMetaInfo).Meta()
 	c.Assert(blogm.Lang(), qt.Equals, "no") // First match
 
@@ -236,6 +256,25 @@
 	c.Assert(err, qt.IsNil)
 	c.Assert(string(b), qt.Equals, "some no content")
 
+	// Check file mappings
+	single, err := rfs.Stat(filepath.FromSlash("content/singles/p1.md"))
+	c.Assert(err, qt.IsNil)
+	c.Assert(single.IsDir(), qt.Equals, false)
+	singlem := single.(FileMetaInfo).Meta()
+	c.Assert(singlem.Lang(), qt.Equals, "no") // First match
+
+	singlesDir, err := rfs.Open(filepath.FromSlash("content/singles"))
+	c.Assert(err, qt.IsNil)
+	defer singlesDir.Close()
+	singles, err := singlesDir.Readdir(-1)
+	c.Assert(err, qt.IsNil)
+	c.Assert(singles, qt.HasLen, 2)
+	for i, lang := range []string{"no", "sv"} {
+		fi := singles[i].(FileMetaInfo)
+		c.Assert(fi.Meta().PathFile(), qt.Equals, lang+".txt")
+		c.Assert(fi.Meta().Lang(), qt.Equals, lang)
+		c.Assert(fi.Name(), qt.Equals, "p1.md")
+	}
 }
 
 func TestRootMappingFsMountOverlap(t *testing.T) {
--- a/hugolib/filesystems/basefs.go
+++ b/hugolib/filesystems/basefs.go
@@ -18,6 +18,7 @@
 import (
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 	"strings"
 	"sync"
@@ -55,6 +56,8 @@
 	theBigFs *filesystemsCollector
 }
 
+// TODO(bep) we can get regular files in here and that is fine, but
+// we need to clean up the naming.
 func (fs *BaseFs) WatchDirs() []hugofs.FileMetaInfo {
 	var dirs []hugofs.FileMetaInfo
 	for _, dir := range fs.AllDirs() {
@@ -62,7 +65,6 @@
 			dirs = append(dirs, dir)
 		}
 	}
-
 	return dirs
 }
 
@@ -90,7 +92,7 @@
 	for _, dir := range b.SourceFilesystems.Content.Dirs {
 		dirname := dir.Meta().Filename()
 		if strings.HasPrefix(filename, dirname) {
-			rel := strings.TrimPrefix(filename, dirname)
+			rel := path.Join(dir.Meta().Path(), strings.TrimPrefix(filename, dirname))
 			return strings.TrimPrefix(rel, filePathSeparator)
 		}
 	}
@@ -298,8 +300,16 @@
 func (d *SourceFilesystem) Path(filename string) string {
 	for _, dir := range d.Dirs {
 		meta := dir.Meta()
+		if !dir.IsDir() {
+			if filename == meta.Filename() {
+				return meta.PathFile()
+			}
+			continue
+		}
+
 		if strings.HasPrefix(filename, meta.Filename()) {
 			p := strings.TrimPrefix(strings.TrimPrefix(filename, meta.Filename()), filePathSeparator)
+			p = path.Join(meta.PathFile(), p)
 			return p
 		}
 	}
@@ -530,11 +540,11 @@
 		fromToStatic  []hugofs.RootMapping
 	)
 
-	absPathify := func(path string) string {
+	absPathify := func(path string) (string, string) {
 		if filepath.IsAbs(path) {
-			return path
+			return "", path
 		}
-		return paths.AbsPathify(md.dir, path)
+		return md.dir, paths.AbsPathify(md.dir, path)
 	}
 
 	for _, mount := range md.Mounts() {
@@ -544,9 +554,12 @@
 			mountWeight++
 		}
 
+		base, filename := absPathify(mount.Source)
+
 		rm := hugofs.RootMapping{
-			From: mount.Target,
-			To:   absPathify(mount.Source),
+			From:      mount.Target,
+			To:        filename,
+			ToBasedir: base,
 			Meta: hugofs.FileMeta{
 				"watch":       md.Watch(),
 				"mountWeight": mountWeight,
@@ -621,7 +634,8 @@
 		if md.isMainProject {
 			return b.p.AbsResourcesDir
 		}
-		return absPathify(files.FolderResources)
+		_, filename := absPathify(files.FolderResources)
+		return filename
 	}
 
 	if collector.overlayMounts == nil {
--- a/hugolib/hugo_modules_test.go
+++ b/hugolib/hugo_modules_test.go
@@ -545,6 +545,85 @@
 	b.AssertFileContent("public/mypage/index.html", "Permalink: https://example.org/mypage/")
 }
 
+// https://github.com/gohugoio/hugo/issues/6684
+func TestMountsContentFile(t *testing.T) {
+	c := qt.New(t)
+	workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-content-file")
+	c.Assert(err, qt.IsNil)
+	defer clean()
+
+	configTemplate := `
+baseURL = "https://example.com"
+title = "My Modular Site"
+workingDir = %q
+
+[module]
+  [[module.mounts]]
+    source = "README.md"
+    target = "content/_index.md"
+  [[module.mounts]]
+    source = "mycontent"
+    target = "content/blog"
+
+`
+
+	config := fmt.Sprintf(configTemplate, workingDir)
+
+	b := newTestSitesBuilder(t).Running()
+
+	b.Fs = hugofs.NewDefault(viper.New())
+
+	b.WithWorkingDir(workingDir).WithConfigFile("toml", config)
+	b.WithTemplatesAdded("index.html", `
+{{ .Title }}
+{{ .Content }}
+
+{{ $readme := .Site.GetPage "/README.md" }}
+{{ with $readme }}README: {{ .Title }}|Filename: {{ path.Join .File.Filename }}|Path: {{ path.Join .File.Path }}|FilePath: {{ path.Join .File.FileInfo.Meta.PathFile }}|{{ end }}
+
+
+{{ $mypage := .Site.GetPage "/blog/mypage.md" }}
+{{ with $mypage }}MYPAGE: {{ .Title }}|Path: {{ path.Join .File.Path }}|FilePath: {{ path.Join .File.FileInfo.Meta.PathFile }}|{{ end }}
+
+`)
+
+	os.Mkdir(filepath.Join(workingDir, "mycontent"), 0777)
+
+	b.WithSourceFile("README.md", `---
+title: "Readme Title"
+---
+
+Readme Content.
+`,
+		filepath.Join("mycontent", "mypage.md"), `
+---
+title: "My Page"
+---
+
+`)
+
+	b.Build(BuildCfg{})
+
+	b.AssertFileContent("public/index.html", `
+README: Readme Title
+/README.md|Path: _index.md|FilePath: README.md
+Readme Content.
+MYPAGE: My Page|Path: blog/mypage.md|FilePath: mycontent/mypage.md|
+`)
+	b.AssertFileContent("public/blog/mypage/index.html", "Single: My Page")
+
+	b.EditFiles("README.md", `---
+title: "Readme Edit"
+---
+`)
+
+	b.Build(BuildCfg{})
+
+	b.AssertFileContent("public/index.html", `
+Readme Edit
+`)
+}
+
 // https://github.com/gohugoio/hugo/issues/6299
 func TestSiteWithGoModButNoModules(t *testing.T) {
 	t.Parallel()
--- a/hugolib/hugo_sites.go
+++ b/hugolib/hugo_sites.go
@@ -895,7 +895,7 @@
 	m.mu.Unlock()
 }
 
-func (m *contentChangeMap) resolveAndRemove(filename string) (string, string, bundleDirType) {
+func (m *contentChangeMap) resolveAndRemove(filename string) (string, bundleDirType) {
 	m.mu.RLock()
 	defer m.mu.RUnlock()
 
@@ -908,22 +908,22 @@
 
 	if _, found := m.branchBundles[dir]; found {
 		delete(m.branchBundles, dir)
-		return dir, dir, bundleBranch
+		return dir, bundleBranch
 	}
 
 	if key, _, found := m.leafBundles.LongestPrefix(dir); found {
 		m.leafBundles.Delete(key)
 		dir = string(key)
-		return dir, dir, bundleLeaf
+		return dir, bundleLeaf
 	}
 
 	fileTp, isContent := classifyBundledFile(name)
 	if isContent && fileTp != bundleNot {
 		// A new bundle.
-		return dir, dir, fileTp
+		return dir, fileTp
 	}
 
-	return dir, filename, bundleNot
+	return dir, bundleNot
 
 }
 
--- a/hugolib/page.go
+++ b/hugolib/page.go
@@ -946,6 +946,24 @@
 	return ""
 }
 
+func (p *pageState) sourceRefs() []string {
+	refs := []string{p.sourceRef()}
+
+	if !p.File().IsZero() {
+		meta := p.File().FileInfo().Meta()
+		path := meta.PathFile()
+
+		if path != "" {
+			ref := "/" + path
+			if ref != refs[0] {
+				refs = append(refs, ref)
+			}
+
+		}
+	}
+	return refs
+}
+
 type pageStatePages []*pageState
 
 // Implement sorting.
--- a/hugolib/pagecollections.go
+++ b/hugolib/pagecollections.go
@@ -151,12 +151,11 @@
 		for _, pageCollection := range []pageStatePages{c.workAllPages, c.headlessPages} {
 			for _, p := range pageCollection {
 				if p.IsPage() {
-					sourceRef := p.sourceRef()
-					if sourceRef != "" {
-						// index the canonical ref
-						// e.g. /section/article.md
-						add(sourceRef, p)
+					sourceRefs := p.sourceRefs()
+					for _, ref := range sourceRefs {
+						add(ref, p)
 					}
+					sourceRef := sourceRefs[0]
 
 					// Ref/Relref supports this potentially ambiguous lookup.
 					add(p.File().LogicalName(), p)
@@ -177,11 +176,9 @@
 					pathWithNoExtensions := path.Join(dir, translationBaseName)
 					add(pathWithNoExtensions, p)
 				} else {
-					// index the canonical, unambiguous ref for any backing file
-					// e.g. /section/_index.md
-					sourceRef := p.sourceRef()
-					if sourceRef != "" {
-						add(sourceRef, p)
+					sourceRefs := p.sourceRefs()
+					for _, ref := range sourceRefs {
+						add(ref, p)
 					}
 
 					ref := p.SectionsPath()
--- a/hugolib/pages_capture.go
+++ b/hugolib/pages_capture.go
@@ -116,7 +116,7 @@
 	} else {
 		dirs := make(map[contentDirKey]bool)
 		for _, filename := range c.filenames {
-			dir, filename, btype := c.tracker.resolveAndRemove(filename)
+			dir, btype := c.tracker.resolveAndRemove(filename)
 			dirs[contentDirKey{dir, filename, btype}] = true
 		}
 
@@ -127,7 +127,7 @@
 			default:
 				// We always start from a directory.
 				collectErr = c.collectDir(dir.dirname, true, func(fim hugofs.FileMetaInfo) bool {
-					return strings.HasSuffix(dir.filename, fim.Meta().Path())
+					return dir.filename == fim.Meta().Filename()
 				})
 			}
 
@@ -211,6 +211,7 @@
 		for _, fi := range readdir {
 			if filter(fi) {
 				filtered = append(filtered, fi)
+
 				if c.tracker != nil {
 					// Track symlinks.
 					c.tracker.addSymbolicLinkMapping(fi)