ref: 80dd6ddde27ce36f5432fb780e94d4974b5277c7
parent: 299731012441378bb9c057ceb0a3c277108aaf01
author: Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
date: Fri Jan 31 12:15:14 EST 2020
Fix module mount in sub folder This addresses a specific issue, but is a also a major simplification of the filesystem file mounts. Fixes #6730
--- a/hugofs/decorators.go
+++ b/hugofs/decorators.go
@@ -79,7 +79,7 @@
}
// NewBaseFileDecorator decorates the given Fs to provide the real filename
-// and an Opener func. If
+// and an Opener func.
func NewBaseFileDecorator(fs afero.Fs) afero.Fs { ffs := &baseFileDecoratorFs{Fs: fs}@@ -102,7 +102,6 @@
opener := func() (afero.File, error) {return ffs.open(filename)
-
}
return decorateFileInfo(fi, ffs, opener, filename, "", meta), nil
--- a/hugofs/fileinfo.go
+++ b/hugofs/fileinfo.go
@@ -18,6 +18,7 @@
"os"
"path/filepath"
"runtime"
+ "sort"
"strings"
"time"
@@ -271,13 +272,21 @@
return nil
}
-func newDirNameOnlyFileInfo(name string, isOrdered bool, fileOpener func() (afero.File, error)) FileMetaInfo {+func newDirNameOnlyFileInfo(name string, meta FileMeta, isOrdered bool, fileOpener func() (afero.File, error)) FileMetaInfo {name = normalizeFilename(name)
_, base := filepath.Split(name)
- return NewFileMetaInfo(&dirNameOnlyFileInfo{name: base}, FileMeta{- metaKeyFilename: name,
- metaKeyIsOrdered: isOrdered,
- metaKeyOpener: fileOpener})
+
+ m := copyFileMeta(meta)
+ if _, found := m[metaKeyFilename]; !found {+ m.setIfNotZero(metaKeyFilename, name)
+ }
+ m[metaKeyOpener] = fileOpener
+ m[metaKeyIsOrdered] = isOrdered
+
+ return NewFileMetaInfo(
+ &dirNameOnlyFileInfo{name: base},+ m,
+ )
}
func decorateFileInfo(
@@ -338,4 +347,19 @@
names[i] = d.Name()
}
return names
+}
+
+func fromSlash(filenames []string) []string {+ for i, name := range filenames {+ filenames[i] = filepath.FromSlash(name)
+ }
+ return filenames
+}
+
+func sortFileInfos(fis []os.FileInfo) {+ sort.Slice(fis, func(i, j int) bool {+ fimi, fimj := fis[i].(FileMetaInfo), fis[j].(FileMetaInfo)
+ return fimi.Meta().Filename() < fimj.Meta().Filename()
+
+ })
}
--- a/hugofs/nosymlink_test.go
+++ b/hugofs/nosymlink_test.go
@@ -137,6 +137,7 @@
c.Assert(err, qt.IsNil)
// There is at least one unsported symlink inside workDir
_, err = f.Readdir(-1)
+ c.Assert(err, qt.IsNil)
f.Close()
c.Assert(logger.WarnCounter.Count(), qt.Equals, uint64(1))
--- a/hugofs/rootmapping_fs.go
+++ b/hugofs/rootmapping_fs.go
@@ -27,15 +27,18 @@
"github.com/spf13/afero"
)
-var filepathSeparator = string(filepath.Separator)
+var (
+ filepathSeparator = string(filepath.Separator)
+)
// NewRootMappingFs creates a new RootMappingFs on top of the provided with
-// of root mappings with some optional metadata about the root.
+// root mappings with some optional metadata about the root.
// Note that From represents a virtual root that maps to the actual filename in To.
func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {rootMapToReal := radix.New()
+ var virtualRoots []RootMapping
- for i, rm := range rms {+ for _, rm := range rms {(&rm).clean()
fromBase := files.ResolveComponentFolder(rm.From)
@@ -56,11 +59,13 @@
}
// Extract "blog" from "content/blog"
rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator)
- if rm.Meta != nil {- rm.Meta[metaKeyBaseDir] = rm.ToBasedir
- rm.Meta[metaKeyMountRoot] = rm.path
+ if rm.Meta == nil {+ rm.Meta = make(FileMeta)
}
+ rm.Meta[metaKeyBaseDir] = rm.ToBasedir
+ rm.Meta[metaKeyMountRoot] = rm.path
+
meta := copyFileMeta(rm.Meta)
if !fi.IsDir() {@@ -70,7 +75,7 @@
rm.fi = NewFileMetaInfo(fi, meta)
- key := rm.rootKey()
+ key := filepathSeparator + rm.From
var mappings []RootMapping
v, found := rootMapToReal.Get(key)
if found {@@ -80,24 +85,31 @@
mappings = append(mappings, rm)
rootMapToReal.Insert(key, mappings)
- rms[i] = rm
+ virtualRoots = append(virtualRoots, rm)
}
- rfs := &RootMappingFs{Fs: fs,- virtualRoots: rms,
- rootMapToReal: rootMapToReal}
+ rootMapToReal.Insert(filepathSeparator, virtualRoots)
+ rfs := &RootMappingFs{+ Fs: fs,
+ rootMapToReal: rootMapToReal,
+ }
+
return rfs, nil
}
-// NewRootMappingFsFromFromTo is a convenicence variant of NewRootMappingFs taking
-// From and To as string pairs.
-func NewRootMappingFsFromFromTo(fs afero.Fs, fromTo ...string) (*RootMappingFs, error) {+func newRootMappingFsFromFromTo(
+ baseDir string,
+ fs afero.Fs,
+ fromTo ...string,
+) (*RootMappingFs, error) {+
rms := make([]RootMapping, len(fromTo)/2)
for i, j := 0, 0; j < len(fromTo); i, j = i+1, j+2 { rms[i] = RootMapping{- From: fromTo[j],
- To: fromTo[j+1],
+ From: fromTo[j],
+ To: fromTo[j+1],
+ ToBasedir: baseDir,
}
}
@@ -104,6 +116,7 @@
return NewRootMappingFs(fs, rms...)
}
+// RootMapping describes a virtual file or directory mount.
type RootMapping struct {From string // The virtual mount.
To string // The source directory or file.
@@ -127,10 +140,6 @@
return filepath.Join(r.To, strings.TrimPrefix(name, r.From))
}
-func (r RootMapping) rootKey() string {- return r.From
-}
-
// A RootMappingFs maps several roots into one. Note that the root of this filesystem
// is directories only, and they will be returned in Readdir and Readdirnames
// in the order given.
@@ -137,11 +146,10 @@
type RootMappingFs struct {afero.Fs
rootMapToReal *radix.Tree
- virtualRoots []RootMapping
- filter func(r RootMapping) bool
}
func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) {+ base = filepathSeparator + fs.cleanName(base)
roots := fs.getRootsWithPrefix(base)
if roots == nil {@@ -176,138 +184,46 @@
return fss, nil
}
-// LstatIfPossible returns the os.FileInfo structure describing a given file.
-func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {- fis, _, b, err := fs.doLstat(name, false)
- if err != nil {- return nil, b, err
- }
- return fis[0], b, nil
-}
-
-func (fs *RootMappingFs) virtualDirOpener(name string, isRoot bool) func() (afero.File, error) {- return func() (afero.File, error) { return &rootMappingFile{name: name, isRoot: isRoot, fs: fs}, nil }-}
-
-func (fs *RootMappingFs) doLstat(name string, allowMultiple bool) ([]FileMetaInfo, []FileMetaInfo, bool, error) {- if fs.isRoot(name) {- return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, true))}, nil, false, nil- }
-
- roots := fs.getRoots(name)
- rootsWithPrefix := fs.getRootsWithPrefix(name)
- hasRootMappingsBelow := len(rootsWithPrefix) != 0
-
- if len(roots) == 0 {- if hasRootMappingsBelow {- // No exact matches, but we have root mappings below name,
- // let's make it look like a directory.
- return []FileMetaInfo{newDirNameOnlyFileInfo(name, true, fs.virtualDirOpener(name, false))}, nil, false, nil- }
-
- return nil, nil, false, os.ErrNotExist
- }
-
- // We may have a mapping for both static and static/subdir.
- // These will not show in any Readdir so append them
- // manually.
- rootsInDir := fs.filterRootsBelow(rootsWithPrefix, name)
-
- var (
- fis []FileMetaInfo
- dirs []FileMetaInfo
- b bool
- root RootMapping
- err error
- )
-
- for _, root = range roots {- var fi os.FileInfo
- fi, b, err = fs.statRoot(root, name)
- if err != nil {- if os.IsNotExist(err) {- continue
+// Filter creates a copy of this filesystem with only mappings matching a filter.
+func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs {+ rootMapToReal := radix.New()
+ fs.rootMapToReal.Walk(func(b string, v interface{}) bool {+ rms := v.([]RootMapping)
+ var nrms []RootMapping
+ for _, rm := range rms {+ if f(rm) {+ nrms = append(nrms, rm)
}
- return nil, nil, false, err
}
- fim := fi.(FileMetaInfo)
-
- fis = append(fis, fim)
- }
-
- for _, root = range rootsInDir {-
- fi, _, err := fs.statRoot(root, "")
- if err != nil {- if os.IsNotExist(err) {- continue
- }
- return nil, nil, false, err
+ if len(nrms) != 0 {+ rootMapToReal.Insert(b, nrms)
}
- fim := fi.(FileMetaInfo)
- dirs = append(dirs, fim)
- }
+ return false
+ })
- if len(fis) == 0 && len(dirs) == 0 {- return nil, nil, false, os.ErrNotExist
- }
+ fs.rootMapToReal = rootMapToReal
- if allowMultiple || len(fis) == 1 {- return fis, dirs, b, nil
- }
-
- if len(fis) == 0 {- return nil, nil, false, os.ErrNotExist
- }
-
- // Open it in this composite filesystem.
- opener := func() (afero.File, error) {- return fs.Open(name)
- }
-
- return []FileMetaInfo{decorateFileInfo(fis[0], fs, opener, "", "", root.Meta)}, nil, b, nil-
+ return &fs
}
-// Open opens the namedrootMappingFile file for reading.
-func (fs *RootMappingFs) Open(name string) (afero.File, error) {- if fs.isRoot(name) {- return &rootMappingFile{name: name, fs: fs, isRoot: true}, nil- }
-
- fis, dirs, _, err := fs.doLstat(name, true)
+// LstatIfPossible returns the os.FileInfo structure describing a given file.
+func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {+ fis, err := fs.doLstat(name)
if err != nil {- return nil, err
+ return nil, false, err
}
+ return fis[0], false, nil
+}
- if len(fis) == 1 {- fi := fis[0]
- meta := fi.(FileMetaInfo).Meta()
- f, err := meta.Open()
- if err != nil {- return nil, err
- }
-
- f = &rootMappingFile{File: f, fs: fs, name: name, meta: meta}+// Open opens the named file for reading.
+func (fs *RootMappingFs) Open(name string) (afero.File, error) {+ fis, err := fs.doLstat(name)
- if len(dirs) > 0 {- return &readDirDirsAppender{File: f, dirs: dirs}, nil- }
-
- return f, nil
- }
-
- f, err := fs.newUnionFile(fis...)
if err != nil {return nil, err
}
- if len(dirs) > 0 {- return &readDirDirsAppender{File: f, dirs: dirs}, nil- }
-
- return f, nil
-
+ return fs.newUnionFile(fis...)
}
// Stat returns the os.FileInfo structure describing a given file. If there is
@@ -318,82 +234,53 @@
}
-// Filter creates a copy of this filesystem with the applied filter.
-func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs {- fs.filter = f
- return &fs
-}
+func (fs *RootMappingFs) hasPrefix(prefix string) bool {+ hasPrefix := false
+ fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool {+ hasPrefix = true
+ return true
+ })
-func (fs *RootMappingFs) isRoot(name string) bool {- return name == "" || name == filepathSeparator
-
+ return hasPrefix
}
-func (fs *RootMappingFs) getRoots(name string) []RootMapping {- name = filepath.Clean(name)
- _, v, found := fs.rootMapToReal.LongestPrefix(name)
+func (fs *RootMappingFs) getRoot(key string) []RootMapping {+ v, found := fs.rootMapToReal.Get(key)
if !found {return nil
}
- rm := v.([]RootMapping)
-
- return fs.applyFilterToRoots(rm)
+ return v.([]RootMapping)
}
-func (fs *RootMappingFs) applyFilterToRoots(rm []RootMapping) []RootMapping {- if fs.filter == nil {- return rm
+func (fs *RootMappingFs) getRoots(key string) (string, []RootMapping) {+ s, v, found := fs.rootMapToReal.LongestPrefix(key)
+ if !found || (s == filepathSeparator && key != filepathSeparator) {+ return "", nil
}
+ return s, v.([]RootMapping)
- var filtered []RootMapping
- for _, m := range rm {- if fs.filter(m) {- filtered = append(filtered, m)
- }
- }
+}
- return filtered
+func (fs *RootMappingFs) debug() {+ fmt.Println("debug():")+ fs.rootMapToReal.Walk(func(s string, v interface{}) bool {+ fmt.Println("Key", s)+ return false
+ })
+
}
func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping {- if fs.isRoot(prefix) {- return fs.virtualRoots
- }
- prefix = filepath.Clean(prefix)
var roots []RootMapping
-
fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool {roots = append(roots, v.([]RootMapping)...)
return false
})
- return fs.applyFilterToRoots(roots)
+ return roots
}
-// Filter out the mappings inside the name directory.
-func (fs *RootMappingFs) filterRootsBelow(roots []RootMapping, name string) []RootMapping {- if len(roots) == 0 {- return nil
- }
-
- sepCount := strings.Count(name, filepathSeparator)
- var filtered []RootMapping
- for _, x := range roots {- if name == x.From {- continue
- }
-
- if strings.Count(x.From, filepathSeparator)-sepCount != 1 {- continue
- }
-
- filtered = append(filtered, x)
-
- }
- return filtered
-}
-
func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {meta := fis[0].Meta()
f, err := meta.Open()
@@ -400,6 +287,10 @@
if err != nil {return nil, err
}
+ if len(fis) == 1 {+ return f, nil
+ }
+
rf := &rootMappingFile{File: f, fs: fs, name: meta.Name(), meta: meta} if len(fis) == 1 {return rf, err
@@ -439,112 +330,86 @@
}
-func (fs *RootMappingFs) statRoot(root RootMapping, name string) (os.FileInfo, bool, error) {- filename := root.filename(name)
+func (fs *RootMappingFs) cleanName(name string) string {+ return strings.Trim(filepath.Clean(name), filepathSeparator)
+}
- var b bool
- var fi os.FileInfo
- var err error
+func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error) {+ prefix = filepathSeparator + fs.cleanName(prefix)
- if ls, ok := fs.Fs.(afero.Lstater); ok {- fi, b, err = ls.LstatIfPossible(filename)
+ var fis []os.FileInfo
+
+ seen := make(map[string]bool) // Prevent duplicate directories
+ level := strings.Count(prefix, filepathSeparator)
+
+ // First add any real files/directories.
+ rms := fs.getRoot(prefix)
+ for _, rm := range rms {+ f, err := rm.fi.Meta().Open()
if err != nil {- return nil, b, err
+ return nil, err
}
-
- } else {- fi, err = fs.Fs.Stat(filename)
+ direntries, err := f.Readdir(-1)
if err != nil {- return nil, b, err
+ f.Close()
+ return nil, err
}
- }
- // Opens the real directory/file.
- opener := func() (afero.File, error) {- return fs.Fs.Open(filename)
- }
+ for _, fi := range direntries {+ meta := fi.(FileMetaInfo).Meta()
+ mergeFileMeta(rm.Meta, meta)
+ if fi.IsDir() {+ name := fi.Name()
+ if seen[name] {+ continue
+ }
+ seen[name] = true
+ opener := func() (afero.File, error) {+ return fs.Open(filepath.Join(rm.From, name))
+ }
+ fi = newDirNameOnlyFileInfo(name, meta, false, opener)
+ }
- if fi.IsDir() {- if name == "" {- name = root.From
+ fis = append(fis, fi)
}
- _, name = filepath.Split(name)
- fi = newDirNameOnlyFileInfo(name, false, opener)
- }
- return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil
-
-}
-
-type rootMappingFile struct {- afero.File
- fs *RootMappingFs
- name string
- meta FileMeta
- isRoot bool
-}
-
-type readDirDirsAppender struct {- afero.File
- dirs []FileMetaInfo
-}
-
-func (f *readDirDirsAppender) Readdir(count int) ([]os.FileInfo, error) {- fis, err := f.File.Readdir(count)
- if err != nil {- return nil, err
+ f.Close()
}
- for _, dir := range f.dirs {- fis = append(fis, dir)
- }
- return fis, nil
+ // Next add any file mounts inside the given directory.
+ prefixInside := prefix + filepathSeparator
+ fs.rootMapToReal.WalkPrefix(prefixInside, func(s string, v interface{}) bool {-}
+ if (strings.Count(s, filepathSeparator) - level) != 1 {+ // This directory is not part of the current, but we
+ // need to include the first name part to make it
+ // navigable.
+ path := strings.TrimPrefix(s, prefixInside)
+ parts := strings.Split(path, filepathSeparator)
+ name := parts[0]
-func (f *readDirDirsAppender) Readdirnames(count int) ([]string, error) {- fis, err := f.Readdir(count)
- if err != nil {- return nil, err
- }
- return fileInfosToNames(fis), nil
-}
+ if seen[name] {+ return false
+ }
+ seen[name] = true
+ opener := func() (afero.File, error) {+ return fs.Open(path)
+ }
-func (f *rootMappingFile) Close() error {- if f.File == nil {- return nil
- }
- return f.File.Close()
-}
+ fi := newDirNameOnlyFileInfo(name, nil, false, opener)
+ fis = append(fis, fi)
-func (f *rootMappingFile) Name() string {- return f.name
-}
+ return false
+ }
-func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) {- if f.File == nil {- filesn := make([]os.FileInfo, 0)
- roots := f.fs.getRootsWithPrefix(f.name)
- seen := make(map[string]bool) // Do not return duplicate directories
-
- j := 0
- for _, rm := range roots {- if count != -1 && j >= count {- break
- }
-
+ rms := v.([]RootMapping)
+ for _, rm := range rms { if !rm.fi.IsDir() {// A single file mount
- filesn = append(filesn, rm.fi)
+ fis = append(fis, rm.fi)
continue
}
-
- from := rm.From
- name := from
- if !f.isRoot {- _, name = filepath.Split(from)
- }
-
+ name := filepath.Base(rm.From)
if seen[name] {continue
}
@@ -551,36 +416,155 @@
seen[name] = true
opener := func() (afero.File, error) {- return f.fs.Open(from)
+ return fs.Open(rm.From)
}
- j++
+ fi := newDirNameOnlyFileInfo(name, rm.Meta, false, opener)
- fi := newDirNameOnlyFileInfo(name, false, opener)
+ fis = append(fis, fi)
- if rm.Meta != nil {- mergeFileMeta(rm.Meta, fi.Meta())
+ }
+
+ return false
+ })
+
+ return fis, nil
+}
+
+func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) {+ name = fs.cleanName(name)
+ key := filepathSeparator + name
+
+ roots := fs.getRoot(key)
+
+ if roots == nil {+ if fs.hasPrefix(key) {+ // We have directories mounted below this.
+ // Make it look like a directory.
+ return []FileMetaInfo{newDirNameOnlyFileInfo(name, nil, true, fs.virtualDirOpener(name))}, nil+ }
+
+ // Find any real files or directories with this key.
+ _, roots := fs.getRoots(key)
+ if roots == nil {+ return nil, os.ErrNotExist
+ }
+
+ var err error
+ var fis []FileMetaInfo
+
+ for _, rm := range roots {+ var fi FileMetaInfo
+ fi, _, err = fs.statRoot(rm, name)
+ if err == nil {+ fis = append(fis, fi)
}
+ }
- filesn = append(filesn, fi)
+ if fis != nil {+ return fis, nil
}
- return filesn, nil
+
+ if err == nil {+ err = os.ErrNotExist
+ }
+
+ return nil, err
}
- if f.File == nil {- panic(fmt.Sprintf("no File for %q", f.name))+ fileCount := 0
+ for _, root := range roots {+ if !root.fi.IsDir() {+ fileCount++
+ }
+ if fileCount > 1 {+ break
+ }
}
- fis, err := f.File.Readdir(count)
+ if fileCount == 0 {+ // Dir only.
+ return []FileMetaInfo{newDirNameOnlyFileInfo(name, roots[0].Meta, true, fs.virtualDirOpener(name))}, nil+ }
+
+ if fileCount > 1 {+ // Not supported by this filesystem.
+ return nil, errors.Errorf("found multiple files with name %q, use .Readdir or the source filesystem directly", name)+
+ }
+
+ return []FileMetaInfo{roots[0].fi}, nil+
+}
+
+func (fs *RootMappingFs) statRoot(root RootMapping, name string) (FileMetaInfo, bool, error) {+ filename := root.filename(name)
+
+ fi, b, err := lstatIfPossible(fs.Fs, filename)
if err != nil {- return nil, err
+ return nil, b, err
}
- for i, fi := range fis {- fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta)
+ var opener func() (afero.File, error)
+ if fi.IsDir() {+ // Make sure metadata gets applied in Readdir.
+ opener = fs.realDirOpener(filename, root.Meta)
+ } else {+ // Opens the real file directly.
+ opener = func() (afero.File, error) {+ return fs.Fs.Open(filename)
+ }
}
- return fis, nil
+ return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil
+
+}
+
+func (fs *RootMappingFs) virtualDirOpener(name string) func() (afero.File, error) {+ return func() (afero.File, error) { return &rootMappingFile{name: name, fs: fs}, nil }+}
+
+func (fs *RootMappingFs) realDirOpener(name string, meta FileMeta) func() (afero.File, error) {+ return func() (afero.File, error) {+ f, err := fs.Fs.Open(name)
+ if err != nil {+ return nil, err
+ }
+ return &rootMappingFile{name: name, meta: meta, fs: fs, File: f}, nil+ }
+}
+
+type rootMappingFile struct {+ afero.File
+ fs *RootMappingFs
+ name string
+ meta FileMeta
+}
+
+func (f *rootMappingFile) Close() error {+ if f.File == nil {+ return nil
+ }
+ return f.File.Close()
+}
+
+func (f *rootMappingFile) Name() string {+ return f.name
+}
+
+func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) {+ if f.File != nil {+ fis, err := f.File.Readdir(count)
+ if err != nil {+ return nil, err
+ }
+
+ for i, fi := range fis {+ fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta)
+ }
+ return fis, nil
+ }
+ return f.fs.collectDirEntries(f.name)
}
func (f *rootMappingFile) Readdirnames(count int) ([]string, error) {--- a/hugofs/rootmapping_fs_test.go
+++ b/hugofs/rootmapping_fs_test.go
@@ -14,9 +14,10 @@
package hugofs
import (
+ "fmt"
"io/ioutil"
- "os"
"path/filepath"
+ "sort"
"testing"
"github.com/spf13/viper"
@@ -34,8 +35,12 @@
fs := NewBaseFileDecorator(afero.NewMemMapFs())
c.Assert(afero.WriteFile(fs, filepath.Join("content/sv/svdir", "main.txt"), []byte("main sv"), 0755), qt.IsNil)+
c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent", "sv-f.txt"), []byte("some sv blog content"), 0755), qt.IsNil) c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent", "en-f.txt"), []byte("some en blog content in a"), 0755), qt.IsNil)+ c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvblogcontent/d1", "sv-d1-f.txt"), []byte("some sv blog content"), 0755), qt.IsNil)+ c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myenblogcontent/d1", "en-d1-f.txt"), []byte("some en blog content in a"), 0755), qt.IsNil)+
c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/myotherenblogcontent", "en-f2.txt"), []byte("some en content"), 0755), qt.IsNil) c.Assert(afero.WriteFile(fs, filepath.Join("themes/a/mysvdocs", "sv-docs.txt"), []byte("some sv docs content"), 0755), qt.IsNil) c.Assert(afero.WriteFile(fs, filepath.Join("themes/b/myenblogcontent", "en-b-f.txt"), []byte("some en content"), 0755), qt.IsNil)@@ -72,19 +77,30 @@
collected, err := collectFilenames(rfs, "content", "content")
c.Assert(err, qt.IsNil)
- c.Assert(collected, qt.DeepEquals, []string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"})+ c.Assert(collected, qt.DeepEquals,
+ []string{"blog/d1/en-d1-f.txt", "blog/d1/sv-d1-f.txt", "blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"}, qt.Commentf("%#v", collected))- bfs := afero.NewBasePathFs(rfs, "content")
- collected, err = collectFilenames(bfs, "", "")
- c.Assert(err, qt.IsNil)
- c.Assert(collected, qt.DeepEquals, []string{"blog/en-f.txt", "blog/en-f2.txt", "blog/sv-f.txt", "blog/svdir/main.txt", "docs/sv-docs.txt"})-
dirs, err := rfs.Dirs(filepath.FromSlash("content/blog"))c.Assert(err, qt.IsNil)
-
c.Assert(len(dirs), qt.Equals, 4)
+ for _, dir := range dirs {+ f, err := dir.Meta().Open()
+ c.Assert(err, qt.IsNil)
+ f.Close()
+ }
+ blog, err := rfs.Open(filepath.FromSlash("content/blog"))+ c.Assert(err, qt.IsNil)
+ fis, err := blog.Readdir(-1)
+ for _, fi := range fis {+ f, err := fi.(FileMetaInfo).Meta().Open()
+ c.Assert(err, qt.IsNil)
+ f.Close()
+ }
+ blog.Close()
+
getDirnames := func(name string, rfs *RootMappingFs) []string {+ c.Helper()
filename := filepath.FromSlash(name)
f, err := rfs.Open(filename)
c.Assert(err, qt.IsNil)
@@ -109,16 +125,16 @@
return rm.Meta.Lang() == "en"
})
- c.Assert(getDirnames("content/blog", rfsEn), qt.DeepEquals, []string{"en-f.txt", "en-f2.txt"})+ c.Assert(getDirnames("content/blog", rfsEn), qt.DeepEquals, []string{"d1", "en-f.txt", "en-f2.txt"}) rfsSv := rfs.Filter(func(rm RootMapping) bool {return rm.Meta.Lang() == "sv"
})
- c.Assert(getDirnames("content/blog", rfsSv), qt.DeepEquals, []string{"sv-f.txt", "svdir"})+ c.Assert(getDirnames("content/blog", rfsSv), qt.DeepEquals, []string{"d1", "sv-f.txt", "svdir"})// Make sure we have not messed with the original
- c.Assert(getDirnames("content/blog", rfs), qt.DeepEquals, []string{"sv-f.txt", "en-f.txt", "svdir", "en-f2.txt"})+ c.Assert(getDirnames("content/blog", rfs), qt.DeepEquals, []string{"d1", "sv-f.txt", "en-f.txt", "svdir", "en-f2.txt"}) c.Assert(getDirnames("content", rfsSv), qt.DeepEquals, []string{"blog", "docs"}) c.Assert(getDirnames("content", rfs), qt.DeepEquals, []string{"blog", "docs"})@@ -135,7 +151,7 @@
c.Assert(fs.Mkdir("f3t", 0755), qt.IsNil) c.Assert(afero.WriteFile(fs, filepath.Join("f2t", testfile), []byte("some content"), 0755), qt.IsNil)- rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", "f1t", "static/cf2", "f2t", "static/af3", "f3t")
+ rfs, err := newRootMappingFsFromFromTo("", fs, "static/bf1", "f1t", "static/cf2", "f2t", "static/af3", "f3t")c.Assert(err, qt.IsNil)
fif, err := rfs.Stat(filepath.Join("static/cf2", testfile))@@ -144,12 +160,12 @@
fifm := fif.(FileMetaInfo).Meta()
c.Assert(fifm.Filename(), qt.Equals, filepath.FromSlash("f2t/myfile.txt"))- root, err := rfs.Open(filepathSeparator)
+ root, err := rfs.Open("static")c.Assert(err, qt.IsNil)
dirnames, err := root.Readdirnames(-1)
c.Assert(err, qt.IsNil)
- c.Assert(dirnames, qt.DeepEquals, []string{"bf1", "cf2", "af3"})+ c.Assert(dirnames, qt.DeepEquals, []string{"af3", "bf1", "cf2"})}
@@ -165,7 +181,7 @@
c.Assert(fs.MkdirAll(filepath.Join(workDir, "f1t/foo"), 0777), qt.IsNil)
c.Assert(afero.WriteFile(fs, testfilename, []byte("content"), 0666), qt.IsNil)- rfs, err := NewRootMappingFsFromFromTo(fs, "static/f1", filepath.Join(workDir, "f1t"), "static/f2", filepath.Join(workDir, "f2t"))
+ rfs, err := newRootMappingFsFromFromTo(workDir, fs, "static/f1", filepath.Join(workDir, "f1t"), "static/f2", filepath.Join(workDir, "f2t"))
c.Assert(err, qt.IsNil)
fi, err := rfs.Stat(filepath.FromSlash("static/f1/foo/file.txt"))@@ -256,12 +272,9 @@
c.Assert(err, qt.IsNil)
c.Assert(string(b), qt.Equals, "some no content")
- // Check file mappings
- single, err := rfs.Stat(filepath.FromSlash("content/singles/p1.md"))- c.Assert(err, qt.IsNil)
- c.Assert(single.IsDir(), qt.Equals, false)
- singlem := single.(FileMetaInfo).Meta()
- c.Assert(singlem.Lang(), qt.Equals, "no") // First match
+ // Ambigous
+ _, err = rfs.Stat(filepath.FromSlash("content/singles/p1.md"))+ c.Assert(err, qt.Not(qt.IsNil))
singlesDir, err := rfs.Open(filepath.FromSlash("content/singles"))c.Assert(err, qt.IsNil)
@@ -308,7 +321,8 @@
rfs, err := NewRootMappingFs(fs, rm...)
c.Assert(err, qt.IsNil)
- getDirnames := func(name string) []string {+ checkDirnames := func(name string, expect []string) {+ c.Helper()
name = filepath.FromSlash(name)
f, err := rfs.Open(name)
c.Assert(err, qt.IsNil)
@@ -315,12 +329,12 @@
defer f.Close()
names, err := f.Readdirnames(-1)
c.Assert(err, qt.IsNil)
- return names
+ c.Assert(names, qt.DeepEquals, expect, qt.Commentf(fmt.Sprintf("%#v", names)))}
- c.Assert(getDirnames("static"), qt.DeepEquals, []string{"a.txt", "b", "e"})- c.Assert(getDirnames("static/b"), qt.DeepEquals, []string{"b.txt", "c"})- c.Assert(getDirnames("static/b/c"), qt.DeepEquals, []string{"c.txt"})+ checkDirnames("static", []string{"a.txt", "b", "e"})+ checkDirnames("static/b", []string{"b.txt", "c"})+ checkDirnames("static/b/c", []string{"c.txt"}) fi, err := rfs.Stat(filepath.FromSlash("static/b/b.txt"))c.Assert(err, qt.IsNil)
@@ -330,21 +344,37 @@
func TestRootMappingFsOs(t *testing.T) {c := qt.New(t)
- fs := afero.NewOsFs()
+ fs := NewBaseFileDecorator(afero.NewOsFs())
- d, err := ioutil.TempDir("", "hugo-root-mapping")+ d, clean, err := htesting.CreateTempDir(fs, "hugo-root-mapping-os")
c.Assert(err, qt.IsNil)
- defer func() {- os.RemoveAll(d)
- }()
+ defer clean()
testfile := "myfile.txt"
c.Assert(fs.Mkdir(filepath.Join(d, "f1t"), 0755), qt.IsNil)
c.Assert(fs.Mkdir(filepath.Join(d, "f2t"), 0755), qt.IsNil)
c.Assert(fs.Mkdir(filepath.Join(d, "f3t"), 0755), qt.IsNil)
+
+ // Deep structure
+ deepDir := filepath.Join(d, "d1", "d2", "d3", "d4", "d5")
+ c.Assert(fs.MkdirAll(deepDir, 0755), qt.IsNil)
+ for i := 1; i <= 3; i++ {+ c.Assert(fs.MkdirAll(filepath.Join(d, "d1", "d2", "d3", "d4", fmt.Sprintf("d4-%d", i)), 0755), qt.IsNil)+ c.Assert(afero.WriteFile(fs, filepath.Join(d, "d1", "d2", "d3", fmt.Sprintf("f-%d.txt", i)), []byte("some content"), 0755), qt.IsNil)+ }
+
c.Assert(afero.WriteFile(fs, filepath.Join(d, "f2t", testfile), []byte("some content"), 0755), qt.IsNil)- rfs, err := NewRootMappingFsFromFromTo(fs, "static/bf1", filepath.Join(d, "f1t"), "static/cf2", filepath.Join(d, "f2t"), "static/af3", filepath.Join(d, "f3t"))
+ rfs, err := newRootMappingFsFromFromTo(
+ d,
+ fs,
+ "static/bf1", filepath.Join(d, "f1t"),
+ "static/cf2", filepath.Join(d, "f2t"),
+ "static/af3", filepath.Join(d, "f3t"),
+ "static/a/b/c", filepath.Join(d, "d1", "d2", "d3"),
+ "layouts", filepath.Join(d, "d1"),
+ )
+
c.Assert(err, qt.IsNil)
fif, err := rfs.Stat(filepath.Join("static/cf2", testfile))@@ -351,11 +381,59 @@
c.Assert(err, qt.IsNil)
c.Assert(fif.Name(), qt.Equals, "myfile.txt")
- root, err := rfs.Open(filepathSeparator)
+ root, err := rfs.Open("static")c.Assert(err, qt.IsNil)
dirnames, err := root.Readdirnames(-1)
c.Assert(err, qt.IsNil)
- c.Assert(dirnames, qt.DeepEquals, []string{"bf1", "cf2", "af3"})+ c.Assert(dirnames, qt.DeepEquals, []string{"a", "af3", "bf1", "cf2"}, qt.Commentf(fmt.Sprintf("%#v", dirnames)))+ getDirnames := func(dirname string) []string {+ dirname = filepath.FromSlash(dirname)
+ f, err := rfs.Open(dirname)
+ c.Assert(err, qt.IsNil)
+ defer f.Close()
+ dirnames, err := f.Readdirnames(-1)
+ c.Assert(err, qt.IsNil)
+ sort.Strings(dirnames)
+ return dirnames
+ }
+
+ c.Assert(getDirnames("static/a/b"), qt.DeepEquals, []string{"c"})+ c.Assert(getDirnames("static/a/b/c"), qt.DeepEquals, []string{"d4", "f-1.txt", "f-2.txt", "f-3.txt"})+ c.Assert(getDirnames("static/a/b/c/d4"), qt.DeepEquals, []string{"d4-1", "d4-2", "d4-3", "d5"})+
+ all, err := collectFilenames(rfs, "static", "static")
+ c.Assert(err, qt.IsNil)
+
+ c.Assert(all, qt.DeepEquals, []string{"a/b/c/f-1.txt", "a/b/c/f-2.txt", "a/b/c/f-3.txt", "cf2/myfile.txt"})+
+ fis, err := collectFileinfos(rfs, "static", "static")
+ c.Assert(err, qt.IsNil)
+
+ c.Assert(fis[9].Meta().PathFile(), qt.Equals, filepath.FromSlash("d1/d2/d3/f-1.txt"))+
+ dirc := fis[3].Meta()
+
+ f, err := dirc.Open()
+ c.Assert(err, qt.IsNil)
+ defer f.Close()
+ fileInfos, err := f.Readdir(-1)
+ c.Assert(err, qt.IsNil)
+ sortFileInfos(fileInfos)
+ i := 0
+ for _, fi := range fileInfos {+ if fi.IsDir() {+ continue
+ }
+ i++
+ meta := fi.(FileMetaInfo).Meta()
+ c.Assert(meta.Filename(), qt.Equals, filepath.Join(d, fmt.Sprintf("/d1/d2/d3/f-%d.txt", i)))+ c.Assert(meta.PathFile(), qt.Equals, filepath.FromSlash(fmt.Sprintf("d1/d2/d3/f-%d.txt", i)))+ }
+
+ _, err = rfs.Stat(filepath.FromSlash("layouts/d2/d3/f-1.txt"))+ c.Assert(err, qt.IsNil)
+ _, err = rfs.Stat(filepath.FromSlash("layouts/d2/d3"))+ c.Assert(err, qt.IsNil)
}
--- a/hugofs/walk.go
+++ b/hugofs/walk.go
@@ -124,7 +124,6 @@
if w.checkErr(w.root, err) {return nil
}
-
return w.walkFn(w.root, nil, errors.Wrapf(err, "walk: %q", w.root))
}
fi = info.(FileMetaInfo)
@@ -154,6 +153,15 @@
logUnsupportedSymlink(filename, w.logger)
return true
}
+
+ if os.IsNotExist(err) {+ // The file may be removed in process.
+ // This may be a ERROR situation, but it is not possible
+ // to determine as a general case.
+ w.logger.WARN.Printf("File %q not found, skipping.", filename)+ return true
+ }
+
return false
}
--- a/hugofs/walk_test.go
+++ b/hugofs/walk_test.go
@@ -176,6 +176,27 @@
}
+func collectFileinfos(fs afero.Fs, base, root string) ([]FileMetaInfo, error) {+ var fis []FileMetaInfo
+
+ walkFn := func(path string, info FileMetaInfo, err error) error {+ if err != nil {+ return err
+ }
+
+ fis = append(fis, info)
+
+ return nil
+ }
+
+ w := NewWalkway(WalkwayConfig{Fs: fs, BasePath: base, Root: root, WalkFn: walkFn})+
+ err := w.Walk()
+
+ return fis, err
+
+}
+
func BenchmarkWalk(b *testing.B) {c := qt.New(b)
fs := NewBaseFileDecorator(afero.NewMemMapFs())
--- a/hugolib/filesystems/basefs.go
+++ b/hugolib/filesystems/basefs.go
@@ -258,6 +258,7 @@
// MakePathRelative creates a relative path from the given filename.
// It will return an empty string if the filename is not a member of this filesystem.
func (d *SourceFilesystem) MakePathRelative(filename string) string {+
for _, dir := range d.Dirs {meta := dir.(hugofs.FileMetaInfo).Meta()
currentPath := meta.Filename()
--- a/hugolib/filesystems/basefs_test.go
+++ b/hugolib/filesystems/basefs_test.go
@@ -173,9 +173,7 @@
filename = filepath.FromSlash(filename)
f, err := fs.Open(filename)
c.Assert(err, qt.IsNil)
- name := f.Name()
f.Close()
- c.Assert(name, qt.Equals, filename)
}
}
}
--- a/hugolib/hugo_modules_test.go
+++ b/hugolib/hugo_modules_test.go
@@ -38,6 +38,47 @@
"github.com/spf13/viper"
)
+// https://github.com/gohugoio/hugo/issues/6730
+func TestHugoModulesTargetInSubFolder(t *testing.T) {+ config := `
+baseURL="https://example.org"
+workingDir = %q
+
+[module]
+[[module.imports]]
+path="github.com/gohugoio/hugoTestModule2"
+ [[module.imports.mounts]]
+ source = "templates/hooks"
+ target = "layouts/_default/_markup"
+
+`
+
+ b := newTestSitesBuilder(t)
+ workingDir, clean, err := htesting.CreateTempDir(hugofs.Os, "hugo-modules-target-in-subfolder-test")
+ b.Assert(err, qt.IsNil)
+ defer clean()
+ b.Fs = hugofs.NewDefault(viper.New())
+ b.WithWorkingDir(workingDir).WithConfigFile("toml", fmt.Sprintf(config, workingDir))+ b.WithTemplates("_default/single.html", `{{ .Content }}`)+ b.WithContent("p1.md", `---+title: "Page"
+---
+
+[A link](https://bep.is)
+
+`)
+ b.WithSourceFile("go.mod", `+module github.com/gohugoio/tests/testHugoModules
+
+
+`)
+
+ b.Build(BuildCfg{})+
+ b.AssertFileContent("public/p1/index.html", `<p>Page|https://bep.is|Title: |Text: A link|END</p>`)+
+}
+
// TODO(bep) this fails when testmodBuilder is also building ...
func TestHugoModules(t *testing.T) { if !isCI() {@@ -588,7 +629,10 @@
{{ $mypage := .Site.GetPage "/blog/mypage.md" }} {{ with $mypage }}MYPAGE: {{ .Title }}|Path: {{ path.Join .File.Path }}|FilePath: {{ path.Join .File.FileInfo.Meta.PathFile }}|{{ end }}+{{ $mybundle := .Site.GetPage "/blog/mybundle" }}+{{ with $mybundle }}MYBUNDLE: {{ .Title }}|Path: {{ path.Join .File.Path }}|FilePath: {{ path.Join .File.FileInfo.Meta.PathFile }}|{{ end }}+
`, "_default/_markup/render-link.html", `
{{ $link := .Destination }} {{ $isRemote := strings.HasPrefix $link "http" }}@@ -640,6 +684,7 @@
/README.md|Path: _index.md|FilePath: README.md
Readme Content.
MYPAGE: My Page|Path: blog/mypage.md|FilePath: mycontent/mypage.md|
+MYBUNDLE: My Bundle|Path: blog/mybundle/index.md|FilePath: mycontent/mybundle/index.md|
`)
b.AssertFileContent("public/blog/mypage/index.html", `<a href="https://example.com/blog/mybundle/">Relative Link From Page</a>