ref: 4d7fa9f114c62ae2ec12257203ed21b0e4d69a04
dir: /hugofs/rootmapping_fs.go/
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hugofs
import (
"fmt"
"os"
"path/filepath"
"strings"
"github.com/gohugoio/hugo/hugofs/files"
"github.com/pkg/errors"
radix "github.com/armon/go-radix"
"github.com/spf13/afero"
)
var (
filepathSeparator = string(filepath.Separator)
)
// NewRootMappingFs creates a new RootMappingFs on top of the provided with
// root mappings with some optional metadata about the root.
// Note that From represents a virtual root that maps to the actual filename in To.
func NewRootMappingFs(fs afero.Fs, rms ...RootMapping) (*RootMappingFs, error) {
rootMapToReal := radix.New()
var virtualRoots []RootMapping
for _, rm := range rms {
(&rm).clean()
fromBase := files.ResolveComponentFolder(rm.From)
if fromBase == "" {
panic("unrecognised component folder in" + rm.From)
}
if len(rm.To) < 2 {
panic(fmt.Sprintf("invalid root mapping; from/to: %s/%s", rm.From, rm.To))
}
fi, err := fs.Stat(rm.To)
if err != nil {
if os.IsNotExist(err) {
continue
}
return nil, err
}
// Extract "blog" from "content/blog"
rm.path = strings.TrimPrefix(strings.TrimPrefix(rm.From, fromBase), filepathSeparator)
if rm.Meta == nil {
rm.Meta = make(FileMeta)
}
rm.Meta[metaKeyBaseDir] = rm.ToBasedir
rm.Meta[metaKeyMountRoot] = rm.path
rm.Meta[metaKeyModule] = rm.Module
meta := copyFileMeta(rm.Meta)
if !fi.IsDir() {
_, name := filepath.Split(rm.From)
meta[metaKeyName] = name
}
rm.fi = NewFileMetaInfo(fi, meta)
key := filepathSeparator + rm.From
var mappings []RootMapping
v, found := rootMapToReal.Get(key)
if found {
// There may be more than one language pointing to the same root.
mappings = v.([]RootMapping)
}
mappings = append(mappings, rm)
rootMapToReal.Insert(key, mappings)
virtualRoots = append(virtualRoots, rm)
}
rootMapToReal.Insert(filepathSeparator, virtualRoots)
rfs := &RootMappingFs{
Fs: fs,
rootMapToReal: rootMapToReal,
}
return rfs, nil
}
func newRootMappingFsFromFromTo(
baseDir string,
fs afero.Fs,
fromTo ...string,
) (*RootMappingFs, error) {
rms := make([]RootMapping, len(fromTo)/2)
for i, j := 0, 0; j < len(fromTo); i, j = i+1, j+2 {
rms[i] = RootMapping{
From: fromTo[j],
To: fromTo[j+1],
ToBasedir: baseDir,
}
}
return NewRootMappingFs(fs, rms...)
}
// RootMapping describes a virtual file or directory mount.
type RootMapping struct {
From string // The virtual mount.
To string // The source directory or file.
ToBasedir string // The base of To. May be empty if an absolute path was provided.
Module string // The module path/ID.
Meta FileMeta // File metadata (lang etc.)
fi FileMetaInfo
path string // The virtual mount point, e.g. "blog".
}
type keyRootMappings struct {
key string
roots []RootMapping
}
func (rm *RootMapping) clean() {
rm.From = strings.Trim(filepath.Clean(rm.From), filepathSeparator)
rm.To = filepath.Clean(rm.To)
}
func (r RootMapping) filename(name string) string {
if name == "" {
return r.To
}
return filepath.Join(r.To, strings.TrimPrefix(name, r.From))
}
// A RootMappingFs maps several roots into one. Note that the root of this filesystem
// is directories only, and they will be returned in Readdir and Readdirnames
// in the order given.
type RootMappingFs struct {
afero.Fs
rootMapToReal *radix.Tree
}
func (fs *RootMappingFs) Dirs(base string) ([]FileMetaInfo, error) {
base = filepathSeparator + fs.cleanName(base)
roots := fs.getRootsWithPrefix(base)
if roots == nil {
return nil, nil
}
fss := make([]FileMetaInfo, len(roots))
for i, r := range roots {
bfs := afero.NewBasePathFs(fs.Fs, r.To)
bfs = decoratePath(bfs, func(name string) string {
p := strings.TrimPrefix(name, r.To)
if r.path != "" {
// Make sure it's mounted to a any sub path, e.g. blog
p = filepath.Join(r.path, p)
}
p = strings.TrimLeft(p, filepathSeparator)
return p
})
fs := decorateDirs(bfs, r.Meta)
fi, err := fs.Stat("")
if err != nil {
return nil, errors.Wrap(err, "RootMappingFs.Dirs")
}
if !fi.IsDir() {
mergeFileMeta(r.Meta, fi.(FileMetaInfo).Meta())
}
fss[i] = fi.(FileMetaInfo)
}
return fss, nil
}
// Filter creates a copy of this filesystem with only mappings matching a filter.
func (fs RootMappingFs) Filter(f func(m RootMapping) bool) *RootMappingFs {
rootMapToReal := radix.New()
fs.rootMapToReal.Walk(func(b string, v interface{}) bool {
rms := v.([]RootMapping)
var nrms []RootMapping
for _, rm := range rms {
if f(rm) {
nrms = append(nrms, rm)
}
}
if len(nrms) != 0 {
rootMapToReal.Insert(b, nrms)
}
return false
})
fs.rootMapToReal = rootMapToReal
return &fs
}
// LstatIfPossible returns the os.FileInfo structure describing a given file.
func (fs *RootMappingFs) LstatIfPossible(name string) (os.FileInfo, bool, error) {
fis, err := fs.doLstat(name)
if err != nil {
return nil, false, err
}
return fis[0], false, nil
}
// Open opens the named file for reading.
func (fs *RootMappingFs) Open(name string) (afero.File, error) {
fis, err := fs.doLstat(name)
if err != nil {
return nil, err
}
return fs.newUnionFile(fis...)
}
// Stat returns the os.FileInfo structure describing a given file. If there is
// an error, it will be of type *os.PathError.
func (fs *RootMappingFs) Stat(name string) (os.FileInfo, error) {
fi, _, err := fs.LstatIfPossible(name)
return fi, err
}
func (fs *RootMappingFs) hasPrefix(prefix string) bool {
hasPrefix := false
fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool {
hasPrefix = true
return true
})
return hasPrefix
}
func (fs *RootMappingFs) getRoot(key string) []RootMapping {
v, found := fs.rootMapToReal.Get(key)
if !found {
return nil
}
return v.([]RootMapping)
}
func (fs *RootMappingFs) getRoots(key string) (string, []RootMapping) {
s, v, found := fs.rootMapToReal.LongestPrefix(key)
if !found || (s == filepathSeparator && key != filepathSeparator) {
return "", nil
}
return s, v.([]RootMapping)
}
func (fs *RootMappingFs) debug() {
fmt.Println("debug():")
fs.rootMapToReal.Walk(func(s string, v interface{}) bool {
fmt.Println("Key", s)
return false
})
}
func (fs *RootMappingFs) getRootsWithPrefix(prefix string) []RootMapping {
var roots []RootMapping
fs.rootMapToReal.WalkPrefix(prefix, func(b string, v interface{}) bool {
roots = append(roots, v.([]RootMapping)...)
return false
})
return roots
}
func (fs *RootMappingFs) getAncestors(prefix string) []keyRootMappings {
var roots []keyRootMappings
fs.rootMapToReal.WalkPath(prefix, func(s string, v interface{}) bool {
if strings.HasPrefix(prefix, s+filepathSeparator) {
roots = append(roots, keyRootMappings{
key: s,
roots: v.([]RootMapping),
})
}
return false
})
return roots
}
func (fs *RootMappingFs) newUnionFile(fis ...FileMetaInfo) (afero.File, error) {
meta := fis[0].Meta()
f, err := meta.Open()
if err != nil {
return nil, err
}
if len(fis) == 1 {
return f, nil
}
rf := &rootMappingFile{File: f, fs: fs, name: meta.Name(), meta: meta}
if len(fis) == 1 {
return rf, err
}
next, err := fs.newUnionFile(fis[1:]...)
if err != nil {
return nil, err
}
uf := &afero.UnionFile{Base: rf, Layer: next}
uf.Merger = func(lofi, bofi []os.FileInfo) ([]os.FileInfo, error) {
// Ignore duplicate directory entries
seen := make(map[string]bool)
var result []os.FileInfo
for _, fis := range [][]os.FileInfo{bofi, lofi} {
for _, fi := range fis {
if fi.IsDir() && seen[fi.Name()] {
continue
}
if fi.IsDir() {
seen[fi.Name()] = true
}
result = append(result, fi)
}
}
return result, nil
}
return uf, nil
}
func (fs *RootMappingFs) cleanName(name string) string {
return strings.Trim(filepath.Clean(name), filepathSeparator)
}
func (fs *RootMappingFs) collectDirEntries(prefix string) ([]os.FileInfo, error) {
prefix = filepathSeparator + fs.cleanName(prefix)
var fis []os.FileInfo
seen := make(map[string]bool) // Prevent duplicate directories
level := strings.Count(prefix, filepathSeparator)
collectDir := func(rm RootMapping, fi FileMetaInfo) error {
f, err := fi.Meta().Open()
if err != nil {
return err
}
direntries, err := f.Readdir(-1)
if err != nil {
f.Close()
return err
}
for _, fi := range direntries {
meta := fi.(FileMetaInfo).Meta()
mergeFileMeta(rm.Meta, meta)
if fi.IsDir() {
name := fi.Name()
if seen[name] {
continue
}
seen[name] = true
opener := func() (afero.File, error) {
return fs.Open(filepath.Join(rm.From, name))
}
fi = newDirNameOnlyFileInfo(name, meta, opener)
}
fis = append(fis, fi)
}
f.Close()
return nil
}
// First add any real files/directories.
rms := fs.getRoot(prefix)
for _, rm := range rms {
if err := collectDir(rm, rm.fi); err != nil {
return nil, err
}
}
// Next add any file mounts inside the given directory.
prefixInside := prefix + filepathSeparator
fs.rootMapToReal.WalkPrefix(prefixInside, func(s string, v interface{}) bool {
if (strings.Count(s, filepathSeparator) - level) != 1 {
// This directory is not part of the current, but we
// need to include the first name part to make it
// navigable.
path := strings.TrimPrefix(s, prefixInside)
parts := strings.Split(path, filepathSeparator)
name := parts[0]
if seen[name] {
return false
}
seen[name] = true
opener := func() (afero.File, error) {
return fs.Open(path)
}
fi := newDirNameOnlyFileInfo(name, nil, opener)
fis = append(fis, fi)
return false
}
rms := v.([]RootMapping)
for _, rm := range rms {
if !rm.fi.IsDir() {
// A single file mount
fis = append(fis, rm.fi)
continue
}
name := filepath.Base(rm.From)
if seen[name] {
continue
}
seen[name] = true
opener := func() (afero.File, error) {
return fs.Open(rm.From)
}
fi := newDirNameOnlyFileInfo(name, rm.Meta, opener)
fis = append(fis, fi)
}
return false
})
// Finally add any ancestor dirs with files in this directory.
ancestors := fs.getAncestors(prefix)
for _, root := range ancestors {
subdir := strings.TrimPrefix(prefix, root.key)
for _, rm := range root.roots {
if rm.fi.IsDir() {
fi, err := rm.fi.Meta().JoinStat(subdir)
if err == nil {
if err := collectDir(rm, fi); err != nil {
return nil, err
}
}
}
}
}
return fis, nil
}
func (fs *RootMappingFs) doLstat(name string) ([]FileMetaInfo, error) {
name = fs.cleanName(name)
key := filepathSeparator + name
roots := fs.getRoot(key)
if roots == nil {
if fs.hasPrefix(key) {
// We have directories mounted below this.
// Make it look like a directory.
return []FileMetaInfo{newDirNameOnlyFileInfo(name, nil, fs.virtualDirOpener(name))}, nil
}
// Find any real files or directories with this key.
_, roots := fs.getRoots(key)
if roots == nil {
return nil, os.ErrNotExist
}
var err error
var fis []FileMetaInfo
for _, rm := range roots {
var fi FileMetaInfo
fi, _, err = fs.statRoot(rm, name)
if err == nil {
fis = append(fis, fi)
}
}
if fis != nil {
return fis, nil
}
if err == nil {
err = os.ErrNotExist
}
return nil, err
}
fileCount := 0
for _, root := range roots {
if !root.fi.IsDir() {
fileCount++
}
if fileCount > 1 {
break
}
}
if fileCount == 0 {
// Dir only.
return []FileMetaInfo{newDirNameOnlyFileInfo(name, roots[0].Meta, fs.virtualDirOpener(name))}, nil
}
if fileCount > 1 {
// Not supported by this filesystem.
return nil, errors.Errorf("found multiple files with name %q, use .Readdir or the source filesystem directly", name)
}
return []FileMetaInfo{roots[0].fi}, nil
}
func (fs *RootMappingFs) statRoot(root RootMapping, name string) (FileMetaInfo, bool, error) {
filename := root.filename(name)
fi, b, err := lstatIfPossible(fs.Fs, filename)
if err != nil {
return nil, b, err
}
var opener func() (afero.File, error)
if fi.IsDir() {
// Make sure metadata gets applied in Readdir.
opener = fs.realDirOpener(filename, root.Meta)
} else {
// Opens the real file directly.
opener = func() (afero.File, error) {
return fs.Fs.Open(filename)
}
}
return decorateFileInfo(fi, fs.Fs, opener, "", "", root.Meta), b, nil
}
func (fs *RootMappingFs) virtualDirOpener(name string) func() (afero.File, error) {
return func() (afero.File, error) { return &rootMappingFile{name: name, fs: fs}, nil }
}
func (fs *RootMappingFs) realDirOpener(name string, meta FileMeta) func() (afero.File, error) {
return func() (afero.File, error) {
f, err := fs.Fs.Open(name)
if err != nil {
return nil, err
}
return &rootMappingFile{name: name, meta: meta, fs: fs, File: f}, nil
}
}
type rootMappingFile struct {
afero.File
fs *RootMappingFs
name string
meta FileMeta
}
func (f *rootMappingFile) Close() error {
if f.File == nil {
return nil
}
return f.File.Close()
}
func (f *rootMappingFile) Name() string {
return f.name
}
func (f *rootMappingFile) Readdir(count int) ([]os.FileInfo, error) {
if f.File != nil {
fis, err := f.File.Readdir(count)
if err != nil {
return nil, err
}
for i, fi := range fis {
fis[i] = decorateFileInfo(fi, f.fs, nil, "", "", f.meta)
}
return fis, nil
}
return f.fs.collectDirEntries(f.name)
}
func (f *rootMappingFile) Readdirnames(count int) ([]string, error) {
dirs, err := f.Readdir(count)
if err != nil {
return nil, err
}
return fileInfosToNames(dirs), nil
}