ref: 32471b57bde51c55a15dbf1db75d6e5f7232c347
dir: /hugolib/pages_capture.go/
// Copyright 2019 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package hugolib
import (
"context"
"fmt"
"os"
pth "path"
"path/filepath"
"reflect"
"github.com/gohugoio/hugo/common/maps"
"github.com/gohugoio/hugo/parser/pageparser"
"github.com/gohugoio/hugo/hugofs/files"
"github.com/gohugoio/hugo/source"
"github.com/gohugoio/hugo/common/loggers"
"github.com/gohugoio/hugo/hugofs"
"github.com/spf13/afero"
)
const (
walkIsRootFileMetaKey = "walkIsRootFileMetaKey"
)
func newPagesCollector(
sp *source.SourceSpec,
contentMap *pageMaps,
logger loggers.Logger,
contentTracker *contentChangeMap,
proc pagesCollectorProcessorProvider, filenames ...string) *pagesCollector {
return &pagesCollector{
fs: sp.SourceFs,
contentMap: contentMap,
proc: proc,
sp: sp,
logger: logger,
filenames: filenames,
tracker: contentTracker,
}
}
type contentDirKey struct {
dirname string
filename string
tp bundleDirType
}
type fileinfoBundle struct {
header hugofs.FileMetaInfo
resources []hugofs.FileMetaInfo
}
func (b *fileinfoBundle) containsResource(name string) bool {
for _, r := range b.resources {
if r.Name() == name {
return true
}
}
return false
}
type pageBundles map[string]*fileinfoBundle
type pagesCollector struct {
sp *source.SourceSpec
fs afero.Fs
logger loggers.Logger
contentMap *pageMaps
// Ordered list (bundle headers first) used in partial builds.
filenames []string
// Content files tracker used in partial builds.
tracker *contentChangeMap
proc pagesCollectorProcessorProvider
}
// isCascadingEdit returns whether the dir represents a cascading edit.
// That is, if a front matter cascade section is removed, added or edited.
// If this is the case we must re-evaluate its descendants.
func (c *pagesCollector) isCascadingEdit(dir contentDirKey) (bool, string) {
// This is either a section or a taxonomy node. Find it.
prefix := cleanTreeKey(dir.dirname)
section := "/"
var isCascade bool
c.contentMap.walkBranchesPrefix(prefix, func(s string, n *contentNode) bool {
if n.fi == nil || dir.filename != n.fi.Meta().Filename() {
return false
}
f, err := n.fi.Meta().Open()
if err != nil {
// File may have been removed, assume a cascading edit.
// Some false positives is not too bad.
isCascade = true
return true
}
pf, err := pageparser.ParseFrontMatterAndContent(f)
f.Close()
if err != nil {
isCascade = true
return true
}
if n.p == nil || n.p.bucket == nil {
return true
}
section = s
maps.ToLower(pf.FrontMatter)
cascade1, ok := pf.FrontMatter["cascade"]
hasCascade := n.p.bucket.cascade != nil && len(n.p.bucket.cascade) > 0
if !ok {
isCascade = hasCascade
return true
}
if !hasCascade {
isCascade = true
return true
}
for _, v := range n.p.bucket.cascade {
isCascade = !reflect.DeepEqual(cascade1, v)
if isCascade {
break
}
}
return true
})
return isCascade, section
}
// Collect.
func (c *pagesCollector) Collect() (collectErr error) {
c.proc.Start(context.Background())
defer func() {
err := c.proc.Wait()
if collectErr == nil {
collectErr = err
}
}()
if len(c.filenames) == 0 {
// Collect everything.
collectErr = c.collectDir("", false, nil)
} else {
for _, pm := range c.contentMap.pmaps {
pm.cfg.isRebuild = true
}
dirs := make(map[contentDirKey]bool)
for _, filename := range c.filenames {
dir, btype := c.tracker.resolveAndRemove(filename)
dirs[contentDirKey{dir, filename, btype}] = true
}
for dir := range dirs {
for _, pm := range c.contentMap.pmaps {
pm.s.ResourceSpec.DeleteBySubstring(dir.dirname)
}
switch dir.tp {
case bundleLeaf:
collectErr = c.collectDir(dir.dirname, true, nil)
case bundleBranch:
isCascading, section := c.isCascadingEdit(dir)
if isCascading {
c.contentMap.deleteSection(section)
}
collectErr = c.collectDir(dir.dirname, !isCascading, nil)
default:
// We always start from a directory.
collectErr = c.collectDir(dir.dirname, true, func(fim hugofs.FileMetaInfo) bool {
return dir.filename == fim.Meta().Filename()
})
}
if collectErr != nil {
break
}
}
}
return
}
func (c *pagesCollector) isBundleHeader(fi hugofs.FileMetaInfo) bool {
class := fi.Meta().Classifier()
return class == files.ContentClassLeaf || class == files.ContentClassBranch
}
func (c *pagesCollector) getLang(fi hugofs.FileMetaInfo) string {
lang := fi.Meta().Lang()
if lang != "" {
return lang
}
return c.sp.DefaultContentLanguage
}
func (c *pagesCollector) addToBundle(info hugofs.FileMetaInfo, btyp bundleDirType, bundles pageBundles) error {
getBundle := func(lang string) *fileinfoBundle {
return bundles[lang]
}
cloneBundle := func(lang string) *fileinfoBundle {
// Every bundled content file needs a content file header.
// Use the default content language if found, else just
// pick one.
var (
source *fileinfoBundle
found bool
)
source, found = bundles[c.sp.DefaultContentLanguage]
if !found {
for _, b := range bundles {
source = b
break
}
}
if source == nil {
panic(fmt.Sprintf("no source found, %d", len(bundles)))
}
clone := c.cloneFileInfo(source.header)
clone.Meta()["lang"] = lang
return &fileinfoBundle{
header: clone,
}
}
lang := c.getLang(info)
bundle := getBundle(lang)
isBundleHeader := c.isBundleHeader(info)
if bundle != nil && isBundleHeader {
// index.md file inside a bundle, see issue 6208.
info.Meta()["classifier"] = files.ContentClassContent
isBundleHeader = false
}
classifier := info.Meta().Classifier()
isContent := classifier == files.ContentClassContent
if bundle == nil {
if isBundleHeader {
bundle = &fileinfoBundle{header: info}
bundles[lang] = bundle
} else {
if btyp == bundleBranch {
// No special logic for branch bundles.
// Every language needs its own _index.md file.
// Also, we only clone bundle headers for lonsesome, bundled,
// content files.
return c.handleFiles(info)
}
if isContent {
bundle = cloneBundle(lang)
bundles[lang] = bundle
}
}
}
if !isBundleHeader && bundle != nil {
bundle.resources = append(bundle.resources, info)
}
if classifier == files.ContentClassFile {
translations := info.Meta().Translations()
for lang, b := range bundles {
if !stringSliceContains(lang, translations...) && !b.containsResource(info.Name()) {
// Clone and add it to the bundle.
clone := c.cloneFileInfo(info)
clone.Meta()["lang"] = lang
b.resources = append(b.resources, clone)
}
}
}
return nil
}
func (c *pagesCollector) cloneFileInfo(fi hugofs.FileMetaInfo) hugofs.FileMetaInfo {
cm := hugofs.FileMeta{}
meta := fi.Meta()
if meta == nil {
panic(fmt.Sprintf("not meta: %v", fi.Name()))
}
for k, v := range meta {
cm[k] = v
}
return hugofs.NewFileMetaInfo(fi, cm)
}
func (c *pagesCollector) collectDir(dirname string, partial bool, inFilter func(fim hugofs.FileMetaInfo) bool) error {
fi, err := c.fs.Stat(dirname)
if err != nil {
if os.IsNotExist(err) {
// May have been deleted.
return nil
}
return err
}
handleDir := func(
btype bundleDirType,
dir hugofs.FileMetaInfo,
path string,
readdir []hugofs.FileMetaInfo) error {
if btype > bundleNot && c.tracker != nil {
c.tracker.add(path, btype)
}
if btype == bundleBranch {
if err := c.handleBundleBranch(readdir); err != nil {
return err
}
// A branch bundle is only this directory level, so keep walking.
return nil
} else if btype == bundleLeaf {
if err := c.handleBundleLeaf(dir, path, readdir); err != nil {
return err
}
return nil
}
if err := c.handleFiles(readdir...); err != nil {
return err
}
return nil
}
filter := func(fim hugofs.FileMetaInfo) bool {
if fim.Meta().SkipDir() {
return false
}
if c.sp.IgnoreFile(fim.Meta().Filename()) {
return false
}
if inFilter != nil {
return inFilter(fim)
}
return true
}
preHook := func(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) ([]hugofs.FileMetaInfo, error) {
var btype bundleDirType
filtered := readdir[:0]
for _, fi := range readdir {
if filter(fi) {
filtered = append(filtered, fi)
if c.tracker != nil {
// Track symlinks.
c.tracker.addSymbolicLinkMapping(fi)
}
}
}
walkRoot := dir.Meta().GetBool(walkIsRootFileMetaKey)
readdir = filtered
// We merge language directories, so there can be duplicates, but they
// will be ordered, most important first.
var duplicates []int
seen := make(map[string]bool)
for i, fi := range readdir {
if fi.IsDir() {
continue
}
meta := fi.Meta()
if walkRoot {
meta[walkIsRootFileMetaKey] = true
}
class := meta.Classifier()
translationBase := meta.TranslationBaseNameWithExt()
key := pth.Join(meta.Lang(), translationBase)
if seen[key] {
duplicates = append(duplicates, i)
continue
}
seen[key] = true
var thisBtype bundleDirType
switch class {
case files.ContentClassLeaf:
thisBtype = bundleLeaf
case files.ContentClassBranch:
thisBtype = bundleBranch
}
// Folders with both index.md and _index.md type of files have
// undefined behaviour and can never work.
// The branch variant will win because of sort order, but log
// a warning about it.
if thisBtype > bundleNot && btype > bundleNot && thisBtype != btype {
c.logger.Warnf("Content directory %q have both index.* and _index.* files, pick one.", dir.Meta().Filename())
// Reclassify it so it will be handled as a content file inside the
// section, which is in line with the <= 0.55 behaviour.
meta["classifier"] = files.ContentClassContent
} else if thisBtype > bundleNot {
btype = thisBtype
}
}
if len(duplicates) > 0 {
for i := len(duplicates) - 1; i >= 0; i-- {
idx := duplicates[i]
readdir = append(readdir[:idx], readdir[idx+1:]...)
}
}
err := handleDir(btype, dir, path, readdir)
if err != nil {
return nil, err
}
if btype == bundleLeaf || partial {
return nil, filepath.SkipDir
}
// Keep walking.
return readdir, nil
}
var postHook hugofs.WalkHook
if c.tracker != nil {
postHook = func(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) ([]hugofs.FileMetaInfo, error) {
if c.tracker == nil {
// Nothing to do.
return readdir, nil
}
return readdir, nil
}
}
wfn := func(path string, info hugofs.FileMetaInfo, err error) error {
if err != nil {
return err
}
return nil
}
fim := fi.(hugofs.FileMetaInfo)
// Make sure the pages in this directory gets re-rendered,
// even in fast render mode.
fim.Meta()[walkIsRootFileMetaKey] = true
w := hugofs.NewWalkway(hugofs.WalkwayConfig{
Fs: c.fs,
Logger: c.logger,
Root: dirname,
Info: fim,
HookPre: preHook,
HookPost: postHook,
WalkFn: wfn})
return w.Walk()
}
func (c *pagesCollector) handleBundleBranch(readdir []hugofs.FileMetaInfo) error {
// Maps bundles to its language.
bundles := pageBundles{}
var contentFiles []hugofs.FileMetaInfo
for _, fim := range readdir {
if fim.IsDir() {
continue
}
meta := fim.Meta()
switch meta.Classifier() {
case files.ContentClassContent:
contentFiles = append(contentFiles, fim)
default:
if err := c.addToBundle(fim, bundleBranch, bundles); err != nil {
return err
}
}
}
// Make sure the section is created before its pages.
if err := c.proc.Process(bundles); err != nil {
return err
}
return c.handleFiles(contentFiles...)
}
func (c *pagesCollector) handleBundleLeaf(dir hugofs.FileMetaInfo, path string, readdir []hugofs.FileMetaInfo) error {
// Maps bundles to its language.
bundles := pageBundles{}
walk := func(path string, info hugofs.FileMetaInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
return c.addToBundle(info, bundleLeaf, bundles)
}
// Start a new walker from the given path.
w := hugofs.NewWalkway(hugofs.WalkwayConfig{
Root: path,
Fs: c.fs,
Logger: c.logger,
Info: dir,
DirEntries: readdir,
WalkFn: walk})
if err := w.Walk(); err != nil {
return err
}
return c.proc.Process(bundles)
}
func (c *pagesCollector) handleFiles(fis ...hugofs.FileMetaInfo) error {
for _, fi := range fis {
if fi.IsDir() {
continue
}
if err := c.proc.Process(fi); err != nil {
return err
}
}
return nil
}
func stringSliceContains(k string, values ...string) bool {
for _, v := range values {
if k == v {
return true
}
}
return false
}