ref: b64617fe4f90da030bcf4a9c5a4913393ce96b14
parent: 17ca8f0c4c636752fb9da2ad551679275dc03dd3
author: Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com>
date: Mon Aug 12 12:43:37 EDT 2019
Add resources.Match and resources.GetMatch Fix #6190
--- /dev/null
+++ b/hugofs/glob.go
@@ -1,0 +1,85 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+ "errors"
+ "path/filepath"
+ "strings"
+
+ "github.com/gohugoio/hugo/hugofs/glob"
+
+ "github.com/spf13/afero"
+)
+
+// Glob walks the fs and passes all matches to the handle func.
+// The handle func can return true to signal a stop.
+func Glob(fs afero.Fs, pattern string, handle func(fi FileMetaInfo) (bool, error)) error {+ pattern = glob.NormalizePath(pattern)
+ if pattern == "" {+ return nil
+ }
+
+ g, err := glob.GetGlob(pattern)
+ if err != nil {+ return nil
+ }
+
+ hasSuperAsterisk := strings.Contains(pattern, "**")
+ levels := strings.Count(pattern, "/")
+ root := glob.ResolveRootDir(pattern)
+
+ // Signals that we're done.
+ done := errors.New("done")+
+ wfn := func(p string, info FileMetaInfo, err error) error {+ p = glob.NormalizePath(p)
+ if info.IsDir() {+ if !hasSuperAsterisk {+ // Avoid walking to the bottom if we can avoid it.
+ if p != "" && strings.Count(p, "/") >= levels {+ return filepath.SkipDir
+ }
+ }
+ return nil
+ }
+
+ if g.Match(p) {+ d, err := handle(info)
+ if err != nil {+ return err
+ }
+ if d {+ return done
+ }
+ }
+
+ return nil
+ }
+
+ w := NewWalkway(WalkwayConfig{+ Root: root,
+ Fs: fs,
+ WalkFn: wfn,
+ })
+
+ err = w.Walk()
+
+ if err != done {+ return err
+ }
+
+ return nil
+
+}
--- /dev/null
+++ b/hugofs/glob/glob.go
@@ -1,0 +1,81 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package glob
+
+import (
+ "path"
+ "path/filepath"
+ "strings"
+ "sync"
+
+ "github.com/gobwas/glob"
+ "github.com/gobwas/glob/syntax"
+)
+
+var (
+ globCache = make(map[string]glob.Glob)
+ globMu sync.RWMutex
+)
+
+func GetGlob(pattern string) (glob.Glob, error) {+ var g glob.Glob
+
+ globMu.RLock()
+ g, found := globCache[pattern]
+ globMu.RUnlock()
+ if !found {+ var err error
+ g, err = glob.Compile(strings.ToLower(pattern), '/')
+ if err != nil {+ return nil, err
+ }
+
+ globMu.Lock()
+ globCache[pattern] = g
+ globMu.Unlock()
+ }
+
+ return g, nil
+
+}
+
+func NormalizePath(p string) string {+ return strings.Trim(filepath.ToSlash(strings.ToLower(p)), "/.")
+}
+
+// ResolveRootDir takes a normalized path on the form "assets/**.json" and
+// determines any root dir, i.e. any start path without any wildcards.
+func ResolveRootDir(p string) string {+ parts := strings.Split(path.Dir(p), "/")
+ var roots []string
+ for _, part := range parts {+ isSpecial := false
+ for i := 0; i < len(part); i++ {+ if syntax.Special(part[i]) {+ isSpecial = true
+ break
+ }
+ }
+ if isSpecial {+ break
+ }
+ roots = append(roots, part)
+ }
+
+ if len(roots) == 0 {+ return ""
+ }
+
+ return strings.Join(roots, "/")
+}
--- /dev/null
+++ b/hugofs/glob/glob_test.go
@@ -1,0 +1,63 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package glob
+
+import (
+ "path/filepath"
+ "testing"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestResolveRootDir(t *testing.T) {+ c := qt.New(t)
+
+ for _, test := range []struct {+ in string
+ expect string
+ }{+ {"data/foo.json", "data"},+ {"a/b/**/foo.json", "a/b"},+ {"dat?a/foo.json", ""},+ {"a/b[a-c]/foo.json", "a"},+ } {+
+ c.Assert(ResolveRootDir(test.in), qt.Equals, test.expect)
+ }
+}
+
+func TestNormalizePath(t *testing.T) {+ c := qt.New(t)
+
+ for _, test := range []struct {+ in string
+ expect string
+ }{+ {filepath.FromSlash("data/FOO.json"), "data/foo.json"},+ {filepath.FromSlash("/data/FOO.json"), "data/foo.json"},+ {filepath.FromSlash("./FOO.json"), "foo.json"},+ {"//", ""},+ } {+
+ c.Assert(NormalizePath(test.in), qt.Equals, test.expect)
+ }
+}
+
+func TestGetGlob(t *testing.T) {+ c := qt.New(t)
+ g, err := GetGlob("**.JSON")+ c.Assert(err, qt.IsNil)
+ c.Assert(g.Match("data/my.json"), qt.Equals, true)+
+}
--- /dev/null
+++ b/hugofs/glob_test.go
@@ -1,0 +1,61 @@
+// Copyright 2019 The Hugo Authors. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package hugofs
+
+import (
+ "path/filepath"
+ "testing"
+
+ "github.com/spf13/afero"
+
+ qt "github.com/frankban/quicktest"
+)
+
+func TestGlob(t *testing.T) {+ c := qt.New(t)
+
+ fs := NewBaseFileDecorator(afero.NewMemMapFs())
+
+ create := func(filename string) {+ err := afero.WriteFile(fs, filepath.FromSlash(filename), []byte("content "+filename), 0777)+ c.Assert(err, qt.IsNil)
+ }
+
+ collect := func(pattern string) []string {+ var paths []string
+ h := func(fi FileMetaInfo) (bool, error) {+ paths = append(paths, fi.Meta().Path())
+ return false, nil
+ }
+ err := Glob(fs, pattern, h)
+ c.Assert(err, qt.IsNil)
+ return paths
+ }
+
+ create("root.json")+ create("jsonfiles/d1.json")+ create("jsonfiles/d2.json")+ create("jsonfiles/sub/d3.json")+ create("jsonfiles/d1.xml")+ create("a/b/c/e/f.json")+
+ c.Assert(collect("**.json"), qt.HasLen, 5)+ c.Assert(collect("**"), qt.HasLen, 6)+ c.Assert(collect(""), qt.HasLen, 0)+ c.Assert(collect("jsonfiles/*.json"), qt.HasLen, 2)+ c.Assert(collect("*.json"), qt.HasLen, 1)+ c.Assert(collect("**.xml"), qt.HasLen, 1)+ c.Assert(collect(filepath.FromSlash("/jsonfiles/*.json")), qt.HasLen, 2)+
+}
--- a/hugolib/resource_chain_test.go
+++ b/hugolib/resource_chain_test.go
@@ -502,3 +502,33 @@
b.AssertFileContent("public/text/pipes.txt", "Hugo Pipes")}
+
+func TestResourcesMatch(t *testing.T) {+ t.Parallel()
+
+ b := newTestSitesBuilder(t)
+
+ b.WithContent("page.md", "")+
+ b.WithSourceFile(
+ "assets/jsons/data1.json", "json1 content",
+ "assets/jsons/data2.json", "json2 content",
+ "assets/jsons/data3.xml", "xml content",
+ )
+
+ b.WithTemplates("index.html", `+{{ $jsons := (resources.Match "jsons/*.json") }}+{{ $json := (resources.GetMatch "jsons/*.json") }}+{{ printf "JSONS: %d" (len $jsons) }}+JSON: {{ $json.RelPermalink }}: {{ $json.Content }}+{{ range $jsons }}+{{- .RelPermalink }}: {{ .Content }}+{{ end }}+`)
+
+ b.Build(BuildCfg{})+
+ b.AssertFileContent("public/index.html",+ "JSON: /jsons/data1.json: json1 content",
+ "JSONS: 2", "/jsons/data1.json: json1 content")
+}
--- a/resources/internal/glob.go
+++ /dev/null
@@ -1,48 +1,0 @@
-// Copyright 2019 The Hugo Authors. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package internal
-
-import (
- "strings"
- "sync"
-
- "github.com/gobwas/glob"
-)
-
-var (
- globCache = make(map[string]glob.Glob)
- globMu sync.RWMutex
-)
-
-func GetGlob(pattern string) (glob.Glob, error) {- var g glob.Glob
-
- globMu.RLock()
- g, found := globCache[pattern]
- globMu.RUnlock()
- if !found {- var err error
- g, err = glob.Compile(strings.ToLower(pattern), '/')
- if err != nil {- return nil, err
- }
-
- globMu.Lock()
- globCache[pattern] = g
- globMu.Unlock()
- }
-
- return g, nil
-
-}
--- a/resources/resource/resources.go
+++ b/resources/resource/resources.go
@@ -17,7 +17,7 @@
"fmt"
"strings"
- "github.com/gohugoio/hugo/resources/internal"
+ "github.com/gohugoio/hugo/hugofs/glob"
)
// Resources represents a slice of resources, which can be a mix of different types.
@@ -44,7 +44,7 @@
// GetMatch finds the first Resource matching the given pattern, or nil if none found.
// See Match for a more complete explanation about the rules used.
func (r Resources) GetMatch(pattern string) Resource {- g, err := internal.GetGlob(pattern)
+ g, err := glob.GetGlob(pattern)
if err != nil {return nil
}
@@ -68,7 +68,7 @@
// path relative to the bundle root with Unix style slashes (/) and no leading slash, e.g. "images/logo.png".
// See https://github.com/gobwas/glob for the full rules set.
func (r Resources) Match(pattern string) Resources {- g, err := internal.GetGlob(pattern)
+ g, err := glob.GetGlob(pattern)
if err != nil {return nil
}
--- a/resources/resource_cache.go
+++ b/resources/resource_cache.go
@@ -37,8 +37,10 @@
rs *Spec
sync.RWMutex
- cache map[string]resource.Resource
+ // Either resource.Resource or resource.Resources.
+ cache map[string]interface{}+
fileCache *filecache.Cache
// Provides named resource locks.
@@ -61,7 +63,7 @@
return &ResourceCache{rs: rs,
fileCache: rs.FileCaches.AssetsCache(),
- cache: make(map[string]resource.Resource),
+ cache: make(map[string]interface{}),nlocker: locker.NewLocker(),
}
}
@@ -70,7 +72,7 @@
c.Lock()
defer c.Unlock()
- c.cache = make(map[string]resource.Resource)
+ c.cache = make(map[string]interface{})c.nlocker = locker.NewLocker()
}
@@ -84,7 +86,7 @@
return strings.TrimPrefix(path.Clean(key), "/")
}
-func (c *ResourceCache) get(key string) (resource.Resource, bool) {+func (c *ResourceCache) get(key string) (interface{}, bool) {c.RLock()
defer c.RUnlock()
r, found := c.cache[key]
@@ -92,6 +94,22 @@
}
func (c *ResourceCache) GetOrCreate(partition, key string, f func() (resource.Resource, error)) (resource.Resource, error) {+ r, err := c.getOrCreate(partition, key, func() (interface{}, error) { return f() })+ if r == nil || err != nil {+ return nil, err
+ }
+ return r.(resource.Resource), nil
+}
+
+func (c *ResourceCache) GetOrCreateResources(partition, key string, f func() (resource.Resources, error)) (resource.Resources, error) {+ r, err := c.getOrCreate(partition, key, func() (interface{}, error) { return f() })+ if r == nil || err != nil {+ return nil, err
+ }
+ return r.(resource.Resources), nil
+}
+
+func (c *ResourceCache) getOrCreate(partition, key string, f func() (interface{}, error)) (interface{}, error) {key = c.cleanKey(path.Join(partition, key))
// First check in-memory cache.
r, found := c.get(key)
@@ -174,7 +192,7 @@
}
-func (c *ResourceCache) set(key string, r resource.Resource) {+func (c *ResourceCache) set(key string, r interface{}) {c.Lock()
defer c.Unlock()
c.cache[key] = r
--- a/resources/resource_factories/create/create.go
+++ b/resources/resource_factories/create/create.go
@@ -16,10 +16,13 @@
package create
import (
+ "path"
"path/filepath"
- "github.com/spf13/afero"
+ "github.com/gohugoio/hugo/hugofs/glob"
+ "github.com/gohugoio/hugo/hugofs"
+
"github.com/gohugoio/hugo/common/hugio"
"github.com/gohugoio/hugo/resources"
"github.com/gohugoio/hugo/resources/resource"
@@ -36,16 +39,73 @@
return &Client{rs: rs}}
-// Get creates a new Resource by opening the given filename in the given filesystem.
-func (c *Client) Get(fs afero.Fs, filename string) (resource.Resource, error) {+// Get creates a new Resource by opening the given filename in the assets filesystem.
+func (c *Client) Get(filename string) (resource.Resource, error) {filename = filepath.Clean(filename)
return c.rs.ResourceCache.GetOrCreate(resources.ResourceKeyPartition(filename), filename, func() (resource.Resource, error) { return c.rs.New(resources.ResourceSourceDescriptor{- Fs: fs,
+ Fs: c.rs.BaseFs.Assets.Fs,
LazyPublish: true,
SourceFilename: filename})
})
+}
+
+// Match gets the resources matching the given pattern from the assets filesystem.
+func (c *Client) Match(pattern string) (resource.Resources, error) {+ return c.match(pattern, false)
+}
+
+// GetMatch gets first resource matching the given pattern from the assets filesystem.
+func (c *Client) GetMatch(pattern string) (resource.Resource, error) {+ res, err := c.match(pattern, true)
+ if err != nil || len(res) == 0 {+ return nil, err
+ }
+ return res[0], err
+}
+
+func (c *Client) match(pattern string, firstOnly bool) (resource.Resources, error) {+ var partition string
+ if firstOnly {+ partition = "__get-match"
+ } else {+ partition = "__match"
+ }
+
+ // TODO(bep) match will be improved as part of https://github.com/gohugoio/hugo/issues/6199
+ partition = path.Join(resources.CACHE_OTHER, partition)
+ key := glob.NormalizePath(pattern)
+
+ return c.rs.ResourceCache.GetOrCreateResources(partition, key, func() (resource.Resources, error) {+ var res resource.Resources
+
+ handle := func(info hugofs.FileMetaInfo) (bool, error) {+ meta := info.Meta()
+ r, err := c.rs.New(resources.ResourceSourceDescriptor{+ LazyPublish: true,
+ OpenReadSeekCloser: func() (hugio.ReadSeekCloser, error) {+ return meta.Open()
+ },
+ RelTargetFilename: meta.Path()})
+
+ if err != nil {+ return true, err
+ }
+
+ res = append(res, r)
+
+ return firstOnly, nil
+
+ }
+
+ if err := hugofs.Glob(c.rs.BaseFs.Assets.Fs, pattern, handle); err != nil {+ return nil, err
+ }
+
+ return res, nil
+
+ })
}
// FromString creates a new Resource from a string with the given relative target path.
--- a/resources/resource_metadata.go
+++ b/resources/resource_metadata.go
@@ -17,7 +17,7 @@
"fmt"
"strconv"
- "github.com/gohugoio/hugo/resources/internal"
+ "github.com/gohugoio/hugo/hugofs/glob"
"github.com/gohugoio/hugo/resources/resource"
"github.com/pkg/errors"
@@ -70,7 +70,7 @@
srcKey := strings.ToLower(cast.ToString(src))
- glob, err := internal.GetGlob(srcKey)
+ glob, err := glob.GetGlob(srcKey)
if err != nil {return errors.Wrap(err, "failed to match resource with metadata")
}
--- a/tpl/resources/resources.go
+++ b/tpl/resources/resources.go
@@ -68,7 +68,7 @@
templatesClient *templates.Client
}
-// Get locates the filename given in Hugo's filesystems: static, assets and content (in that order)
+// Get locates the filename given in Hugo's assets filesystem
// and creates a Resource object that can be used for further transformations.
func (ns *Namespace) Get(filename interface{}) (resource.Resource, error) {filenamestr, err := cast.ToStringE(filename)
@@ -78,10 +78,48 @@
filenamestr = filepath.Clean(filenamestr)
- // Resource Get'ing is currently limited to /assets to make it simpler
- // to control the behaviour of publishing and partial rebuilding.
- return ns.createClient.Get(ns.deps.BaseFs.Assets.Fs, filenamestr)
+ return ns.createClient.Get(filenamestr)
+}
+
+// GetMatch finds the first Resource matching the given pattern, or nil if none found.
+//
+// It looks for files in the assets file system.
+//
+// See Match for a more complete explanation about the rules used.
+func (ns *Namespace) GetMatch(pattern interface{}) (resource.Resource, error) {+ patternStr, err := cast.ToStringE(pattern)
+ if err != nil {+ return nil, err
+ }
+
+ return ns.createClient.GetMatch(patternStr)
+
+}
+
+// Match gets all resources matching the given base path prefix, e.g
+// "*.png" will match all png files. The "*" does not match path delimiters (/),
+// so if you organize your resources in sub-folders, you need to be explicit about it, e.g.:
+// "images/*.png". To match any PNG image anywhere in the bundle you can do "**.png", and
+// to match all PNG images below the images folder, use "images/**.jpg".
+//
+// The matching is case insensitive.
+//
+// Match matches by using the files name with path relative to the file system root
+// with Unix style slashes (/) and no leading slash, e.g. "images/logo.png".
+//
+// See https://github.com/gobwas/glob for the full rules set.
+//
+// It looks for files in the assets file system.
+//
+// See Match for a more complete explanation about the rules used.
+func (ns *Namespace) Match(pattern interface{}) (resource.Resources, error) {+ patternStr, err := cast.ToStringE(pattern)
+ if err != nil {+ return nil, err
+ }
+
+ return ns.createClient.Match(patternStr)
}
// Concat concatenates a slice of Resource objects. These resources must