From c2ddc7031138b5cf8a097987c1bf70b2882e7e70 Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Sat, 27 Dec 2014 14:40:10 +1100 Subject: [PATCH 01/12] Merge to stay up to date with current dev Feature: GetJson and GetJson in short codes or other layout files. For more details please see: http://cyrillschumacher.com/2014/12/21/dynamic-pages-with-gohugo.io/ --- commands/hugo.go | 12 +- helpers/path.go | 25 ++++ helpers/path_test.go | 28 +++++ tpl/template_resources.go | 220 +++++++++++++++++++++++++++++++++ tpl/template_resources_test.go | 181 +++++++++++++++++++++++++++ 5 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 tpl/template_resources.go create mode 100644 tpl/template_resources_test.go diff --git a/commands/hugo.go b/commands/hugo.go index 2a3f99b1eae..4d7c3610508 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -56,7 +56,7 @@ var hugoCmdV *cobra.Command //Flags that are to be added to commands. var BuildWatch, Draft, Future, UglyUrls, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, PluralizeListTitles, NoTimes bool -var Source, Destination, Theme, BaseUrl, CfgFile, LogFile, Editor string +var Source, CacheDir, Destination, Theme, BaseUrl, CfgFile, LogFile, Editor string //Execute adds all child commands to the root command HugoCmd and sets flags appropriately. func Execute() { @@ -83,6 +83,7 @@ func init() { HugoCmd.PersistentFlags().BoolVar(&DisableRSS, "disableRSS", false, "Do not build RSS files") HugoCmd.PersistentFlags().BoolVar(&DisableSitemap, "disableSitemap", false, "Do not build Sitemap file") HugoCmd.PersistentFlags().StringVarP(&Source, "source", "s", "", "filesystem path to read files relative from") + HugoCmd.PersistentFlags().StringVarP(&CacheDir, "cacheDir", "", "$TMPDIR/hugo_cache/", "filesystem path to cache directory") HugoCmd.PersistentFlags().StringVarP(&Destination, "destination", "d", "", "filesystem path to write files to") HugoCmd.PersistentFlags().StringVarP(&Theme, "theme", "t", "", "theme to use (located in /themes/THEMENAME/)") HugoCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose output") @@ -203,6 +204,15 @@ func InitializeConfig() { viper.Set("WorkingDir", dir) } + if CacheDir != "" { + if helpers.FilePathSeparator != CacheDir[len(CacheDir)-1:] { + CacheDir = CacheDir + helpers.FilePathSeparator + } + viper.Set("CacheDir", CacheDir) + } else { + viper.Set("CacheDir", helpers.GetTempDir("hugo_cache", hugofs.SourceFs)) + } + if VerboseLog || Logging || (viper.IsSet("LogFile") && viper.GetString("LogFile") != "") { if viper.IsSet("LogFile") && viper.GetString("LogFile") != "" { jww.SetLogFile(viper.GetString("LogFile")) diff --git a/helpers/path.go b/helpers/path.go index 9c3c2ba154a..f6c77f9c380 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -437,3 +437,28 @@ func WriteToDisk(inpath string, r io.Reader, fs afero.Fs) (err error) { _, err = io.Copy(file, r) return } + +// GetTempDir returns the OS default temp directory with trailing slash +// if subPath is not empty then it will be created recursively +func GetTempDir(subPath string, fs afero.Fs) string { + dir := os.TempDir() + if FilePathSeparator != dir[len(dir)-1:] { + dir = dir + FilePathSeparator + } + if subPath != "" { + dir = dir + MakePath(subPath) + + if exists, _ := Exists(dir, fs); exists { + return dir + } + + err := fs.MkdirAll(dir, 0777) // rwx, rw, r + if err != nil { + panic(err) + } + if FilePathSeparator != dir[len(dir)-1:] { + dir = dir + FilePathSeparator + } + } + return dir +} diff --git a/helpers/path_test.go b/helpers/path_test.go index bc0a547901a..9be1c9fc43e 100644 --- a/helpers/path_test.go +++ b/helpers/path_test.go @@ -647,3 +647,31 @@ func TestWriteToDisk(t *testing.T) { reader.Seek(0, 0) } } + +func TestGetTempDir(t *testing.T) { + dir := os.TempDir() + if FilePathSeparator != dir[len(dir)-1:] { + dir = dir + FilePathSeparator + } + testDir := "hugoTestFolder" + FilePathSeparator + tests := []struct { + input string + expected string + }{ + {"", dir}, + {testDir + " Foo bar ", dir + testDir + "--Foo-bar" + FilePathSeparator}, + {testDir + "Foo.Bar/foo_Bar-Foo", dir + testDir + "Foo.Bar/foo_Bar-Foo" + FilePathSeparator}, + {testDir + "fOO,bar:foo%bAR", dir + testDir + "fOObarfoobAR" + FilePathSeparator}, + {testDir + "FOo/BaR.html", dir + testDir + "FOo/BaR.html" + FilePathSeparator}, + {testDir + "трям/трям", dir + testDir + "трям/трям" + FilePathSeparator}, + {testDir + "은행", dir + testDir + "은행" + FilePathSeparator}, + {testDir + "Банковский кассир", dir + testDir + "Банковский-кассир" + FilePathSeparator}, + } + + for _, test := range tests { + output := GetTempDir(test.input, new(afero.MemMapFs)) + if output != test.expected { + t.Errorf("Expected %#v, got %#v\n", test.expected, output) + } + } +} diff --git a/tpl/template_resources.go b/tpl/template_resources.go new file mode 100644 index 00000000000..d4e105fcb87 --- /dev/null +++ b/tpl/template_resources.go @@ -0,0 +1,220 @@ +// Copyright © 2013-14 Steve Francia . +// +// Licensed under the Simple Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://opensource.org/licenses/Simple-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tpl + +import ( + "bytes" + "encoding/csv" + "encoding/json" + "errors" + "io/ioutil" + "net/http" + "net/url" + "strings" + "sync" + + "github.com/spf13/afero" + "github.com/spf13/hugo/helpers" + "github.com/spf13/hugo/hugofs" + jww "github.com/spf13/jwalterweatherman" + "github.com/spf13/viper" +) + +var remoteUrlLock = &remoteLock{m: make(map[string]*sync.Mutex)} + +type remoteLock struct { + sync.RWMutex + m map[string]*sync.Mutex +} + +// resLock locks an URL during download +func (l *remoteLock) UrlLock(url string) { + l.Lock() + if _, ok := l.m[url]; !ok { + l.m[url] = &sync.Mutex{} + } + l.Unlock() // call this Unlock before the next lock will be called. NFI why but defer doesn't work. + l.m[url].Lock() +} + +// resUnlock unlocks an URL when the download has been finished. Use only in defer calls. +func (l *remoteLock) UrlUnlock(url string) { + l.RLock() + defer l.RUnlock() + if um, ok := l.m[url]; ok { + um.Unlock() + } +} + +// getFileID returns the cache ID for a string +func getCacheFileID(id string) string { + return viper.GetString("CacheDir") + url.QueryEscape(id) +} + +// resGetCache returns the content for an ID from the file cache or an error +// if the file is not found returns nil,nil +func resGetCache(id string, fs afero.Fs) ([]byte, error) { + fID := getCacheFileID(id) + isExists, err := helpers.Exists(fID, fs) + if err != nil { + return nil, err + } + if !isExists { + return nil, nil + } + + f, err := fs.Open(fID) + if err != nil { + return nil, err + } + + return ioutil.ReadAll(f) +} + +// resWriteCache writes bytes to an ID into the file cache +func resWriteCache(id string, c []byte, fs afero.Fs) error { + fID := getCacheFileID(id) + f, err := fs.Create(fID) + if err != nil { + return err + } + n, err := f.Write(c) + if n == 0 { + return errors.New("No bytes written to file: " + fID) + } + return err +} + +// resGetRemote loads the content of a remote file. This method is thread safe. +func resGetRemote(url string, fs afero.Fs, hc *http.Client) ([]byte, error) { + + c, err := resGetCache(url, fs) + if c != nil && err == nil { + return c, nil + } + if err != nil { + return nil, err + } + + // avoid race condition with locks, block other goroutines if the current url is processing + remoteUrlLock.UrlLock(url) + defer func() { remoteUrlLock.UrlUnlock(url) }() + + // avoid multiple locks due to calling resGetCache twice + c, err = resGetCache(url, fs) + if c != nil && err == nil { + return c, nil + } + if err != nil { + return nil, err + } + + jww.INFO.Printf("Downloading: %s ...", url) + res, err := hc.Get(url) + if err != nil { + return nil, err + } + c, err = ioutil.ReadAll(res.Body) + res.Body.Close() + if err != nil { + return nil, err + } + err = resWriteCache(url, c, fs) + if err != nil { + return nil, err + } + jww.INFO.Printf("... and cached to: %s", getCacheFileID(url)) + return c, nil +} + +// resGetLocal loads the content of a local file +func resGetLocal(url string, fs afero.Fs) ([]byte, error) { + p := "" + if viper.GetString("WorkingDir") != "" { + p = viper.GetString("WorkingDir") + if helpers.FilePathSeparator != p[len(p)-1:] { + p = p + helpers.FilePathSeparator + } + } + jFile := p + url + if e, err := helpers.Exists(jFile, fs); !e { + return nil, err + } + + f, err := fs.Open(jFile) + if err != nil { + return nil, err + } + return ioutil.ReadAll(f) +} + +// resGetResource loads the content of a local or remote file +func resGetResource(url string) ([]byte, error) { + if url == "" { + return nil, nil + } + if strings.Contains(url, "://") { + return resGetRemote(url, hugofs.SourceFs, http.DefaultClient) + } + return resGetLocal(url, hugofs.SourceFs) +} + +// GetJson expects the url to a resource which can either be a local or a remote one. +// GetJson returns nil or parsed JSON to use in a short code. +func GetJson(url string) interface{} { + c, err := resGetResource(url) + if err != nil { + jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err) + return nil + } + + var v interface{} + err = json.Unmarshal(c, &v) + if err != nil { + jww.ERROR.Printf("Cannot read json from resource %s with error message %s", url, err) + return nil + } + return v +} + +// parseCsv parses bytes of csv data into a slice slice string or an error +func parseCsv(c []byte, sep string) ([][]string, error) { + if len(sep) != 1 { + return nil, errors.New("Incorrect length of csv separator: " + sep) + } + b := bytes.NewReader(c) + r := csv.NewReader(b) + rSep := []rune(sep) + r.Comma = rSep[0] + r.FieldsPerRecord = 0 + return r.ReadAll() +} + +// GetCsv expects the url to a resource which can either be a local or a remote one and the type +// of the data separator which can be comma, semi-colon, pipe, but only one character. +// GetCsv returns nil or a slice slice to use in a short code. +func GetCsv(url string, sep string) [][]string { + + c, err := resGetResource(url) + if err != nil { + jww.ERROR.Printf("Failed to get csv resource %s with error message %s", url, err) + return nil + } + d, err := parseCsv(c, sep) + if err != nil { + jww.ERROR.Printf("Failed to read csv resource %s with error message %s", url, err) + return nil + } + return d +} diff --git a/tpl/template_resources_test.go b/tpl/template_resources_test.go new file mode 100644 index 00000000000..40ea1fc4350 --- /dev/null +++ b/tpl/template_resources_test.go @@ -0,0 +1,181 @@ +// Copyright © 2013-14 Steve Francia . +// +// Licensed under the Simple Public License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://opensource.org/licenses/Simple-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package tpl + +import ( + "bytes" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/spf13/afero" + "github.com/spf13/hugo/helpers" +) + +func TestScpCache(t *testing.T) { + + tests := []struct { + path string + content []byte + }{ + {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`)}, + {"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`)}, + {"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`)}, + {"трям/трям", []byte(`T€st трям/трям Content 123`)}, + {"은행", []byte(`T€st C은행ontent 123`)}, + {"Банковский кассир", []byte(`Банковский кассир T€st Content 123`)}, + } + + fs := new(afero.MemMapFs) + + for _, test := range tests { + c, err := resGetCache(test.path, fs) + if err != nil { + t.Errorf("Error getting cache: %s", err) + } + if c != nil { + t.Errorf("There is content where there should not be anything: %s", string(c)) + } + + err = resWriteCache(test.path, test.content, fs) + if err != nil { + t.Errorf("Error writing cache: %s", err) + } + + c, err = resGetCache(test.path, fs) + if err != nil { + t.Errorf("Error getting cache after writing: %s", err) + } + if bytes.Compare(c, test.content) != 0 { + t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c)) + } + } +} + +func TestScpGetLocal(t *testing.T) { + fs := new(afero.MemMapFs) + ps := helpers.FilePathSeparator + tests := []struct { + path string + content []byte + }{ + {"testpath" + ps + "test.txt", []byte(`T€st Content 123 fOO,bar:foo%bAR`)}, + {"FOo" + ps + "BaR.html", []byte(`FOo/BaR.html T€st Content 123`)}, + {"трям" + ps + "трям", []byte(`T€st трям/трям Content 123`)}, + {"은행", []byte(`T€st C은행ontent 123`)}, + {"Банковский кассир", []byte(`Банковский кассир T€st Content 123`)}, + } + + for _, test := range tests { + r := bytes.NewReader(test.content) + err := helpers.WriteToDisk(test.path, r, fs) + if err != nil { + t.Error(err) + } + + c, err := resGetLocal(test.path, fs) + if err != nil { + t.Errorf("Error getting resource content: %s", err) + } + if bytes.Compare(c, test.content) != 0 { + t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c)) + } + } + +} + +func getTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*httptest.Server, *http.Client) { + testServer := httptest.NewServer(http.HandlerFunc(handler)) + client := &http.Client{ + Transport: &http.Transport{Proxy: func(*http.Request) (*url.URL, error) { return url.Parse(testServer.URL) }}, + } + return testServer, client +} + +func TestScpGetRemote(t *testing.T) { + fs := new(afero.MemMapFs) + + tests := []struct { + path string + content []byte + }{ + {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`)}, + {"http://Doppel.Gänger/foo_Bar-Foo", []byte(`T€st Cont€nt 123`)}, + {"http://Doppel.Gänger/Fizz_Bazz-Foo", []byte(`T€st Банковский кассир Cont€nt 123`)}, + } + + for _, test := range tests { + + srv, cl := getTestServer(func(w http.ResponseWriter, r *http.Request) { + w.Write(test.content) + }) + defer func() { srv.Close() }() + + c, err := resGetRemote(test.path, fs, cl) + if err != nil { + t.Errorf("Error getting resource content: %s", err) + } + if bytes.Compare(c, test.content) != 0 { + t.Errorf("\nNet Expected: %s\nNet Actual: %s\n", string(test.content), string(c)) + } + cc, cErr := resGetCache(test.path, fs) + if cErr != nil { + t.Error(cErr) + } + if bytes.Compare(cc, test.content) != 0 { + t.Errorf("\nCache Expected: %s\nCache Actual: %s\n", string(test.content), string(c)) + } + } +} + +func TestParseCsv(t *testing.T) { + + tests := []struct { + csv []byte + sep string + exp string + err bool + }{ + {[]byte("a,b,c\nd,e,f\n"), "", "", true}, + {[]byte("a,b,c\nd,e,f\n"), "~/", "", true}, + {[]byte("a,b,c\nd,e,f"), "|", "a,b,cd,e,f", false}, + {[]byte("q,w,e\nd,e,f"), ",", "qwedef", false}, + {[]byte("a|b|c\nd|e|f|g"), "|", "abcdefg", true}, + {[]byte("z|y|c\nd|e|f"), "|", "zycdef", false}, + } + for _, test := range tests { + csv, err := parseCsv(test.csv, test.sep) + if test.err && err == nil { + t.Error("Expecting an error") + } + if test.err { + continue + } + if !test.err && err != nil { + t.Error(err) + } + + act := "" + for _, v := range csv { + act = act + strings.Join(v, "") + } + + if act != test.exp { + t.Errorf("\nExpected: %s\nActual: %s\n%#v\n", test.exp, act, csv) + } + + } +} From acdc0964341cf88059c9891e0f3667506ceed05a Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Sun, 28 Dec 2014 10:10:58 +1100 Subject: [PATCH 02/12] Fix bug on windows where the backslash got removed in func MakePath() --- helpers/path.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/helpers/path.go b/helpers/path.go index f6c77f9c380..884d807cb23 100644 --- a/helpers/path.go +++ b/helpers/path.go @@ -446,7 +446,14 @@ func GetTempDir(subPath string, fs afero.Fs) string { dir = dir + FilePathSeparator } if subPath != "" { + // preserve windows backslash :-( + if FilePathSeparator == "\\" { + subPath = strings.Replace(subPath, "\\", "____", -1) + } dir = dir + MakePath(subPath) + if FilePathSeparator == "\\" { + dir = strings.Replace(dir, "____", "\\", -1) + } if exists, _ := Exists(dir, fs); exists { return dir From 7823f25e09341a2cefe81e422e2875856505d089 Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Mon, 12 Jan 2015 08:59:42 +1100 Subject: [PATCH 03/12] Adding getJson and getCsv to funcMap --- tpl/template.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tpl/template.go b/tpl/template.go index 3377b9a3d67..320f969c836 100644 --- a/tpl/template.go +++ b/tpl/template.go @@ -1300,6 +1300,8 @@ func init() { "replace": Replace, "trim": Trim, "dateFormat": DateFormat, + "getJson": GetJson, + "getCsv": GetCsv, } } From e196c1bc0d97fe4aed980f308dd70c9d4301ff75 Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Mon, 2 Feb 2015 20:14:59 +1100 Subject: [PATCH 04/12] Add --ignoreCache CLI flag with description "Ignores the cache directory for reading but still writes to it" as @spf13 suggested. --- commands/hugo.go | 13 ++++++++- tpl/template_resources.go | 9 ++++--- tpl/template_resources_test.go | 48 ++++++++++++++++++++++------------ 3 files changed, 50 insertions(+), 20 deletions(-) diff --git a/commands/hugo.go b/commands/hugo.go index 4d7c3610508..64befed1dd2 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -55,7 +55,7 @@ Complete documentation is available at http://gohugo.io`, var hugoCmdV *cobra.Command //Flags that are to be added to commands. -var BuildWatch, Draft, Future, UglyUrls, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, PluralizeListTitles, NoTimes bool +var BuildWatch, IgnoreCache, Draft, Future, UglyUrls, Verbose, Logging, VerboseLog, DisableRSS, DisableSitemap, PluralizeListTitles, NoTimes bool var Source, CacheDir, Destination, Theme, BaseUrl, CfgFile, LogFile, Editor string //Execute adds all child commands to the root command HugoCmd and sets flags appropriately. @@ -84,6 +84,7 @@ func init() { HugoCmd.PersistentFlags().BoolVar(&DisableSitemap, "disableSitemap", false, "Do not build Sitemap file") HugoCmd.PersistentFlags().StringVarP(&Source, "source", "s", "", "filesystem path to read files relative from") HugoCmd.PersistentFlags().StringVarP(&CacheDir, "cacheDir", "", "$TMPDIR/hugo_cache/", "filesystem path to cache directory") + HugoCmd.PersistentFlags().BoolVarP(&IgnoreCache, "ignoreCache", "", false, "Ignores the cache directory for reading but still writes to it") HugoCmd.PersistentFlags().StringVarP(&Destination, "destination", "d", "", "filesystem path to write files to") HugoCmd.PersistentFlags().StringVarP(&Theme, "theme", "t", "", "theme to use (located in /themes/THEMENAME/)") HugoCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose output") @@ -127,6 +128,7 @@ func InitializeConfig() { viper.SetDefault("BuildFuture", false) viper.SetDefault("UglyUrls", false) viper.SetDefault("Verbose", false) + viper.SetDefault("IgnoreCache", false) viper.SetDefault("CanonifyUrls", false) viper.SetDefault("Taxonomies", map[string]string{"tag": "tags", "category": "categories"}) viper.SetDefault("Permalinks", make(hugolib.PermalinkOverrides, 0)) @@ -204,10 +206,19 @@ func InitializeConfig() { viper.Set("WorkingDir", dir) } + if hugoCmdV.PersistentFlags().Lookup("ignoreCache").Changed { + viper.Set("IgnoreCache", IgnoreCache) + } + if CacheDir != "" { if helpers.FilePathSeparator != CacheDir[len(CacheDir)-1:] { CacheDir = CacheDir + helpers.FilePathSeparator } + isDir, err := helpers.DirExists(CacheDir, hugofs.SourceFs) + utils.CheckErr(err) + if isDir == false { + mkdir(CacheDir) + } viper.Set("CacheDir", CacheDir) } else { viper.Set("CacheDir", helpers.GetTempDir("hugo_cache", hugofs.SourceFs)) diff --git a/tpl/template_resources.go b/tpl/template_resources.go index d4e105fcb87..4f198b40f88 100644 --- a/tpl/template_resources.go +++ b/tpl/template_resources.go @@ -64,7 +64,10 @@ func getCacheFileID(id string) string { // resGetCache returns the content for an ID from the file cache or an error // if the file is not found returns nil,nil -func resGetCache(id string, fs afero.Fs) ([]byte, error) { +func resGetCache(id string, fs afero.Fs, ignoreCache bool) ([]byte, error) { + if ignoreCache { + return nil, nil + } fID := getCacheFileID(id) isExists, err := helpers.Exists(fID, fs) if err != nil { @@ -99,7 +102,7 @@ func resWriteCache(id string, c []byte, fs afero.Fs) error { // resGetRemote loads the content of a remote file. This method is thread safe. func resGetRemote(url string, fs afero.Fs, hc *http.Client) ([]byte, error) { - c, err := resGetCache(url, fs) + c, err := resGetCache(url, fs, viper.GetBool("IgnoreCache")) if c != nil && err == nil { return c, nil } @@ -112,7 +115,7 @@ func resGetRemote(url string, fs afero.Fs, hc *http.Client) ([]byte, error) { defer func() { remoteUrlLock.UrlUnlock(url) }() // avoid multiple locks due to calling resGetCache twice - c, err = resGetCache(url, fs) + c, err = resGetCache(url, fs, viper.GetBool("IgnoreCache")) if c != nil && err == nil { return c, nil } diff --git a/tpl/template_resources_test.go b/tpl/template_resources_test.go index 40ea1fc4350..0222e558faa 100644 --- a/tpl/template_resources_test.go +++ b/tpl/template_resources_test.go @@ -30,19 +30,21 @@ func TestScpCache(t *testing.T) { tests := []struct { path string content []byte + ignore bool }{ - {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`)}, - {"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`)}, - {"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`)}, - {"трям/трям", []byte(`T€st трям/трям Content 123`)}, - {"은행", []byte(`T€st C은행ontent 123`)}, - {"Банковский кассир", []byte(`Банковский кассир T€st Content 123`)}, + {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`), false}, + {"fOO,bar:foo%bAR", []byte(`T€st Content 123 fOO,bar:foo%bAR`), false}, + {"FOo/BaR.html", []byte(`FOo/BaR.html T€st Content 123`), false}, + {"трям/трям", []byte(`T€st трям/трям Content 123`), false}, + {"은행", []byte(`T€st C은행ontent 123`), false}, + {"Банковский кассир", []byte(`Банковский кассир T€st Content 123`), false}, + {"Банковский кассир", []byte(`Банковский кассир T€st Content 456`), true}, } fs := new(afero.MemMapFs) for _, test := range tests { - c, err := resGetCache(test.path, fs) + c, err := resGetCache(test.path, fs, test.ignore) if err != nil { t.Errorf("Error getting cache: %s", err) } @@ -55,12 +57,18 @@ func TestScpCache(t *testing.T) { t.Errorf("Error writing cache: %s", err) } - c, err = resGetCache(test.path, fs) + c, err = resGetCache(test.path, fs, test.ignore) if err != nil { t.Errorf("Error getting cache after writing: %s", err) } - if bytes.Compare(c, test.content) != 0 { - t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c)) + if test.ignore { + if c != nil { + t.Errorf("Cache ignored but content is not nil: %s", string(c)) + } + } else { + if bytes.Compare(c, test.content) != 0 { + t.Errorf("\nExpected: %s\nActual: %s\n", string(test.content), string(c)) + } } } } @@ -111,10 +119,12 @@ func TestScpGetRemote(t *testing.T) { tests := []struct { path string content []byte + ignore bool }{ - {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`)}, - {"http://Doppel.Gänger/foo_Bar-Foo", []byte(`T€st Cont€nt 123`)}, - {"http://Doppel.Gänger/Fizz_Bazz-Foo", []byte(`T€st Банковский кассир Cont€nt 123`)}, + {"http://Foo.Bar/foo_Bar-Foo", []byte(`T€st Content 123`), false}, + {"http://Doppel.Gänger/foo_Bar-Foo", []byte(`T€st Cont€nt 123`), false}, + {"http://Doppel.Gänger/Fizz_Bazz-Foo", []byte(`T€st Банковский кассир Cont€nt 123`), false}, + {"http://Doppel.Gänger/Fizz_Bazz-Bar", []byte(`T€st Банковский кассир Cont€nt 456`), true}, } for _, test := range tests { @@ -131,12 +141,18 @@ func TestScpGetRemote(t *testing.T) { if bytes.Compare(c, test.content) != 0 { t.Errorf("\nNet Expected: %s\nNet Actual: %s\n", string(test.content), string(c)) } - cc, cErr := resGetCache(test.path, fs) + cc, cErr := resGetCache(test.path, fs, test.ignore) if cErr != nil { t.Error(cErr) } - if bytes.Compare(cc, test.content) != 0 { - t.Errorf("\nCache Expected: %s\nCache Actual: %s\n", string(test.content), string(c)) + if test.ignore { + if cc != nil { + t.Errorf("Cache ignored but content is not nil: %s", string(cc)) + } + } else { + if bytes.Compare(cc, test.content) != 0 { + t.Errorf("\nCache Expected: %s\nCache Actual: %s\n", string(test.content), string(cc)) + } } } } From 111ce0554d8383b247294e3b80774c8bfc1c40d9 Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Sun, 8 Feb 2015 15:33:30 +1100 Subject: [PATCH 05/12] GetJson and GetCsv have now variadic URL parts so that you can submit pre or post parts of an URL. The parts will be joined to the final URL. --- tpl/template_resources.go | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tpl/template_resources.go b/tpl/template_resources.go index 4f198b40f88..0ac7686cff7 100644 --- a/tpl/template_resources.go +++ b/tpl/template_resources.go @@ -173,9 +173,11 @@ func resGetResource(url string) ([]byte, error) { return resGetLocal(url, hugofs.SourceFs) } -// GetJson expects the url to a resource which can either be a local or a remote one. +// GetJson expects one or n-parts of a URL to a resource which can either be a local or a remote one. +// If you provide multiple parts they will be joined together to the final URL. // GetJson returns nil or parsed JSON to use in a short code. -func GetJson(url string) interface{} { +func GetJson(urlParts ...string) interface{} { + url := strings.Join(urlParts, "") c, err := resGetResource(url) if err != nil { jww.ERROR.Printf("Failed to get json resource %s with error message %s", url, err) @@ -204,11 +206,13 @@ func parseCsv(c []byte, sep string) ([][]string, error) { return r.ReadAll() } -// GetCsv expects the url to a resource which can either be a local or a remote one and the type -// of the data separator which can be comma, semi-colon, pipe, but only one character. +// GetCsv expects a data separator and one or n-parts of a URL to a resource which +// can either be a local or a remote one. +// The data separator can be a comma, semi-colon, pipe, etc, but only one character. +// If you provide multiple parts for the URL they will be joined together to the final URL. // GetCsv returns nil or a slice slice to use in a short code. -func GetCsv(url string, sep string) [][]string { - +func GetCsv(sep string, urlParts ...string) [][]string { + url := strings.Join(urlParts, "") c, err := resGetResource(url) if err != nil { jww.ERROR.Printf("Failed to get csv resource %s with error message %s", url, err) From 21bc4564b4d3782a15e9c4cb368a435365737a74 Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Mon, 9 Feb 2015 09:22:50 +1100 Subject: [PATCH 06/12] Fix bug where running hugo without options causes to create $TMPDIR/hugo_cache/ directory. --- commands/hugo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commands/hugo.go b/commands/hugo.go index 64befed1dd2..986036e7a84 100644 --- a/commands/hugo.go +++ b/commands/hugo.go @@ -83,7 +83,7 @@ func init() { HugoCmd.PersistentFlags().BoolVar(&DisableRSS, "disableRSS", false, "Do not build RSS files") HugoCmd.PersistentFlags().BoolVar(&DisableSitemap, "disableSitemap", false, "Do not build Sitemap file") HugoCmd.PersistentFlags().StringVarP(&Source, "source", "s", "", "filesystem path to read files relative from") - HugoCmd.PersistentFlags().StringVarP(&CacheDir, "cacheDir", "", "$TMPDIR/hugo_cache/", "filesystem path to cache directory") + HugoCmd.PersistentFlags().StringVarP(&CacheDir, "cacheDir", "", "", "filesystem path to cache directory. Defaults: $TMPDIR/hugo_cache/") HugoCmd.PersistentFlags().BoolVarP(&IgnoreCache, "ignoreCache", "", false, "Ignores the cache directory for reading but still writes to it") HugoCmd.PersistentFlags().StringVarP(&Destination, "destination", "d", "", "filesystem path to write files to") HugoCmd.PersistentFlags().StringVarP(&Theme, "theme", "t", "", "theme to use (located in /themes/THEMENAME/)") From e466ba0d92707695308c1ffa5811afbcfa99d77d Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Thu, 12 Feb 2015 10:16:03 +1100 Subject: [PATCH 07/12] Initial doc file: Dynamic Content --- docs/content/extras/datafiles.md | 2 +- docs/content/extras/dynamiccontent.md | 28 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 docs/content/extras/dynamiccontent.md diff --git a/docs/content/extras/datafiles.md b/docs/content/extras/datafiles.md index 60e89b858ab..ba57f748fb5 100644 --- a/docs/content/extras/datafiles.md +++ b/docs/content/extras/datafiles.md @@ -5,7 +5,7 @@ date: 2015-01-22 menu: main: parent: extras -next: /extras/highlighting +next: /extras/dynamiccontent prev: /extras/scratch title: Data Files weight: 90 diff --git a/docs/content/extras/dynamiccontent.md b/docs/content/extras/dynamiccontent.md new file mode 100644 index 00000000000..67794a6383a --- /dev/null +++ b/docs/content/extras/dynamiccontent.md @@ -0,0 +1,28 @@ +--- +aliases: +- /doc/dynamiccontent/ +date: 2015-02-14 +menu: + main: + parent: extras +next: /extras/highlighting +prev: /extras/datafiles +title: Dynamic Content +weight: 91 +--- + +Dynamic content with a static site generator? Yes it is possible! + +In addition to the [built-in variables](/templates/variables/) available from Hugo, you can specify your own custom data that can be accessed via templates or shortcodes. + +Hugo supports loading data from [YAML](http://yaml.org/), [JSON](http://www.json.org/), and [TOML](https://github.com/toml-lang/toml) files located in the `data` directory. + +**It even works with [LiveReload](/extras/livereload/).** + +## The Data Folder + +As explained in [Source Organization](/overview/source-directory/), the `data` folder is where you can store additional data for Hugo to use when generating your site. These files must be YAML, JSON or TOML files (using either the `.yml`, `.yaml`, `.json` or `toml` extension) and the data will be accessible as a `map` in `.Site.Data`. + +**The keys in this map will be a dot chained set of _path_, _filename_ and _key_ in file (if applicable).** + +This is best explained with an example: From 452b1452ba4ffea91695ee9cf17ad4bccd382687 Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Thu, 12 Feb 2015 11:40:06 +1100 Subject: [PATCH 08/12] Add getSQL idea --- docs/content/extras/dynamiccontent.md | 36 ++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/content/extras/dynamiccontent.md b/docs/content/extras/dynamiccontent.md index 67794a6383a..60bf336dd23 100644 --- a/docs/content/extras/dynamiccontent.md +++ b/docs/content/extras/dynamiccontent.md @@ -13,7 +13,7 @@ weight: 91 Dynamic content with a static site generator? Yes it is possible! -In addition to the [built-in variables](/templates/variables/) available from Hugo, you can specify your own custom data that can be accessed via templates or shortcodes. +Besides the [data files](/extras/datafiles/) available from Hugo, you can specify your own custom data that can be accessed via templates or shortcodes. Hugo supports loading data from [YAML](http://yaml.org/), [JSON](http://www.json.org/), and [TOML](https://github.com/toml-lang/toml) files located in the `data` directory. @@ -26,3 +26,37 @@ As explained in [Source Organization](/overview/source-directory/), the `data` f **The keys in this map will be a dot chained set of _path_, _filename_ and _key_ in file (if applicable).** This is best explained with an example: + +## The Future: getSQL + +The outlook to support more sources is of course implementing SQL support. + +Maybe adding two new CLI switches: + + --sqlDriver=mysql|postres|mssql + --sqlSource=string|filename + +#### `--sqlDriver` + +specifies the driver to use which can be one from [https://github.com/golang/go/wiki/SQLDrivers](https://github.com/golang/go/wiki/SQLDrivers) + +#### `--sqlSource` + +You can either provide the connection string on the command file OR an existing file which contains the connection string. + +How the connection string looks like depends heavily on the used driver. For MySQL: + + hugo --sqlDriver=mysql \ + --sqlSource=username:password@protocol(address)/dbname?param=value + +or with a file name: + + hugo --sqlDriver=mysql --sqlSource=path/to/myCredentials.txt + +The file myCredentials.txt contains the connection string: `username:password@protocol(address)/dbname?param=value` and nothing more! + + + +``` +$data := getSQL "SELECT id,artist,genre,title from musicTable" +``` From f6f71ea2fc85318d85bf69fa4edbedf8415a4f4d Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Fri, 13 Feb 2015 17:21:26 +1100 Subject: [PATCH 09/12] Update doc --- docs/content/extras/dynamiccontent.md | 154 ++++++++++++++++++++++---- 1 file changed, 131 insertions(+), 23 deletions(-) diff --git a/docs/content/extras/dynamiccontent.md b/docs/content/extras/dynamiccontent.md index 60bf336dd23..7727279d485 100644 --- a/docs/content/extras/dynamiccontent.md +++ b/docs/content/extras/dynamiccontent.md @@ -13,50 +13,158 @@ weight: 91 Dynamic content with a static site generator? Yes it is possible! -Besides the [data files](/extras/datafiles/) available from Hugo, you can specify your own custom data that can be accessed via templates or shortcodes. +Besides the [data files](/extras/datafiles/) feature, we have also implemented the feature "Dynamic Content" +which lets you load any [JSON](http://www.json.org/) or [CSV](http://en.wikipedia.org/wiki/Comma-separated_values) +file from nearly any resource. -Hugo supports loading data from [YAML](http://yaml.org/), [JSON](http://www.json.org/), and [TOML](https://github.com/toml-lang/toml) files located in the `data` directory. +"Dynamic Content" consists at the moment of two functions `getJson` and `getCsv` which are available in +**all template files**. -**It even works with [LiveReload](/extras/livereload/).** +## Implementation details -## The Data Folder +### Calling the functions with an URL -As explained in [Source Organization](/overview/source-directory/), the `data` folder is where you can store additional data for Hugo to use when generating your site. These files must be YAML, JSON or TOML files (using either the `.yml`, `.yaml`, `.json` or `toml` extension) and the data will be accessible as a `map` in `.Site.Data`. +In any template file call the functions like: -**The keys in this map will be a dot chained set of _path_, _filename_ and _key_ in file (if applicable).** +``` + {{ $dataJ := getJson "url" }} + {{ $dataC := getCsv "separator" "url" }} +``` -This is best explained with an example: +or if you use a prefix or postfix for the URL the functions +accepts [variadic arguments](http://en.wikipedia.org/wiki/Variadic_function): -## The Future: getSQL +``` + {{ $dataJ := getJson "url prefix" "arg1" "arg2" "arg n" }} + {{ $dataC := getCsv "separator" "url prefix" "arg1" "arg2" "arg n" }} +``` -The outlook to support more sources is of course implementing SQL support. +The separator for `getCsv` must be put on the first position and can be only one character long. -Maybe adding two new CLI switches: +All passed arguments will be joined to the final URL, example: - --sqlDriver=mysql|postres|mssql - --sqlSource=string|filename +``` + {{ $urlPre := "https://api.github.com" }} + {{ $gistJ := getJson $urlPre "/users/GITHUB_USERNAME/gists" }} +``` -#### `--sqlDriver` +will resolve internally to: -specifies the driver to use which can be one from [https://github.com/golang/go/wiki/SQLDrivers](https://github.com/golang/go/wiki/SQLDrivers) +``` + {{ $gistJ := getJson "https://api.github.com/users/GITHUB_USERNAME/gists" }} +``` -#### `--sqlSource` +Eventually you can range or the map/array/slice. This example will output the first 5 Github gists for a user: + +``` +
    + {{ $urlPre := "https://api.github.com" }} + {{ $gistJ := getJson $urlPre "/users/GITHUB_USERNAME/gists" }} + {{range first 5 $gistJ }} + {{ if .public }} +
  • {{.description}}
    • + {{ end }} + {{end}} +
+``` -You can either provide the connection string on the command file OR an existing file which contains the connection string. +### Example for CSV files -How the connection string looks like depends heavily on the used driver. For MySQL: +For `getCsv` the one character long separator must be placed on the first position followed by the URL. + +``` + + + + + + + + + + {{ $url := "http://a-big-corp.com/finance/employee-salaries.csv" }} + {{ $sep := "," }} + {{ range $i, $r := getCsv $sep $url }} + + + + + + {{ end }} + +
NamePositionSalary
{{ index $r 0 }}{{ index $r 1 }}{{ index $r 2 }}
+``` + +### Caching of URLs + +Each downloaded URL will be cached in the default folder `$TMPDIR/hugo_cache/`. The variable `$TMPDIR` will be +resolved to your system dependent temporary directory. + +With the command line flag `--cacheDir` you can specify any folder on your system as a caching directory. + +If you don't like caching at all you can fully disabled to read from the cache with the command line +flag `--ignoreCache`. But hugo will still write on each build of the site to the cache folder (silent backup). + +### Authentication when using REST URLs - hugo --sqlDriver=mysql \ - --sqlSource=username:password@protocol(address)/dbname?param=value +At the moment you can only use those authentication methods which can be put into an URL. +OAuth or other stuff is not implemented. -or with a file name: +### Loading local files - hugo --sqlDriver=mysql --sqlSource=path/to/myCredentials.txt +To load local files with the two functions `getJson` and `getCsv` the source files must reside within +Hugos working directory. The file extension does not matter but the content. -The file myCredentials.txt contains the connection string: `username:password@protocol(address)/dbname?param=value` and nothing more! +It applies the same logic as in the topic: *Calling the functions with an URL*. +## Live reload +There is not chance to trigger a [LiveReload](/extras/livereload/) when the content of an URL changes. +But when a local JSON/CSV file changes then of course a live reload will be triggered. Symlinks not supported. + +**URLs and Live reload**: If you change any local file and the live reload got trigger Hugo will +either read the URL content from the cache or if you have disabled the cache Hugo will re-download the content. +This can create a huge traffic and also you may reach API limits quickly. + +As downloading of content takes a while Hugo stops with processing your markdown files until the content +has been downloaded. + +## The Future: + +### YAML and TOML + +If the community demands the implementation of *getYaml* [YAML](http://yaml.org/) or +*getToml* [TOML](https://github.com/toml-lang/toml) these functions will for sure follow. + +### getSql + +The outlook to support more sources is of course implementing SQL support. + +Maybe adding a new CLI option: + + --sqlSource=path/to/filename.ext + +#### `--sqlSource` + +The file must start with `[mysql|postres|mssql|...]_whatever.ext` + +The part until the first underscore specifies the driver to use which can be one +from [https://github.com/golang/go/wiki/SQLDrivers](https://github.com/golang/go/wiki/SQLDrivers). + +The file itself contains only the connection string and no other comments or characters. + +How the connection string looks like depends heavily on the used driver. For MySQL: + + hugo --sqlSource=path/to/mysql_Credentials.txt + +The file `mysql_Credentials.txt` contains the connection string: +`username:password@protocol(address)/dbname?param=value` and nothing more! + +In your template you can process as with the `getCsv` function: ``` -$data := getSQL "SELECT id,artist,genre,title from musicTable" +$data := getSql "SELECT id,artist,genre,title from musicTable" ``` + +Abusing `getSql` with [DML](http://en.wikipedia.org/wiki/Data_manipulation_language) or +[DDL](http://en.wikipedia.org/wiki/Data_definition_language) statements is up to you. From 0cd98e9cf944ef71faea59164beaef18808ce10f Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Sun, 15 Feb 2015 10:01:58 +1100 Subject: [PATCH 10/12] Update Dynamic Content docs --- docs/content/extras/dynamiccontent.md | 194 ++++++++++++++------------ 1 file changed, 108 insertions(+), 86 deletions(-) diff --git a/docs/content/extras/dynamiccontent.md b/docs/content/extras/dynamiccontent.md index 7727279d485..ca9c59b9269 100644 --- a/docs/content/extras/dynamiccontent.md +++ b/docs/content/extras/dynamiccontent.md @@ -13,12 +13,14 @@ weight: 91 Dynamic content with a static site generator? Yes it is possible! -Besides the [data files](/extras/datafiles/) feature, we have also implemented the feature "Dynamic Content" -which lets you load any [JSON](http://www.json.org/) or [CSV](http://en.wikipedia.org/wiki/Comma-separated_values) -file from nearly any resource. +Besides the [data files](/extras/datafiles/) feature, we have also +implemented the feature "Dynamic Content" which lets you load +any [JSON](http://www.json.org/) or +[CSV](http://en.wikipedia.org/wiki/Comma-separated_values) file +from nearly any resource. -"Dynamic Content" consists at the moment of two functions `getJson` and `getCsv` which are available in -**all template files**. +"Dynamic Content" consists at the moment of two functions `getJson` +and `getCsv` which are available in **all template files**. ## Implementation details @@ -26,119 +28,129 @@ file from nearly any resource. In any template file call the functions like: -``` + {{ $dataJ := getJson "url" }} - {{ $dataC := getCsv "separator" "url" }} -``` + {{ $dataC := getCsv "separator" "url" }} + or if you use a prefix or postfix for the URL the functions accepts [variadic arguments](http://en.wikipedia.org/wiki/Variadic_function): -``` {{ $dataJ := getJson "url prefix" "arg1" "arg2" "arg n" }} {{ $dataC := getCsv "separator" "url prefix" "arg1" "arg2" "arg n" }} -``` -The separator for `getCsv` must be put on the first position and can be only one character long. +The separator for `getCsv` must be put on the first position and can be +only one character long. All passed arguments will be joined to the final URL, example: -``` {{ $urlPre := "https://api.github.com" }} - {{ $gistJ := getJson $urlPre "/users/GITHUB_USERNAME/gists" }} -``` + {{ $gistJ := getJson $urlPre "/users/GITHUB_USERNAME/gists" }} will resolve internally to: -``` {{ $gistJ := getJson "https://api.github.com/users/GITHUB_USERNAME/gists" }} -``` -Eventually you can range or the map/array/slice. This example will output the first 5 Github gists for a user: +Eventually you can range over the array. This example will output the +first 5 Github gists for a user: + +
    + {{ $urlPre := "https://api.github.com" }} + {{ $gistJ := getJson $urlPre "/users/GITHUB_USERNAME/gists" }} + {{range first 5 $gistJ }} + {{ if .public }} +
  • {{.description}}
    • + {{ end }} + {{end}} +
-``` -
    - {{ $urlPre := "https://api.github.com" }} - {{ $gistJ := getJson $urlPre "/users/GITHUB_USERNAME/gists" }} - {{range first 5 $gistJ }} - {{ if .public }} -
  • {{.description}}
    • - {{ end }} - {{end}} -
-``` ### Example for CSV files -For `getCsv` the one character long separator must be placed on the first position followed by the URL. - -``` - - - - - - - - - - {{ $url := "http://a-big-corp.com/finance/employee-salaries.csv" }} - {{ $sep := "," }} - {{ range $i, $r := getCsv $sep $url }} - - - - - - {{ end }} - -
NamePositionSalary
{{ index $r 0 }}{{ index $r 1 }}{{ index $r 2 }}
-``` +For `getCsv` the one character long separator must be placed on the +first position followed by the URL. + + + + + + + + + + + {{ $url := "http://a-big-corp.com/finance/employee-salaries.csv" }} + {{ $sep := "," }} + {{ range $i, $r := getCsv $sep $url }} + + + + + + {{ end }} + +
NamePositionSalary
{{ index $r 0 }}{{ index $r 1 }}{{ index $r 2 }}
+ +The expression `{{index $r number}}` must be used to output the nth-column from +the current row. ### Caching of URLs -Each downloaded URL will be cached in the default folder `$TMPDIR/hugo_cache/`. The variable `$TMPDIR` will be -resolved to your system dependent temporary directory. +Each downloaded URL will be cached in the default folder `$TMPDIR/hugo_cache/`. +The variable `$TMPDIR` will be resolved to your system dependent +temporary directory. -With the command line flag `--cacheDir` you can specify any folder on your system as a caching directory. +With the command line flag `--cacheDir` you can specify any folder on +your system as a caching directory. -If you don't like caching at all you can fully disabled to read from the cache with the command line -flag `--ignoreCache`. But hugo will still write on each build of the site to the cache folder (silent backup). +If you don't like caching at all you can fully disable to read from the +cache with the command line flag `--ignoreCache`. But hugo will always +write, on each build of the site, to the cache folder (silent backup). ### Authentication when using REST URLs -At the moment you can only use those authentication methods which can be put into an URL. -OAuth or other stuff is not implemented. +At the moment you can only use those authentication methods which can +be put into an URL. OAuth or other stuff is not implemented. ### Loading local files -To load local files with the two functions `getJson` and `getCsv` the source files must reside within -Hugos working directory. The file extension does not matter but the content. +To load local files with the two functions `getJson` and `getCsv` the +source files must reside within Hugos working directory. The file +extension does not matter but the content. -It applies the same logic as in the topic: *Calling the functions with an URL*. +It applies the same output logic as in the topic: *Calling the functions with an URL*. ## Live reload -There is not chance to trigger a [LiveReload](/extras/livereload/) when the content of an URL changes. -But when a local JSON/CSV file changes then of course a live reload will be triggered. Symlinks not supported. +There is no chance to trigger a [LiveReload](/extras/livereload/) when +the content of an URL changes. But when a local JSON/CSV file changes +then of course a live reload will be triggered. Symlinks not supported. + +**URLs and Live reload**: If you change any local file and the live reload +got trigger Hugo will either read the URL content from the cache or if +you have disabled the cache Hugo will re-download the content. +This can creates a huge traffic and also you may reach API limits quickly. -**URLs and Live reload**: If you change any local file and the live reload got trigger Hugo will -either read the URL content from the cache or if you have disabled the cache Hugo will re-download the content. -This can create a huge traffic and also you may reach API limits quickly. +As downloading of content takes a while Hugo stops with processing +your markdown files until the content has been downloaded. -As downloading of content takes a while Hugo stops with processing your markdown files until the content -has been downloaded. +## Examples -## The Future: +- Photo gallery JSON powered: [https://github.com/pcdummy/hugo-lightslider-example](https://github.com/pcdummy/hugo-lightslider-example) +- Github Starred Repositories [in a posts](https://github.com/SchumacherFM/blog-cs/blob/master/content%2Fposts%2Fgithub-starred.md) with the related [short code](https://github.com/SchumacherFM/blog-cs/blob/master/layouts%2Fshortcodes%2FghStarred.html). +- more? + +## The Future ### YAML and TOML -If the community demands the implementation of *getYaml* [YAML](http://yaml.org/) or -*getToml* [TOML](https://github.com/toml-lang/toml) these functions will for sure follow. +If the community demands the implementation of *getYaml* +[YAML](http://yaml.org/) or *getToml* [TOML](https://github.com/toml-lang/toml) +these functions will for sure follow. ### getSql -The outlook to support more sources is of course implementing SQL support. +The outlook to support more sources is of course implementing SQL support. The following description is an initial idea and may change. Maybe adding a new CLI option: @@ -146,25 +158,35 @@ Maybe adding a new CLI option: #### `--sqlSource` -The file must start with `[mysql|postres|mssql|...]_whatever.ext` +The file must start with `[mysql|postres|mssql|...]_[\w]+.ext` -The part until the first underscore specifies the driver to use which can be one -from [https://github.com/golang/go/wiki/SQLDrivers](https://github.com/golang/go/wiki/SQLDrivers). +The part until the first underscore specifies the driver to use which can +be one from [https://github.com/golang/go/wiki/SQLDrivers](https://github.com/golang/go/wiki/SQLDrivers). -The file itself contains only the connection string and no other comments or characters. +The file itself contains only the connection string and no other comments +or characters. -How the connection string looks like depends heavily on the used driver. For MySQL: +How the connection string looks like depends heavily on the used driver. +For MySQL: - hugo --sqlSource=path/to/mysql_Credentials.txt + hugo --sqlSource=path/to/mysql_credentials.txt -The file `mysql_Credentials.txt` contains the connection string: +The file `mysql_credentials.txt` contains the connection string: `username:password@protocol(address)/dbname?param=value` and nothing more! -In your template you can process as with the `getCsv` function: +In your template you can process `getSql` the same way as with +the `getJson` function: + + {{ $rows := getSql "SELECT id,artist,genre,title from musicTable" }} +
    + {{range first 5 $rows }} +
  • #{{ .id }} {{.artist}}
    • + {{end}} +
-``` -$data := getSql "SELECT id,artist,genre,title from musicTable" -``` +Queries with `SELECT * from table` are of course also possible. +Returned **values** will be converted to **strings**. -Abusing `getSql` with [DML](http://en.wikipedia.org/wiki/Data_manipulation_language) or -[DDL](http://en.wikipedia.org/wiki/Data_definition_language) statements is up to you. +Abusing `getSql` with [DML](http://en.wikipedia.org/wiki/Data_manipulation_language) +or [DDL](http://en.wikipedia.org/wiki/Data_definition_language) statements +is up to you. From 87e2844157fc3ec99d40617dd831d58fc13ccebd Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Sun, 15 Feb 2015 17:41:46 +1100 Subject: [PATCH 11/12] getJson and getCsv documentation proof reading by @msjulias --- docs/content/extras/dynamiccontent.md | 48 ++++++++++++++------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/docs/content/extras/dynamiccontent.md b/docs/content/extras/dynamiccontent.md index ca9c59b9269..337e55d4bcd 100644 --- a/docs/content/extras/dynamiccontent.md +++ b/docs/content/extras/dynamiccontent.md @@ -13,28 +13,28 @@ weight: 91 Dynamic content with a static site generator? Yes it is possible! -Besides the [data files](/extras/datafiles/) feature, we have also -implemented the feature "Dynamic Content" which lets you load +In addition to the [data files](/extras/datafiles/) feature, we have also +implemented the feature "Dynamic Content", which lets you load any [JSON](http://www.json.org/) or [CSV](http://en.wikipedia.org/wiki/Comma-separated_values) file from nearly any resource. -"Dynamic Content" consists at the moment of two functions `getJson` -and `getCsv` which are available in **all template files**. +"Dynamic Content" currently consists of two functions, `getJson` +and `getCsv`, which are available in **all template files**. ## Implementation details ### Calling the functions with an URL -In any template file call the functions like: +In any HTML template or Markdown document call the functions like: {{ $dataJ := getJson "url" }} - {{ $dataC := getCsv "separator" "url" }} + {{ $dataC := getCsv "separator" "url" }} or if you use a prefix or postfix for the URL the functions -accepts [variadic arguments](http://en.wikipedia.org/wiki/Variadic_function): +accept [variadic arguments](http://en.wikipedia.org/wiki/Variadic_function): {{ $dataJ := getJson "url prefix" "arg1" "arg2" "arg n" }} {{ $dataC := getCsv "separator" "url prefix" "arg1" "arg2" "arg n" }} @@ -45,7 +45,7 @@ only one character long. All passed arguments will be joined to the final URL, example: {{ $urlPre := "https://api.github.com" }} - {{ $gistJ := getJson $urlPre "/users/GITHUB_USERNAME/gists" }} + {{ $gistJ := getJson $urlPre "/users/GITHUB_USERNAME/gists" }} will resolve internally to: @@ -103,14 +103,15 @@ temporary directory. With the command line flag `--cacheDir` you can specify any folder on your system as a caching directory. -If you don't like caching at all you can fully disable to read from the -cache with the command line flag `--ignoreCache`. But hugo will always +If you don't like caching at all, you can fully disable to read from the +cache with the command line flag `--ignoreCache`. However Hugo will always write, on each build of the site, to the cache folder (silent backup). ### Authentication when using REST URLs -At the moment you can only use those authentication methods which can -be put into an URL. OAuth or other stuff is not implemented. +Currently you can only use those authentication methods that can +be put into an URL. [OAuth](http://en.wikipedia.org/wiki/OAuth) or +other authentication methods are not implemented. ### Loading local files @@ -123,15 +124,15 @@ It applies the same output logic as in the topic: *Calling the functions with an ## Live reload There is no chance to trigger a [LiveReload](/extras/livereload/) when -the content of an URL changes. But when a local JSON/CSV file changes -then of course a live reload will be triggered. Symlinks not supported. +the content of an URL changes. However when a local JSON/CSV file changes +then a live reload will be triggered of course. Symlinks not supported. **URLs and Live reload**: If you change any local file and the live reload -got trigger Hugo will either read the URL content from the cache or if -you have disabled the cache Hugo will re-download the content. -This can creates a huge traffic and also you may reach API limits quickly. +got triggered Hugo will either read the URL content from the cache or, if +you have disabled the cache, Hugo will re-download the content. +This can create huge traffic and you may also reach API limits quickly. -As downloading of content takes a while Hugo stops with processing +As downloading of content takes a while, Hugo stops with processing your markdown files until the content has been downloaded. ## Examples @@ -146,7 +147,7 @@ your markdown files until the content has been downloaded. If the community demands the implementation of *getYaml* [YAML](http://yaml.org/) or *getToml* [TOML](https://github.com/toml-lang/toml) -these functions will for sure follow. +these functions will certainly follow. ### getSql @@ -160,7 +161,7 @@ Maybe adding a new CLI option: The file must start with `[mysql|postres|mssql|...]_[\w]+.ext` -The part until the first underscore specifies the driver to use which can +The part before the first underscore specifies the driver to use, which can be one from [https://github.com/golang/go/wiki/SQLDrivers](https://github.com/golang/go/wiki/SQLDrivers). The file itself contains only the connection string and no other comments @@ -177,16 +178,17 @@ The file `mysql_credentials.txt` contains the connection string: In your template you can process `getSql` the same way as with the `getJson` function: - {{ $rows := getSql "SELECT id,artist,genre,title from musicTable" }} + {{ $rows := getSql "SELECT id,artist,genre,title FROM musicTable" }}
    {{range first 5 $rows }}
  • #{{ .id }} {{.artist}}
    • {{end}}
-Queries with `SELECT * from table` are of course also possible. +Queries with `SELECT * FROM table` are of course also possible but +should not be used. Returned **values** will be converted to **strings**. Abusing `getSql` with [DML](http://en.wikipedia.org/wiki/Data_manipulation_language) or [DDL](http://en.wikipedia.org/wiki/Data_definition_language) statements -is up to you. +causes Hugo to hang up. From 866ce168db4124054c8fa09343b40df31907c2df Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Mon, 16 Feb 2015 08:43:42 +1100 Subject: [PATCH 12/12] Moved futured getSQL topic to another branch --- docs/content/extras/dynamiccontent.md | 52 --------------------------- 1 file changed, 52 deletions(-) diff --git a/docs/content/extras/dynamiccontent.md b/docs/content/extras/dynamiccontent.md index 337e55d4bcd..4155f04e87a 100644 --- a/docs/content/extras/dynamiccontent.md +++ b/docs/content/extras/dynamiccontent.md @@ -140,55 +140,3 @@ your markdown files until the content has been downloaded. - Photo gallery JSON powered: [https://github.com/pcdummy/hugo-lightslider-example](https://github.com/pcdummy/hugo-lightslider-example) - Github Starred Repositories [in a posts](https://github.com/SchumacherFM/blog-cs/blob/master/content%2Fposts%2Fgithub-starred.md) with the related [short code](https://github.com/SchumacherFM/blog-cs/blob/master/layouts%2Fshortcodes%2FghStarred.html). - more? - -## The Future - -### YAML and TOML - -If the community demands the implementation of *getYaml* -[YAML](http://yaml.org/) or *getToml* [TOML](https://github.com/toml-lang/toml) -these functions will certainly follow. - -### getSql - -The outlook to support more sources is of course implementing SQL support. The following description is an initial idea and may change. - -Maybe adding a new CLI option: - - --sqlSource=path/to/filename.ext - -#### `--sqlSource` - -The file must start with `[mysql|postres|mssql|...]_[\w]+.ext` - -The part before the first underscore specifies the driver to use, which can -be one from [https://github.com/golang/go/wiki/SQLDrivers](https://github.com/golang/go/wiki/SQLDrivers). - -The file itself contains only the connection string and no other comments -or characters. - -How the connection string looks like depends heavily on the used driver. -For MySQL: - - hugo --sqlSource=path/to/mysql_credentials.txt - -The file `mysql_credentials.txt` contains the connection string: -`username:password@protocol(address)/dbname?param=value` and nothing more! - -In your template you can process `getSql` the same way as with -the `getJson` function: - - {{ $rows := getSql "SELECT id,artist,genre,title FROM musicTable" }} -
    - {{range first 5 $rows }} -
  • #{{ .id }} {{.artist}}
    • - {{end}} -
- -Queries with `SELECT * FROM table` are of course also possible but -should not be used. -Returned **values** will be converted to **strings**. - -Abusing `getSql` with [DML](http://en.wikipedia.org/wiki/Data_manipulation_language) -or [DDL](http://en.wikipedia.org/wiki/Data_definition_language) statements -causes Hugo to hang up.