templates: Change functions, add front matter support, better markdown

modules/caddyhttp/templates/frontmatter.go

@@ -0,0 +1,100 @@
+package templates
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+	"unicode"
+
+	"github.com/naoina/toml"
+	"gopkg.in/yaml.v2"
+)
+
+func extractFrontMatter(input string) (map[string]interface{}, string, error) {
+	// get the bounds of the first non-empty line
+	var firstLineStart, firstLineEnd int
+	lineEmpty := true
+	for i, b := range input {
+		if b == '\n' {
+			firstLineStart = firstLineEnd
+			if firstLineStart > 0 {
+				firstLineStart++ // skip newline character
+			}
+			firstLineEnd = i
+			if !lineEmpty {
+				break
+			}
+			continue
+		}
+		lineEmpty = lineEmpty && unicode.IsSpace(b)
+	}
+	firstLine := input[firstLineStart:firstLineEnd]
+
+	// see what kind of front matter there is, if any
+	var closingFence string
+	var fmParser func([]byte) (map[string]interface{}, error)
+	switch string(firstLine) {
+	case yamlFrontMatterFenceOpen:
+		fmParser = yamlFrontMatter
+		closingFence = yamlFrontMatterFenceClose
+	case tomlFrontMatterFenceOpen:
+		fmParser = tomlFrontMatter
+		closingFence = tomlFrontMatterFenceClose
+	case jsonFrontMatterFenceOpen:
+		fmParser = jsonFrontMatter
+		closingFence = jsonFrontMatterFenceClose
+	default:
+		// no recognized front matter; whole document is body
+		return nil, input, nil
+	}
+
+	// find end of front matter
+	fmEndFenceStart := strings.Index(input[firstLineEnd:], "\n"+closingFence)
+	if fmEndFenceStart < 0 {
+		return nil, "", fmt.Errorf("unterminated front matter")
+	}
+	fmEndFenceStart += firstLineEnd + 1 // add 1 to account for newline
+
+	// extract and parse front matter
+	frontMatter := input[firstLineEnd:fmEndFenceStart]
+	fm, err := fmParser([]byte(frontMatter))
+	if err != nil {
+		return nil, "", err
+	}
+
+	// the rest is the body
+	body := input[fmEndFenceStart+len(closingFence):]
+
+	return fm, body, nil
+}
+
+func yamlFrontMatter(input []byte) (map[string]interface{}, error) {
+	m := make(map[string]interface{})
+	err := yaml.Unmarshal(input, &m)
+	return m, err
+}
+
+func tomlFrontMatter(input []byte) (map[string]interface{}, error) {
+	m := make(map[string]interface{})
+	err := toml.Unmarshal(input, &m)
+	return m, err
+}
+
+func jsonFrontMatter(input []byte) (map[string]interface{}, error) {
+	input = append([]byte{'{'}, input...)
+	input = append(input, '}')
+	m := make(map[string]interface{})
+	err := json.Unmarshal(input, &m)
+	return m, err
+}
+
+type parsedMarkdownDoc struct {
+	Meta map[string]interface{} `json:"meta,omitempty"`
+	Body string                 `json:"body,omitempty"`
+}
+
+const (
+	yamlFrontMatterFenceOpen, yamlFrontMatterFenceClose = "---", "---"
+	tomlFrontMatterFenceOpen, tomlFrontMatterFenceClose = "+++", "+++"
+	jsonFrontMatterFenceOpen, jsonFrontMatterFenceClose = "{", "}"
+)

modules/caddyhttp/templates/templates.go

@@ -31,9 +31,18 @@ func init() {
 
 // Templates is a middleware which execute response bodies as templates.
 type Templates struct {
-	IncludeRoot string   `json:"include_root,omitempty"`
-	MIMETypes   []string `json:"mime_types,omitempty"`
-	Delimiters  []string `json:"delimiters,omitempty"`
+	// The root path from which to load files. Required if template functions
+	// accessing the file system are used (such as include). Default is
+	// `{http.vars.root}` if set, or current working directory otherwise.
+	FileRoot string `json:"file_root,omitempty"`
+
+	// The MIME types for which to render templates. It is important to use
+	// this if the route matchers do not exclude images or other binary files.
+	// Default is text/plain, text/markdown, and text/html.
+	MIMETypes []string `json:"mime_types,omitempty"`
+
+	// The template action delimiters.
+	Delimiters []string `json:"delimiters,omitempty"`
 }
 
 // CaddyModule returns the Caddy module information.
@@ -49,8 +58,8 @@ func (t *Templates) Provision(ctx caddy.Context) error {
 	if t.MIMETypes == nil {
 		t.MIMETypes = defaultMIMETypes
 	}
-	if t.IncludeRoot == "" {
-		t.IncludeRoot = "{http.vars.root}"
+	if t.FileRoot == "" {
+		t.FileRoot = "{http.vars.root}"
 	}
 	return nil
 }
@@ -100,10 +109,9 @@ func (t *Templates) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddy
 	rec.Header().Del("Last-Modified") // useless for dynamic content since it's always changing
 
 	// we don't know a way to guickly generate etag for dynamic content,
-	// but we can convert this to a weak etag to kind of indicate that
-	if etag := rec.Header().Get("Etag"); etag != "" {
-		rec.Header().Set("Etag", "W/"+etag)
-	}
+	// and weak etags still cause browsers to rely on it even after a
+	// refresh, so disable them until we find a better way to do this
+	rec.Header().Del("Etag")
 
 	rec.WriteResponse()
 
@@ -113,9 +121,9 @@ func (t *Templates) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddy
 // executeTemplate executes the template contained in wb.buf and replaces it with the results.
 func (t *Templates) executeTemplate(rr caddyhttp.ResponseRecorder, r *http.Request) error {
 	var fs http.FileSystem
-	if t.IncludeRoot != "" {
+	if t.FileRoot != "" {
 		repl := r.Context().Value(caddy.ReplacerCtxKey).(caddy.Replacer)
-		fs = http.Dir(repl.ReplaceAll(t.IncludeRoot, "."))
+		fs = http.Dir(repl.ReplaceAll(t.FileRoot, "."))
 	}
 
 	ctx := &templateContext{

modules/caddyhttp/templates/tplcontext.go

@@ -27,8 +27,13 @@ import (
 	"sync"
 
 	"github.com/Masterminds/sprig/v3"
+	"github.com/alecthomas/chroma/formatters/html"
 	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
-	"github.com/russross/blackfriday/v2"
+	"github.com/yuin/goldmark"
+	highlighting "github.com/yuin/goldmark-highlighting"
+	"github.com/yuin/goldmark/extension"
+	"github.com/yuin/goldmark/parser"
+	gmhtml "github.com/yuin/goldmark/renderer/html"
 )
 
 // templateContext is the templateContext with which HTTP templates are executed.
@@ -41,11 +46,18 @@ type templateContext struct {
 	config *Templates
 }
 
-// Include returns the contents of filename relative to the site root.
+// OriginalReq returns the original, unmodified, un-rewritten request as
+// it originally came in over the wire.
+func (c templateContext) OriginalReq() http.Request {
+	or, _ := c.Req.Context().Value(caddyhttp.OriginalRequestCtxKey).(http.Request)
+	return or
+}
+
+// funcInclude returns the contents of filename relative to the site root.
 // Note that included files are NOT escaped, so you should only include
 // trusted files. If it is not trusted, be sure to use escaping functions
 // in your template.
-func (c templateContext) Include(filename string, args ...interface{}) (template.HTML, error) {
+func (c templateContext) funcInclude(filename string, args ...interface{}) (template.HTML, error) {
 	if c.Root == nil {
 		return "", fmt.Errorf("root file system not specified")
 	}
@@ -75,11 +87,11 @@ func (c templateContext) Include(filename string, args ...interface{}) (template
 	return template.HTML(bodyBuf.String()), nil
 }
 
-// HTTPInclude returns the body of a virtual (lightweight) request
+// funcHTTPInclude returns the body of a virtual (lightweight) request
 // to the given URI on the same server. Note that included bodies
 // are NOT escaped, so you should only include trusted resources.
 // If it is not trusted, be sure to use escaping functions yourself.
-func (c templateContext) HTTPInclude(uri string) (template.HTML, error) {
+func (c templateContext) funcHTTPInclude(uri string) (template.HTML, error) {
 	// prevent virtual request loops by counting how many levels
 	// deep we are; and if we get too deep, return an error
 	recursionCount := 1
@@ -124,11 +136,22 @@ func (c templateContext) HTTPInclude(uri string) (template.HTML, error) {
 }
 
 func (c templateContext) executeTemplateInBuffer(tplName string, buf *bytes.Buffer) error {
-	tpl := template.New(tplName).Funcs(sprig.FuncMap())
+	tpl := template.New(tplName)
 	if len(c.config.Delimiters) == 2 {
 		tpl.Delims(c.config.Delimiters[0], c.config.Delimiters[1])
 	}
 
+	tpl.Funcs(sprigFuncMap)
+
+	tpl.Funcs(template.FuncMap{
+		"include":          c.funcInclude,
+		"httpInclude":      c.funcHTTPInclude,
+		"stripHTML":        c.funcStripHTML,
+		"markdown":         c.funcMarkdown,
+		"splitFrontMatter": c.funcSplitFrontMatter,
+		"listFiles":        c.funcListFiles,
+	})
+
 	parsedTpl, err := tpl.Parse(buf.String())
 	if err != nil {
 		return err
@@ -173,9 +196,9 @@ func (c templateContext) Host() (string, error) {
 	return host, nil
 }
 
-// StripHTML returns s without HTML tags. It is fairly naive
+// funcStripHTML returns s without HTML tags. It is fairly naive
 // but works with most valid HTML inputs.
-func (c templateContext) StripHTML(s string) string {
+func (c templateContext) funcStripHTML(s string) string {
 	var buf bytes.Buffer
 	var inTag, inQuotes bool
 	var tagStart int
@@ -206,15 +229,53 @@ func (c templateContext) StripHTML(s string) string {
 	return buf.String()
 }
 
-// Markdown renders the markdown body as HTML. The resulting
+// funcMarkdown renders the markdown body as HTML. The resulting
 // HTML is NOT escaped so that it can be rendered as HTML.
-func (c templateContext) Markdown(body string) template.HTML {
-	return template.HTML(blackfriday.Run([]byte(body)))
+func (c templateContext) funcMarkdown(input interface{}) (template.HTML, error) {
+	inputStr := toString(input)
+
+	md := goldmark.New(
+		goldmark.WithExtensions(
+			extension.GFM,
+			extension.Table,
+			highlighting.NewHighlighting(
+				highlighting.WithFormatOptions(
+					html.WithClasses(true),
+				),
+			),
+		),
+		goldmark.WithParserOptions(
+			parser.WithAutoHeadingID(),
+		),
+		goldmark.WithRendererOptions(
+			gmhtml.WithHardWraps(),
+			gmhtml.WithUnsafe(), // TODO: this is not awesome, maybe should be configurable?
+		),
+	)
+
+	buf := bufPool.Get().(*bytes.Buffer)
+	buf.Reset()
+	defer bufPool.Put(buf)
+
+	md.Convert([]byte(inputStr), buf)
+
+	return template.HTML(buf.String()), nil
 }
 
-// ListFiles reads and returns a slice of names from the given
+// splitFrontMatter parses front matter out from the beginning of input,
+// and returns the separated key-value pairs and the body/content. input
+// must be a "stringy" value.
+func (c templateContext) funcSplitFrontMatter(input interface{}) (parsedMarkdownDoc, error) {
+	meta, body, err := extractFrontMatter(toString(input))
+	if err != nil {
+		return parsedMarkdownDoc{}, err
+	}
+	return parsedMarkdownDoc{Meta: meta, Body: body}, nil
+}
+
+// funcListFiles reads and returns a slice of names from the given
 // directory relative to the root of c.
-func (c templateContext) ListFiles(name string) ([]string, error) {
+func (c templateContext) funcListFiles(name string) ([]string, error) {
 	if c.Root == nil {
 		return nil, fmt.Errorf("root file system not specified")
 	}
@@ -273,10 +334,29 @@ func (h tplWrappedHeader) Del(field string) string {
 	return ""
 }
 
+func toString(input interface{}) string {
+	switch v := input.(type) {
+	case string:
+		return v
+	case template.HTML:
+		return string(v)
+	case fmt.Stringer:
+		return v.String()
+	case error:
+		return v.Error()
+	default:
+		return fmt.Sprintf("%s", input)
+	}
+}
+
 var bufPool = sync.Pool{
 	New: func() interface{} {
 		return new(bytes.Buffer)
 	},
 }
 
+// at time of writing, sprig.FuncMap() makes a copy, thus
+// involves iterating the whole map, so do it just once
+var sprigFuncMap = sprig.FuncMap()
+
 const recursionPreventionHeader = "Caddy-Templates-Include"

modules/caddyhttp/templates/tplcontext_test.go

@@ -31,6 +31,7 @@ package templates
 import (
 	"bytes"
 	"fmt"
+	"html/template"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -47,17 +48,20 @@ func TestMarkdown(t *testing.T) {
 
 	for i, test := range []struct {
 		body   string
-		expect string
+		expect template.HTML
 	}{
 		{
 			body:   "- str1\n- str2\n",
 			expect: "<ul>\n<li>str1</li>\n<li>str2</li>\n</ul>\n",
 		},
 	} {
-		result := string(context.Markdown(test.body))
+		result, err := context.funcMarkdown(test.body)
 		if result != test.expect {
 			t.Errorf("Test %d: expected '%s' but got '%s'", i, test.expect, result)
 		}
+		if err != nil {
+			t.Errorf("Test %d: got error: %v", i, result)
+		}
 	}
 }
 
@@ -180,7 +184,7 @@ func TestStripHTML(t *testing.T) {
 			expect: `<h1hi`,
 		},
 	} {
-		actual := context.StripHTML(test.input)
+		actual := context.funcStripHTML(test.input)
 		if actual != test.expect {
 			t.Errorf("Test %d: Expected %s, found %s. Input was StripHTML(%s)", i, test.expect, actual, test.input)
 		}
@@ -249,7 +253,7 @@ func TestFileListing(t *testing.T) {
 
 		// perform test
 		input := filepath.ToSlash(filepath.Join(filepath.Base(dirPath), test.inputBase))
-		actual, err := context.ListFiles(input)
+		actual, err := context.funcListFiles(input)
 		if err != nil {
 			if !test.shouldErr {
 				t.Errorf("Test %d: Expected no error, got: '%s'", i, err)