feat(readme): Render readme component as html

api/render.go

@@ -29,12 +29,23 @@ func (h *RenderHandlers) RenderHandler(w http.ResponseWriter, r *http.Request) {
 	}
 
 	p := &lib.RenderParams{
-		Ref:            HTTPPathToQriPath(r.URL.Path[len("/render"):]),
-		TemplateFormat: "html",
+		Ref:       HTTPPathToQriPath(r.URL.Path[len("/render"):]),
+		OutFormat: "html",
+	}
+
+	if r.FormValue("readme") == "true" {
+		p.UseFSI = r.FormValue("fsi") == "true"
+		var text string
+		if err := h.RenderReadme(p, &text); err != nil {
+			apiutil.WriteErrResponse(w, http.StatusInternalServerError, err)
+			return
+		}
+		w.Write([]byte(text))
+		return
 	}
 
 	data := []byte{}
-	if err := h.Render(p, &data); err != nil {
+	if err := h.RenderTemplate(p, &data); err != nil {
 		apiutil.WriteErrResponse(w, http.StatusInternalServerError, err)
 		return
 	}

api/render_test.go

@@ -2,6 +2,10 @@ package api
 
 import (
 	"testing"
+
+	"github.com/qri-io/dataset"
+	"github.com/qri-io/qri/lib"
+	"github.com/qri-io/qri/repo"
 )
 
 func TestRenderHandler(t *testing.T) {
@@ -16,3 +20,45 @@ func TestRenderHandler(t *testing.T) {
 	h := NewRenderHandlers(r)
 	runHandlerTestCases(t, "render", h.RenderHandler, cases, false)
 }
+
+func TestRenderReadmeHandler(t *testing.T) {
+	node, teardown := newTestNode(t)
+	defer teardown()
+
+	inst := newTestInstanceWithProfileFromNode(node)
+	h := NewRenderHandlers(inst.Repo())
+	dr := lib.NewDatasetRequests(node, nil)
+
+	// TODO(dlong): Copied from fsi_test, refactor into a common utility
+	saveParams := lib.SaveParams{
+		Ref: "me/render_readme_test",
+		Dataset: &dataset.Dataset{
+			Meta: &dataset.Meta{
+				Title: "title one",
+			},
+			Readme: &dataset.Readme{
+				ScriptBytes: []byte("# hi\n\ntest"),
+			},
+		},
+		BodyPath: "testdata/cities/data.csv",
+	}
+	res := repo.DatasetRef{}
+	if err := dr.Save(&saveParams, &res); err != nil {
+		t.Fatal(err)
+	}
+
+	// Checkout the dataset
+	actualStatusCode, actualBody := APICall(
+		"/render/peer/render_readme_test?readme=true",
+		h.RenderHandler)
+	if actualStatusCode != 200 {
+		t.Errorf("expected status code 200, got %d", actualStatusCode)
+	}
+	expectBody := `<h1>hi</h1>
+
+<p>test</p>
+`
+	if expectBody != actualBody {
+		t.Errorf("expected body {%s}, got {%s}", expectBody, actualBody)
+	}
+}

base/render_readme.go

@@ -0,0 +1,21 @@
+package base
+
+import (
+	"context"
+	"io/ioutil"
+
+	"github.com/microcosm-cc/bluemonday"
+	"github.com/qri-io/qfs"
+	"github.com/russross/blackfriday/v2"
+)
+
+// RenderReadme converts the markdown from the file into html.
+func RenderReadme(ctx context.Context, file qfs.File) (string, error) {
+	data, err := ioutil.ReadAll(file)
+	if err != nil {
+		return "", err
+	}
+	unsafe := blackfriday.Run(data)
+	htmlBytes := bluemonday.UGCPolicy().SanitizeBytes(unsafe)
+	return string(htmlBytes), nil
+}

base/render_readme_test.go

@@ -0,0 +1,38 @@
+package base
+
+import (
+	"context"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/qri-io/qfs"
+)
+
+func TestRenderReadme(t *testing.T) {
+	ctx := context.Background()
+
+	f := qfs.NewMemfileBytes("test.md", []byte(`# hi
+
+three things:
+
+* one
+* two
+* three`))
+	htmlStr, err := RenderReadme(ctx, f)
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectStr := `<h1>hi</h1>
+
+<p>three things:</p>
+
+<ul>
+<li>one</li>
+<li>two</li>
+<li>three</li>
+</ul>
+`
+	if diff := cmp.Diff(expectStr, htmlStr); diff != "" {
+		t.Errorf("body component (-want +got):\n%s", diff)
+	}
+}

cmd/render.go

@@ -15,21 +15,26 @@ func NewRenderCommand(f Factory, ioStreams ioes.IOStreams) *cobra.Command {
 	o := &RenderOptions{IOStreams: ioStreams}
 	cmd := &cobra.Command{
 		Use:   "render",
-		Short: "Execute a template against a dataset",
+		Short: "Render a dataset readme or a dataset template",
 		Long: `
-You can use html templates, formatted in the go/html template style, 
-to render visualizations from your dataset. These visualizations can be charts, 
-graphs, or just display your dataset in a different format.
+Render a dataset either by converting its readme from markdown to
+html, or by filling in a template using the go/html template style.
 
 Use the ` + "`--output`" + ` flag to save the rendered html to a file.
 
 Use the ` + "`--template`" + ` flag to use a custom template. If no template is
-provided, Qri will render the dataset with a default template.`,
+provided, Qri will render the dataset with a default template.
+
+Use the ` + "`--readme`" + ` flag to convert the dataset's readme
+component from markdown to html.`,
 		Example: `  render a dataset called me/schools:
   $ qri render -o=schools.html me/schools
 
   render a dataset with a custom template:
-  $ qri render --template=template.html me/schools`,
+  $ qri render --template=template.html me/schools
+
+  render a dataset's readme file:
+  $ qri render --readme me/schools`,
 		Annotations: map[string]string{
 			"group": "dataset",
 		},
@@ -42,6 +47,7 @@ provided, Qri will render the dataset with a default template.`,
 	}
 
 	cmd.Flags().StringVarP(&o.Template, "template", "t", "", "path to template file")
+	cmd.Flags().BoolVarP(&o.UseReadme, "readme", "r", false, "whether to use the readme component")
 	cmd.Flags().StringVarP(&o.Output, "output", "o", "", "path to write output file")
 
 	return cmd
@@ -51,9 +57,10 @@ provided, Qri will render the dataset with a default template.`,
 type RenderOptions struct {
 	ioes.IOStreams
 
-	Refs     *RefSelect
-	Template string
-	Output   string
+	Refs      *RefSelect
+	Template  string
+	UseReadme bool
+	Output    string
 
 	RenderRequests *lib.RenderRequests
 }
@@ -70,9 +77,21 @@ func (o *RenderOptions) Complete(f Factory, args []string) (err error) {
 }
 
 // Run executes the render command
-func (o *RenderOptions) Run() (err error) {
-	var template []byte
+func (o *RenderOptions) Run() error {
+	if o.Template != "" && o.UseReadme {
+		return fmt.Errorf("can not specify both --template and --readme flags")
+	}
+
+	if o.UseReadme {
+		return o.RunReadmeRender()
+	}
 
+	return o.RunTemplateRender()
+}
+
+// RunTemplateRender renders a dataset template as html
+func (o *RenderOptions) RunTemplateRender() (err error) {
+	var template []byte
 	if o.Template != "" {
 		template, err = ioutil.ReadFile(o.Template)
 		if err != nil {
@@ -81,13 +100,13 @@ func (o *RenderOptions) Run() (err error) {
 	}
 
 	p := &lib.RenderParams{
-		Ref:            o.Refs.Ref(),
-		Template:       template,
-		TemplateFormat: "html",
+		Ref:       o.Refs.Ref(),
+		Template:  template,
+		OutFormat: "html",
 	}
 
 	res := []byte{}
-	if err = o.RenderRequests.Render(p, &res); err != nil {
+	if err := o.RenderRequests.RenderTemplate(p, &res); err != nil {
 		if err == repo.ErrEmptyRef {
 			return lib.NewError(err, "peername and dataset name needed in order to render, for example:\n   $ qri render me/dataset_name\nsee `qri render --help` from more info")
 		}
@@ -101,3 +120,26 @@ func (o *RenderOptions) Run() (err error) {
 	}
 	return nil
 }
+
+// RunReadmeRender renders a readme file as html
+func (o *RenderOptions) RunReadmeRender() error {
+	printRefSelect(o.Out, o.Refs)
+
+	p := &lib.RenderParams{
+		Ref:       o.Refs.Ref(),
+		UseFSI:    o.Refs.IsLinked(),
+		OutFormat: "html",
+	}
+
+	var res string
+	if err := o.RenderRequests.RenderReadme(p, &res); err != nil {
+		return err
+	}
+
+	if o.Output == "" {
+		fmt.Fprint(o.Out, res)
+	} else {
+		ioutil.WriteFile(o.Output, []byte(res), 0777)
+	}
+	return nil
+}

go.mod

@@ -36,7 +36,10 @@ require (
 	github.com/libp2p/go-libp2p-core v0.2.3
 	github.com/libp2p/go-libp2p-peerstore v0.1.3
 	github.com/libp2p/go-libp2p-swarm v0.2.2
+	github.com/microcosm-cc/bluemonday v1.0.2
 	github.com/mitchellh/go-homedir v1.1.0
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
 	github.com/mr-tron/base58 v1.1.2
 	github.com/multiformats/go-multiaddr v0.1.1
 	github.com/multiformats/go-multicodec v0.1.6
@@ -52,7 +55,10 @@ require (
 	github.com/qri-io/qfs v0.1.1-0.20191025195012-9971677b190d
 	github.com/qri-io/starlib v0.4.2-0.20191025202035-0f16a7d50967
 	github.com/qri-io/varName v0.1.0
+	github.com/russross/blackfriday v1.5.2
+	github.com/russross/blackfriday/v2 v2.0.2-0.20190629151518-3e56bb68c887
 	github.com/sergi/go-diff v1.0.0
+	github.com/shurcooL/sanitized_anchor_name v1.0.0 // indirect
 	github.com/sirupsen/logrus v1.2.0
 	github.com/spf13/cobra v0.0.5
 	github.com/theckman/go-flock v0.7.1

go.sum

@@ -602,6 +602,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4=
 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
+github.com/microcosm-cc/bluemonday v1.0.2 h1:5lPfLTTAvAbtS0VqT+94yOtFnGfUWYyx0+iToC3Os3s=
+github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
 github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.12 h1:WMhc1ik4LNkTg8U9l3hI1LvxKmIL+f1+WV/SZtCbDDA=
 github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -736,8 +738,14 @@ github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday v2.0.0+incompatible h1:cBXrhZNUf9C+La9/YpS+UHpUT8YD6Td9ZMSU9APFcsk=
+github.com/russross/blackfriday v2.0.0+incompatible/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday/v2 v2.0.2-0.20190629151518-3e56bb68c887 h1:qTeswcWdSOibMXeqd4WQHd4+DyHeyheZuOy3dOt4WUU=
+github.com/russross/blackfriday/v2 v2.0.2-0.20190629151518-3e56bb68c887/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
@@ -875,6 +883,7 @@ golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190227160552-c95aed5357e7/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

lib/render.go

@@ -5,7 +5,10 @@ import (
 	"fmt"
 	"net/rpc"
 
+	"github.com/qri-io/dataset"
 	"github.com/qri-io/qri/base"
+	"github.com/qri-io/qri/base/dsfs"
+	"github.com/qri-io/qri/fsi"
 	"github.com/qri-io/qri/repo"
 )
 
@@ -35,15 +38,16 @@ func (RenderRequests) CoreRequestsName() string { return "render" }
 
 // RenderParams defines parameters for the Render method
 type RenderParams struct {
-	Ref            string
-	Template       []byte
-	TemplateFormat string
+	Ref       string
+	Template  []byte
+	UseFSI    bool
+	OutFormat string
 }
 
-// Render executes a template against a template
-func (r *RenderRequests) Render(p *RenderParams, res *[]byte) (err error) {
+// RenderTemplate executes a template against a dataset
+func (r *RenderRequests) RenderTemplate(p *RenderParams, res *[]byte) (err error) {
 	if r.cli != nil {
-		return r.cli.Call("RenderRequests.Render", p, res)
+		return r.cli.Call("RenderRequests.RenderTemplate", p, res)
 	}
 	ctx := context.TODO()
 
@@ -61,3 +65,39 @@ func (r *RenderRequests) Render(p *RenderParams, res *[]byte) (err error) {
 	*res, err = base.Render(ctx, r.repo, ref, p.Template)
 	return err
 }
+
+// RenderReadme renders the readme into html for the given dataset
+func (r *RenderRequests) RenderReadme(p *RenderParams, res *string) (err error) {
+	if r.cli != nil {
+		return r.cli.Call("RenderRequests.RenderReadme", p, res)
+	}
+	ctx := context.TODO()
+
+	ref, err := base.ToDatasetRef(p.Ref, r.repo, p.UseFSI)
+	if err != nil {
+		return err
+	}
+
+	var ds *dataset.Dataset
+	if p.UseFSI {
+		if ref.FSIPath == "" {
+			return fsi.ErrNoLink
+		}
+		if ds, err = fsi.ReadDir(ref.FSIPath); err != nil {
+			return fmt.Errorf("loading linked dataset: %s", err)
+		}
+	} else {
+		ds, err = dsfs.LoadDataset(ctx, r.repo.Store(), ref.Path)
+		if err != nil {
+			return fmt.Errorf("loading dataset: %s", err)
+		}
+	}
+
+	err = ds.Readme.OpenScriptFile(ctx, r.repo.Filesystem())
+	if err != nil {
+		return err
+	}
+
+	*res, err = base.RenderReadme(ctx, ds.Readme.ScriptFile())
+	return err
+}