feat: automate CLI and HTTP API doc generation (#875)

* move http-api-docs here
* add automation to regenerate api on go-ipfs tag
* add cli docs generation
* refactor: pin third-party action
* style: update-on-new-ipfs-tag
* chore: schedule/cron

Co-authored-by: Marcin Rataj <lidel@lidel.org>

.github/actions/latest-ipfs-tag/Dockerfile

@@ -0,0 +1,3 @@
+FROM golang:1.17
+COPY entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/latest-ipfs-tag/action.yml

@@ -0,0 +1,7 @@
+name: 'Find latest go-ipfs tag'
+outputs:
+  latest_tag:
+    description: "latest go-ipfs tag name"
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/actions/latest-ipfs-tag/entrypoint.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env sh
+set -eu
+
+# extract IPFS release
+cd /tmp
+git clone https://github.com/ipfs/go-ipfs.git
+cd go-ipfs
+LATEST_IPFS_TAG=`git describe --tags --abbrev=0`
+echo "The latest IPFS tag is ${LATEST_IPFS_TAG}"
+echo "::set-output name=latest_tag::${LATEST_IPFS_TAG}"

.github/actions/update-on-new-ipfs-tag/Dockerfile

@@ -0,0 +1,3 @@
+FROM golang:1.17
+COPY entrypoint.sh /entrypoint.sh
+ENTRYPOINT ["/entrypoint.sh"]

.github/actions/update-on-new-ipfs-tag/action.yml

@@ -0,0 +1,11 @@
+name: 'Update on new go-ipfs tag'
+inputs:
+  latest_ipfs_tag:
+      description: "latest go ipfs tag"
+      required: true
+outputs:
+  updated_branch:
+    description: "name of pushed branch with updated doc"
+runs:
+  using: 'docker'
+  image: 'Dockerfile'

.github/actions/update-on-new-ipfs-tag/entrypoint.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env sh
+set -eu
+
+API_FILE=`pwd`/docs/reference/http/api.md
+ROOT=`pwd`
+cd tools/http-api-docs
+
+# extract go-ipfs release tag used in http-api-docs from go.mod in this repo
+CURRENT_IPFS_TAG=`grep 'github.com/ipfs/go-ipfs ' ./go.mod | awk '{print $2}'`
+echo "The currently used go-ipfs tag in http-api-docs is ${CURRENT_IPFS_TAG}"
+
+# extract IPFS release
+LATEST_IPFS_TAG=$INPUT_LATEST_IPFS_TAG
+echo "The latest IPFS tag is ${LATEST_IPFS_TAG}"
+
+# make the upgrade, if newer go-ipfs tags exist
+if [ "$CURRENT_IPFS_TAG" = "$LATEST_IPFS_TAG" ]; then
+    echo "http-api-docs already uses the latest go-ipfs tag."
+else
+     # update http-api-docs
+     git checkout -b bump-http-api-docs-ipfs-to-$LATEST_IPFS_TAG
+     sed "s/^\s*github.com\/ipfs\/go-ipfs\s\+$CURRENT_IPFS_TAG\s*$/	github.com\/ipfs\/go-ipfs $LATEST_IPFS_TAG/" go.mod > go.mod2
+     mv go.mod2 go.mod
+     go mod tidy
+     make
+     http-api-docs > $API_FILE
+
+     # update cli docs
+     cd $ROOT # go back to root of ipfs-docs repo
+     git clone https://github.com/ipfs/go-ipfs.git
+     cd go-ipfs
+     git fetch --all --tags
+     git checkout tags/$LATEST_IPFS_TAG
+     go install ./cmd/ipfs
+     cd $ROOT/docs/reference
+     ./generate-cli-docs.sh
+
+     # submit a PR
+     cd $ROOT # go back to root of ipfs-docs repo
+     git config --global user.email "${GITHUB_ACTOR}"
+     git config --global user.name "${GITHUB_ACTOR}@users.noreply.github.com"
+     git add -u
+     git commit -m "Bumped go-ipfs dependence of http-api-docs to tag $LATEST_IPFS_TAG."
+     git push -u origin bump-http-api-docs-ipfs-to-$LATEST_IPFS_TAG
+fi
+echo "::set-output name=updated_branch::bump-http-api-docs-ipfs-to-$LATEST_IPFS_TAG"

.github/workflows/update-on-new-ipfs-tag.yml

@@ -0,0 +1,29 @@
+name: Update docs on new ipfs-tag release
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '30 5,17 * * *' # run every day at 5:30am and 5:30pm UTC
+
+jobs:
+  update:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout ipfs-docs
+        uses: actions/checkout@v2
+      - name: Find latest go-ipfs tag
+        id: latest_ipfs
+        uses: ./.github/actions/latest-ipfs-tag
+      - name: Update http-api-docs
+        id: update
+        uses: ./.github/actions/update-on-new-ipfs-tag
+        with:
+          latest_ipfs_tag: ${{ steps.latest_ipfs.outputs.latest_tag }}
+      - name: pull-request  # don't create a pr if there was no new ipfs tag
+        uses: repo-sync/pull-request@65194d8015be7624d231796ddee1cd52a5023cb3 #v2.16
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          source_branch: ${{ steps.update.outputs.updated_branch }}
+          destination_branch: "main"
+          pr_title: "Bump CLI and HTTP API reference to go-ipfs ${{ steps.latest_ipfs.outputs.latest_tag }}"
+          pr_body: "Release Notes: https://github.com/ipfs/go-ipfs/releases/${{ steps.latest_ipfs.outputs.latest_tag }}"
+          pr_label: "needs/triage,P0"

tools/http-api-docs/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Hector Sanjuan
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tools/http-api-docs/Makefile

@@ -0,0 +1,3 @@
+all: install
+install:
+	GO111MODULE=on go install ./http-api-docs

tools/http-api-docs/README.md

@@ -0,0 +1,54 @@
+# http-api-docs
+
+[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://ipn.io)
+[![](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](http://ipfs.io/)
+[![standard-readme compliant](https://img.shields.io/badge/standard--readme-OK-green.svg?style=flat-square)](https://github.com/RichardLitt/standard-readme)
+[![Build Status](https://travis-ci.com/ipfs/http-api-docs.svg?branch=master)](https://travis-ci.org/ipfs/http-api-docs)
+
+> A generator for go-ipfs API endpoints documentation.
+
+Note: This is just the generator for the docs that are available on ipfs.io, which can be found here: https://docs.ipfs.io/reference/http/api/
+
+The original docs are written in Markdown format and are available for community contributions here: https://github.com/ipfs/ipfs-docs
+
+## Table of Contents
+
+- [Install](#install)
+- [Usage](#usage)
+- [Captain](#captain)
+- [Contribute](#contribute)
+- [License](#license)
+
+## Install
+
+In order to build this project, you need to first install Go, clone this repo, and finally run `make install`:
+
+```sh
+> git clone https://github.com/ipfs/http-api-docs "$(go env GOPATH)/src/github.com/ipfs/http-api-docs"
+> cd "$(go env GOPATH)/src/github.com/ipfs/http-api-docs"
+> make install
+```
+
+## Usage
+
+After installing you can run:
+
+```
+> http-api-docs
+```
+
+This should spit out a Markdown document. This is exactly the `api.md` documentation at https://github.com/ipfs/ipfs-docs/blob/master/docs/reference/http/api.md, so you can redirect the output to just overwrite that file.
+
+## Captain
+
+This project is captained by @hsanjuan.
+
+## Contribute
+
+PRs accepted.
+
+Small note: If editing the README, please conform to the [standard-readme](https://github.com/RichardLitt/standard-readme) specification.
+
+## License
+
+MIT (C) Protocol Labs, Inc.

tools/http-api-docs/endpoints.go

@@ -0,0 +1,157 @@
+// Package docs can be used to gather go-ipfs commands and automatically
+// generate documentation or tests.
+package docs
+
+import (
+	"fmt"
+	"sort"
+
+	jsondoc "github.com/Stebalien/go-json-doc"
+	cid "github.com/ipfs/go-cid"
+	config "github.com/ipfs/go-ipfs"
+	cmds "github.com/ipfs/go-ipfs-cmds"
+	corecmds "github.com/ipfs/go-ipfs/core/commands"
+	peer "github.com/libp2p/go-libp2p-core/peer"
+	multiaddr "github.com/multiformats/go-multiaddr"
+)
+
+var JsondocGlossary = jsondoc.NewGlossary().
+	WithSchema(new(cid.Cid), jsondoc.Object{"/": "<cid-string>"}).
+	WithName(new(multiaddr.Multiaddr), "multiaddr-string").
+	WithName(new(peer.ID), "peer-id").
+	WithSchema(new(peer.AddrInfo),
+		jsondoc.Object{"ID": "peer-id", "Addrs": []string{"<multiaddr-string>"}})
+
+var ignoreOptsPerEndpoint = map[string]map[string]struct{}{
+	"/api/v0/add": {
+		cmds.RecLong:     struct{}{},
+		cmds.DerefLong:   struct{}{},
+		cmds.StdinName:   struct{}{},
+		cmds.Hidden:      struct{}{},
+		cmds.Ignore:      struct{}{},
+		cmds.IgnoreRules: struct{}{},
+	},
+}
+
+// A map of single endpoints to be skipped (subcommands are processed though).
+var IgnoreEndpoints = map[string]bool{}
+
+// How much to indent when generating the response schemas
+const IndentLevel = 4
+
+// Failsafe when traversing objects containing objects of the same type
+const MaxIndent = 20
+
+// Endpoint defines an IPFS RPC API endpoint.
+type Endpoint struct {
+	Name        string
+	Arguments   []*Argument
+	Options     []*Argument
+	Description string
+	Response    string
+	Group       string
+}
+
+// Argument defines an IPFS RPC API endpoint argument.
+type Argument struct {
+	Endpoint    string
+	Name        string
+	Description string
+	Type        string
+	Required    bool
+	Default     string
+}
+
+type sorter []*Endpoint
+
+func (a sorter) Len() int           { return len(a) }
+func (a sorter) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a sorter) Less(i, j int) bool { return a[i].Name < a[j].Name }
+
+const APIPrefix = "/api/v0"
+
+// AllEndpoints gathers all the endpoints from go-ipfs.
+func AllEndpoints() []*Endpoint {
+	return Endpoints(APIPrefix, corecmds.Root)
+}
+
+func IPFSVersion() string {
+	return config.CurrentVersionNumber
+}
+
+// Endpoints receives a name and a go-ipfs command and returns the endpoints it
+// defines] (sorted). It does this by recursively gathering endpoints defined by
+// subcommands. Thus, calling it with the core command Root generates all
+// the endpoints.
+func Endpoints(name string, cmd *cmds.Command) (endpoints []*Endpoint) {
+	var arguments []*Argument
+	var options []*Argument
+
+	ignore := cmd.Run == nil || IgnoreEndpoints[name]
+	if !ignore { // Extract arguments, options...
+		for _, arg := range cmd.Arguments {
+			argType := "string"
+			if arg.Type == cmds.ArgFile {
+				argType = "file"
+			}
+			arguments = append(arguments, &Argument{
+				Endpoint:    name,
+				Name:        arg.Name,
+				Type:        argType,
+				Required:    arg.Required,
+				Description: arg.Description,
+			})
+		}
+
+		for _, opt := range cmd.Options {
+			if ignoreOpts, ok := ignoreOptsPerEndpoint[name]; ok {
+				if _, ok := ignoreOpts[opt.Names()[0]]; ok {
+					// skip this option for this endpoint.
+					continue
+				}
+			}
+
+			def := fmt.Sprint(opt.Default())
+			if def == "<nil>" {
+				def = ""
+			}
+			options = append(options, &Argument{
+				Name:        opt.Names()[0],
+				Type:        opt.Type().String(),
+				Description: opt.Description(),
+				Default:     def,
+			})
+		}
+
+		res := buildResponse(cmd.Type)
+
+		endpoints = []*Endpoint{
+			{
+				Name:        name,
+				Description: cmd.Helptext.Tagline,
+				Arguments:   arguments,
+				Options:     options,
+				Response:    res,
+			},
+		}
+	}
+
+	for n, cmd := range cmd.Subcommands {
+		endpoints = append(endpoints,
+			Endpoints(fmt.Sprintf("%s/%s", name, n), cmd)...)
+	}
+	sort.Sort(sorter(endpoints))
+	return endpoints
+}
+
+func buildResponse(res interface{}) string {
+	// Commands with a nil type return text. This is a bad thing.
+	if res == nil {
+		return "This endpoint returns a `text/plain` response body."
+	}
+	desc, err := JsondocGlossary.Describe(res)
+	if err != nil {
+		panic(err)
+	}
+	return desc
+}

tools/http-api-docs/endpoints_test.go

@@ -0,0 +1,7 @@
+package docs
+
+import "testing"
+
+func TestEndpoints(t *testing.T) {
+	AllEndpoints()
+}

tools/http-api-docs/formatter.go

@@ -0,0 +1,30 @@
+package docs
+
+import "bytes"
+
+// Formatter allows to implement generation of docs in different formats.
+type Formatter interface {
+	GenerateIntro() string
+	GenerateIndex(endp []*Endpoint) string
+	GenerateEndpointBlock(endp *Endpoint) string
+	GenerateArgumentsBlock(args []*Argument, opts []*Argument) string
+	GenerateBodyBlock(args []*Argument) string
+	GenerateResponseBlock(response string) string
+	GenerateExampleBlock(endp *Endpoint) string
+}
+
+// GenerateDocs uses a formatter to generate documentation for every endpoint
+func GenerateDocs(api []*Endpoint, formatter Formatter) string {
+	buf := new(bytes.Buffer)
+	buf.WriteString(formatter.GenerateIntro())
+	// In docs.ipfs.io this is handled by the TOC.
+	// buf.WriteString(formatter.GenerateIndex(api))
+	for _, endp := range api {
+		buf.WriteString(formatter.GenerateEndpointBlock(endp))
+		buf.WriteString(formatter.GenerateArgumentsBlock(endp.Arguments, endp.Options))
+		buf.WriteString(formatter.GenerateBodyBlock(endp.Arguments))
+		buf.WriteString(formatter.GenerateResponseBlock(endp.Response))
+		buf.WriteString(formatter.GenerateExampleBlock(endp))
+	}
+	return buf.String()
+}