build.go

package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
)

var (
	gitMail = flag.String("git-author-mail", "cs3org-bot@hugo.labkode.com", "Git author mail")
	gitName = flag.String("git-author-name", "cs3org-bot", "Git author name")
	gitSSH  = flag.Bool("git-ssh", false, "Use git protocol instead of https for cloning repos")

	_only_build = flag.Bool("only-build", false, "Build all protos and languages but do not push to language repos")
	_all        = flag.Bool("all", false, "Compile, build and publish for all available languages, mean to be run in CI platform like Drone")

	_buildProto = flag.Bool("build-proto", false, "Compile Protobuf definitions")

	_buildGo = flag.Bool("build-go", false, "Build Go library")
	_pushGo  = flag.Bool("push-go", false, "Push Go library to github.com/cs3org/go-cs3apis")

	_buildPython = flag.Bool("build-python", false, "Build Python library")
	_pushPython  = flag.Bool("push-python", false, "Push Python library to github.com/cs3org/python-cs3apis")

	_buildJs = flag.Bool("build-js", false, "Build Js library")
	_pushJs  = flag.Bool("push-js", false, "Push Js library to github.com/cs3org/js-cs3apis")

	_buildNode = flag.Bool("build-node", false, "Build Node.js library")
	_pushNode  = flag.Bool("push-node", false, "Push Node.js library to github.com/cs3org/node-cs3apis")
)

func init() {
	flag.Parse()

	if *_all {
		*_buildProto = true
		*_buildGo = true
		*_buildPython = true
		*_buildJs = true
		*_buildNode = true

		*_pushGo = true
		*_pushPython = true
		*_pushJs = true
		*_pushNode = true
	}

	if *_only_build {
		*_buildProto = true
		*_buildGo = true
		*_buildPython = true
		*_buildJs = true
		*_buildNode = true
	}
}

func getProtoOS() string {
	switch runtime.GOOS {
	case "darwin":
		return "osx"
	case "linux":
		return "linux"
	default:
		panic("no build procedure for " + runtime.GOOS)
	}
}

func clone(repo, dir string) {
	repo = getRepo(repo) // get git or https repo location
	cmd := exec.Command("git", "clone", "--quiet", repo)
	cmd.Dir = dir
	run(cmd)
}

func checkout(branch, dir string) {
	// See https://stackoverflow.com/questions/26961371/switch-on-another-branch-create-if-not-exists-without-checking-if-already-exi
	cmd := exec.Command("bash", "-c", fmt.Sprintf("git checkout %s || git checkout -b %s", branch, branch))
	cmd.Dir = dir
	run(cmd)
}

func update(dir string) error {
	cmd := exec.Command("git", "pull", "--quiet")
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Dir = dir
	return cmd.Run()
}

func isRepoDirty(repo string) bool {
	cmd := exec.Command("git", "status", "-s")
	cmd.Dir = repo
	changes := runAndGet(cmd)
	if changes != "" {
		fmt.Println("repo is dirty")
		fmt.Println(changes)
	}
	return changes != ""
}

func getCommitID(dir string) string {
	if os.Getenv("BUILD_GIT_COMMIT") != "" {
		return os.Getenv("BUILD_GIT_COMMIT")
	}

	cmd := exec.Command("git", "rev-parse", "HEAD")
	cmd.Dir = dir
	commit := runAndGet(cmd)
	return commit
}

func getRepo(repo string) string {
	if *gitSSH {
		return fmt.Sprintf("git@github.com:%s", repo)
	}
	return fmt.Sprintf("https://github.com/%s", repo)
}

func commit(repo, msg string) {
	// set correct author name and mail
	cmd := exec.Command("git", "config", "user.email", *gitMail)
	cmd.Dir = repo
	run(cmd)

	cmd = exec.Command("git", "config", "user.name", *gitName)
	cmd.Dir = repo
	run(cmd)

	// check if repo is dirty
	if !isRepoDirty(repo) {
		// nothing to do
		return
	}

	cmd = exec.Command("git", "add", ".")
	cmd.Dir = repo
	run(cmd)

	cmd = exec.Command("git", "commit", "-m", msg)
	cmd.Dir = repo
	run(cmd)
}

func push(repo string) {
	protoBranch := getGitBranch(".")
	cmd := exec.Command("git", "push", "--set-upstream", "origin", protoBranch)
	cmd.Dir = repo
	run(cmd)
}

func getGitBranch(repo string) string {
	// check if branch is provided by env variable
	if os.Getenv("BUILD_GIT_BRANCH") != "" {
		return os.Getenv("BUILD_GIT_BRANCH")
	}

	// obtain branch from repo
	cmd := exec.Command("git", "rev-parse", "--abbrev-ref", "HEAD")
	cmd.Dir = repo
	branch := runAndGet(cmd)
	return branch
}

// getVersionFromGit returns a version string that identifies the currently
// checked out git commit.
func getVersionFromGit(repodir string) string {
	cmd := exec.Command("git", "describe",
		"--long", "--tags", "--dirty", "--always")
	cmd.Dir = repodir
	out, err := cmd.Output()
	if err != nil {
		panic(fmt.Sprintf("git describe returned error: %v\n", err))
	}

	version := strings.TrimSpace(string(out))
	return version
}

func run(cmd *exec.Cmd) {
	var b bytes.Buffer
	mw := io.MultiWriter(os.Stdout, &b)
	cmd.Stdout = mw
	cmd.Stderr = mw
	err := cmd.Run()
	fmt.Println(cmd.Dir, cmd.Args)
	fmt.Println(b.String())
	if err != nil {
		fmt.Println("ERROR: ", err.Error())
		os.Exit(1)
	}
}

func runAndGet(cmd *exec.Cmd) string {
	var b bytes.Buffer
	mw := io.MultiWriter(os.Stdout, &b)
	cmd.Stderr = mw
	out, err := cmd.Output()
	fmt.Println(cmd.Dir, cmd.Args)
	fmt.Println(b.String())
	if err != nil {
		fmt.Println("ERROR: ", err.Error())
		os.Exit(1)
	}
	return strings.TrimSpace(string(out))
}

// Works with Go 1.8+
// https://stackoverflow.com/questions/32649770/how-to-get-current-gopath-from-code
func getGoPath() string {
	gopath := os.Getenv("GOPATH")
	if gopath == "" {
		gopath = build.Default.GOPATH
	}
	return gopath
}

func sed(dir, suffix, old, new string) {
	err := filepath.Walk(dir, func(path string, f os.FileInfo, err error) error {
		if strings.HasSuffix(path, suffix) {
			data, err := ioutil.ReadFile(path)
			if err != nil {
				return err
			}
			newData := strings.ReplaceAll(string(data), old, new)
			err = ioutil.WriteFile(path, []byte(newData), 0)
			if err != nil {
				return err
			}
		}
		return nil
	})
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

func find(patterns ...string) []string {
	var files []string
	for _, p := range patterns {
		fs, err := filepath.Glob(p)
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		files = append(files, fs...)
	}
	return files
}

func findProtos() []string {
	return find("cs3/*/*.proto", "cs3/*/*/*.proto", "cs3/*/*/*/*.proto")
}

func findFolders() []string {
	var folders []string
	err := filepath.Walk("cs3",
		func(path string, info os.FileInfo, err error) error {
			if err != nil {
				fmt.Println(err)
				os.Exit(1)
			}
			if info.IsDir() {
				folders = append(folders, path)
			}
			return nil
		})
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	return folders
}

func buildProto() {
	dir := "."
	cmd := exec.Command("prototool", "compile", "--walk-timeout", "10s")
	cmd.Dir = dir
	run(cmd)

	cmd = exec.Command("protolock", "status")
	cmd.Dir = dir
	run(cmd)

	// lint
	cmd = exec.Command("prototool", "format", "-w", "--walk-timeout", "10s")
	cmd.Dir = dir
	run(cmd)
	cmd = exec.Command("prototool", "lint", "--walk-timeout", "10s")
	cmd.Dir = dir
	run(cmd)
	cmd = exec.Command("go", "run", "tools/check-license/check-license.go")
	cmd.Dir = dir
	run(cmd)

	os.RemoveAll("docs")
	os.MkdirAll("docs", 0755)

	files := findProtos()
	fmt.Println(files)

	args := []string{"--doc_out=./docs", "--doc_opt=html,index.html", "-I=.", "-I=./third_party"}
	args = append(args, files...)
	cmd = exec.Command("protoc", args...)
	run(cmd)
}

func buildGo() {

	// Remove build dir
	os.RemoveAll("build/go-cs3apis")
	os.MkdirAll("build", 0755)

	// Clone Go repo and set branch to current branch
	clone("cs3org/go-cs3apis", "build")
	protoBranch := getGitBranch(".")
	goBranch := getGitBranch("build/go-cs3apis")
	fmt.Printf("Proto branch: %s\nGo branch: %s\n", protoBranch, goBranch)

	if goBranch != protoBranch {
		checkout(protoBranch, "build/go-cs3apis")
	}

	// remove leftovers (existing defs)
	os.RemoveAll("build/go-cs3apis/cs3")

	cmd := exec.Command("prototool", "generate", "--walk-timeout", "10s")
	run(cmd)

	sed("build/go-cs3apis", ".go", "github.com/cs3org/go-cs3apis/build/go-cs3apis/cs3/", "github.com/cs3org/go-cs3apis/cs3/")

	if !isRepoDirty("build/go-cs3apis") {
		fmt.Println("Repo is clean, nothing to do")
	}

	// get proto repo commit id
	hash := getCommitID(".")
	repo := "build/go-cs3apis"
	msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
	commit(repo, msg)
}

func buildPython() {

	// Remove build dir
	os.RemoveAll("build/python-cs3apis")
	os.MkdirAll("build", 0755)

	// Clone Go repo and set branch to current branch
	clone("cs3org/python-cs3apis", "build")
	protoBranch := getGitBranch(".")
	buildBranch := getGitBranch("build/python-cs3apis")
	fmt.Printf("Proto branch: %s\nBuild branch: %s\n", protoBranch, buildBranch)

	if buildBranch != protoBranch {
		checkout(protoBranch, "build/python-cs3apis")
	}

	// remove leftovers (existing defs)
	os.RemoveAll("build/python-cs3apis/cs3")

	files := findProtos()

	args := []string{"-m", "grpc_tools.protoc", "--python_out=./build/python-cs3apis", "-I.", "-I./third_party", "--grpc_python_out=./build/python-cs3apis"}
	args = append(args, files...)
	cmd := exec.Command("python3", args...)
	run(cmd)

	modules := findFolders()

	var initFiles []string
	for _, f := range modules {
		initPy := fmt.Sprintf("%s/%s/%s", "build/python-cs3apis", f, "__init__.py")
		initFiles = append(initFiles, initPy)
	}

	cmd = exec.Command("touch", initFiles...)
	run(cmd)

	// get proto repo commit id
	hash := getCommitID(".")
	repo := "build/python-cs3apis"
	msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
	commit(repo, msg)
}

func buildJS() {
	// Remove build dir
	os.RemoveAll("build/js-cs3apis")
	os.MkdirAll("build", 0755)

	// Clone repo and set branch to current branch
	clone("cs3org/js-cs3apis", "build")
	protoBranch := getGitBranch(".")
	buildBranch := getGitBranch("build/js-cs3apis")
	fmt.Printf("Proto branch: %s\nBuild branch: %s\n", protoBranch, buildBranch)

	if buildBranch != protoBranch {
		checkout(protoBranch, "build/js-cs3apis")
	}

	// remove leftovers (existing defs)
	os.RemoveAll("build/js-cs3apis/cs3")

	files := findProtos()

	args := []string{"--js_out=import_style=commonjs:./build/js-cs3apis", "--grpc-web_out=import_style=commonjs,mode=grpcwebtext:./build/js-cs3apis/", "-I.", "-I./third_party"}
	args = append(args, files...)
	cmd := exec.Command("protoc", args...)
	run(cmd)

	// get proto repo commit id
	hash := getCommitID(".")
	repo := "build/js-cs3apis"
	msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
	commit(repo, msg)
}

func buildNode() {
	// Remove build dir
	os.RemoveAll("build/node-cs3apis")
	os.MkdirAll("build", 0755)

	// Clone repo and set branch to current branch
	clone("cs3org/node-cs3apis", "build")
	protoBranch := getGitBranch(".")
	buildBranch := getGitBranch("build/node-cs3apis")
	fmt.Printf("Proto branch: %s\nBuild branch: %s\n", protoBranch, buildBranch)

	if buildBranch != protoBranch {
		checkout(protoBranch, "build/node-cs3apis")
	}

	nodeProtocPlugin, err := exec.LookPath("grpc_tools_node_protoc_plugin")

	if err != nil {
		panic(fmt.Sprintf("grpc_tools_node_protoc_plugin binary not found in PATH: %v\n", err))
	}

	// remove leftovers (existing defs)
	os.RemoveAll("build/node-cs3apis/cs3")

	files := findProtos()

	args := []string{"--js_out=import_style=commonjs,binary:./build/node-cs3apis", "--grpc_out=./build/node-cs3apis/", "--plugin=protoc-gen-grpc=" + nodeProtocPlugin}
	args = append(args, files...)
	cmd := exec.Command("grpc_tools_node_protoc", args...)
	run(cmd)

	// get proto repo commit id
	hash := getCommitID(".")
	repo := "build/node-cs3apis"
	msg := "Synced to https://github.com/cs3org/cs3apis/tree/" + hash
	commit(repo, msg)
}

func pushPython() {
	push("build/python-cs3apis")
}

func pushGo() {
	push("build/go-cs3apis")
}

func pushJS() {
	push("build/js-cs3apis")
}

func pushNode() {
	push("build/node-cs3apis")
}

func main() {
	if *_buildProto {
		fmt.Println("Compiling and linting protobufs ...")
		buildProto()
	}

	if *_buildGo {
		fmt.Println("Building Go ...")
		buildGo()
	}

	if *_pushGo {
		fmt.Println("Pushing Go ...")
		pushGo()
	}

	if *_buildPython {
		fmt.Println("Building Python ...")
		buildPython()
	}

	if *_pushPython {
		fmt.Println("Pushing Python ...")
		pushPython()
	}

	if *_buildJs {
		fmt.Println("Building JS ...")
		buildJS()
	}

	if *_pushJs {
		fmt.Println("Pushing Js ...")
		pushJS()
	}

	if *_buildNode {
		fmt.Println("Building Node.js ...")
		buildNode()
	}

	if *_pushNode {
		fmt.Println("Pushing Node.js ...")
		pushNode()
	}
}