Entrypoint: Support for multiple pods and multiple stages

entrypoint/.gitignore

@@ -1,2 +1,4 @@
 bin/*
 srv/*
+.vscode/*
+**/.minio.sys/*

entrypoint/Makefile

@@ -2,9 +2,9 @@ my_dir := $(abspath $(shell dirname $(lastword $(MAKEFILE_LIST))))
 version = $(shell date +%Y-%m-%d).$(shell git rev-parse --short HEAD)~$(shell test -n "`git status -s`" && echo dirty || echo clean)
 cosa_dir = $(shell test -d /usr/lib/coreos-assembler && echo /usr/lib/coreos-assembler)
 ldflags=-X main.version=${version} -X main.cosaDir=${cosa_dir}
-test_tags := $(shell which minio 2> /dev/null &>1 && echo "ci,integration" || echo "ci")
 
-export PATH := $(my_dir)/bin:$(PATH)
+export PATH := $(my_dir)/bin:$(shell readlink -f ../tools/bin):$(PATH)
+
 PREFIX ?= /usr
 DESTDIR ?=
 ARCH:=$(shell uname -m)
@@ -17,21 +17,38 @@ build: test
 .PHONY: fmt
 fmt:
 	gofmt -d -e -l $(shell find . -iname "*.go"  -not -path "./vendor/*")
+	golangci-lint run -v  ./...
 
 .PHONY: test
 test: fmt
-	go test -mod=vendor -tags ${test_tags} -i ${pkgs} && \
-	go test -mod=vendor -tags ${test_tags} -cover ${pkgs}
+	go test -mod=vendor -tags ${test_tags} -v  -i ${pkgs} && \
+	go test -mod=vendor -tags ${test_tags} -v -cover ${pkgs}
 
 .PHONY: clean
 clean:
 	@go clean .
 	@rm -rf bin
 
+.PHYON: schema
+schema: schema_version = v1
+schema:
+	schematyper \
+	                ../src/schema/$(schema_version).json \
+	                -o cosa/v1.go \
+	                --package=cosa \
+	                --root-type=Build \
+	                --ptr-for-omit
+
 .PHONY: install
 install: bin/entry
 	install -v -D -t $(DESTDIR)$(PREFIX)/bin bin/entry
 
+.PHONY: go-deps
+go-deps:
+	go mod tidy
+	go mod vendor
+	go mod download
+
 my_uid = $(shell id -u)
 .PHONY: devtest
 devtest: build

entrypoint/cmd/build.go

@@ -6,10 +6,21 @@ package main
 	"builder" is the sub-command that should be used as an
 	the container entrypoint, i.e.:
 		/usr/bin/dumbinit /usr/bin/entry builder
+
+
+	Using the 'oc' binary was seriouslly considereed. However, the goals
+	were to:
+		1. Create and run concurrent pods
+		2. Capture the logs
+		3. Sanely clean-up after outselfs
+	Using the API just made more sense.
+
 */
 
 import (
 	"context"
+	"fmt"
+	"time"
 
 	"github.com/coreos/entrypoint/ocp"
 	log "github.com/sirupsen/logrus"
@@ -35,25 +46,30 @@ func init() {
 func runOCP(c *cobra.Command, args []string) {
 	defer cancel()
 
+	// Terminal "keep alive" helper. When following logs via the `oc` commands,
+	// cloud-deployed will send an EOF. To get around the EOF, the func sends a
+	// null character that is not printed to the screen or reflected in the logs.
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				fmt.Print("\x00")
+				time.Sleep(1 * time.Second)
+			}
+		}
+	}()
+
 	b, err := ocp.NewBuilder(ctx)
 	if err != nil {
 		log.Fatal("Failed to find the OCP build environment.")
 	}
 
-	if err := b.PrepareEnv(); err != nil {
-		log.WithFields(log.Fields{
-			"err": err,
-		}).Fatal("Failed to prepare environment.")
-	}
-
-	if b.JobSpec != nil {
-		spec = *b.JobSpec
-		log.Info("Jobspec will apply to templated commands.")
-	}
-
 	if err := b.Exec(ctx); err != nil {
 		log.WithFields(log.Fields{
 			"err": err,
 		}).Fatal("Failed to prepare environment.")
 	}
+
 }

entrypoint/cmd/entry.go

@@ -140,5 +140,5 @@ func preRun(c *cobra.Command, args []string) {
 		log.WithFields(log.Fields{"input file": specFile, "error": err}).Fatal(
 			"Failed reading file")
 	}
-	spec = *ns
+	spec = ns
 }

entrypoint/cosa/build.go

@@ -0,0 +1,195 @@
+// Copyright 2020 Red Hat, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cosa
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"reflect"
+	"runtime"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	// ErrMetaFailsValidation is thrown on reading and invalid meta.json
+	ErrMetaFailsValidation = errors.New("meta.json failed schema validation")
+
+	// ErrMetaNotFound is thrown when a meta.json cannot be found
+	ErrMetaNotFound = errors.New("meta.json was not found")
+)
+
+const (
+	// CosaMetaJSON is the meta.json file
+	CosaMetaJSON = "meta.json"
+)
+
+// BuilderArch converts the GOARCH to the build arch.
+// In other words, it translates amd64 to x86_64.
+func BuilderArch() string {
+	arch := runtime.GOARCH
+	if arch == "amd64" {
+		arch = "x86_64"
+	}
+	return arch
+}
+
+// ReadBuild returns a build upon finding a meta.json.
+// If build is "", use latest
+func ReadBuild(dir, buildID, arch string) (*Build, string, error) {
+	if arch == "" {
+		arch = BuilderArch()
+	}
+
+	if buildID == "" {
+		b, err := getBuilds(dir)
+		if err == nil {
+			latest, ok := b.getLatest(arch)
+			if !ok {
+				return nil, "", ErrNoBuildsFound
+			}
+			buildID = latest
+		}
+	}
+
+	if buildID == "" {
+		return nil, "", fmt.Errorf("build is undefined")
+	}
+
+	p := filepath.Join(dir, "builds", buildID, arch)
+	f, err := os.Open(filepath.Join(p, CosaMetaJSON))
+	if err != nil {
+		return nil, "", fmt.Errorf("failed to open %s to read meta.json: %w", p, err)
+	}
+
+	b, err := buildParser(f)
+	return b, p, err
+}
+
+func buildParser(r io.Reader) (*Build, error) {
+	dec := json.NewDecoder(r)
+	dec.DisallowUnknownFields()
+	var cosaBuild *Build
+	if err := dec.Decode(&cosaBuild); err != nil {
+		return nil, errors.Wrapf(err, "failed to parse build")
+	}
+	if errs := cosaBuild.Validate(); len(errs) > 0 {
+		return nil, errors.Wrapf(ErrMetaFailsValidation, "%v", errs)
+	}
+	return cosaBuild, nil
+}
+
+// ParseBuild parses the meta.json and reutrns a build
+func ParseBuild(path string) (*Build, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed to open %s", path)
+	}
+	defer f.Close()
+	b, err := buildParser(f)
+	if err != nil {
+		return nil, errors.Wrapf(err, "failed parsing of %s", path)
+	}
+	return b, err
+}
+
+// WriteMeta records the meta-data
+func (build *Build) WriteMeta(path string, validate bool) error {
+	if validate {
+		if err := build.Validate(); len(err) != 0 {
+			return errors.New("data is not compliant with schema")
+		}
+	}
+	out, err := json.MarshalIndent(build, "", "    ")
+	if err != nil {
+		return err
+	}
+	return ioutil.WriteFile(path, out, 0644)
+}
+
+// GetArtifact returns an artifact by JSON tag
+func (build *Build) GetArtifact(artifact string) (*Artifact, error) {
+	r, ok := build.artifacts()[artifact]
+	if ok {
+		return r, nil
+	}
+	return nil, errors.New("artifact not defined")
+}
+
+// IsArtifact takes a path and returns the artifact type and a bool if
+// the artifact is described in the build.
+func (build *Build) IsArtifact(path string) (string, bool) {
+	path = filepath.Base(path)
+	for k, v := range build.artifacts() {
+		if v.Path == path {
+			return k, true
+		}
+	}
+	return "", false
+}
+
+// CanArtifact reports whether an artifact name is buildable by COSA based
+// on the meta.json name. CanArtifact is used to signal if the artifact is a known
+// artifact type.
+func CanArtifact(artifact string) bool {
+	b := new(Build)
+	b.BuildArtifacts = new(BuildArtifacts)
+	_, ok := b.artifacts()[artifact]
+	return ok
+}
+
+// artifact returns a string map of artifacts, where the key
+// is the JSON tag. Reflection was over a case statement to make meta.json
+// and the schema authoritative for adding and removing artifacts.
+func (build *Build) artifacts() map[string]*Artifact {
+	ret := make(map[string]*Artifact)
+	var ba BuildArtifacts = *build.BuildArtifacts
+	rv := reflect.TypeOf(ba)
+	for i := 0; i < rv.NumField(); i++ {
+		tag := rv.Field(i).Tag.Get("json")
+		tag = strings.Split(tag, ",")[0]
+		field := reflect.ValueOf(&ba).Elem().Field(i)
+
+		// If the field is zero, then we create a stub artifact.
+		if field.IsZero() {
+			ret[strings.ToLower(tag)] = &Artifact{}
+			continue
+		}
+
+		// When the json struct tag does not have "omitempty"
+		// then we get an actual struct not the pointer.
+		if field.Kind() == reflect.Struct {
+			r, ok := field.Interface().(Artifact)
+			if ok {
+				ret[strings.ToLower(tag)] = &r
+			}
+			continue
+		}
+
+		// Optional structs (i.e. "omitempty") are pointers a struct.
+		if field.Addr().Elem().CanInterface() {
+			r, ok := reflect.ValueOf(&ba).Elem().Field(i).Elem().Interface().(Artifact)
+			if ok {
+				ret[strings.ToLower(tag)] = &r
+			}
+		}
+	}
+	return ret
+}

entrypoint/cosa/builds.go

@@ -0,0 +1,55 @@
+package cosa
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+)
+
+const (
+	// CosaBuildsJSON is the COSA build.json file name
+	CosaBuildsJSON = "builds.json"
+)
+
+var (
+	// ErrNoBuildsFound is thrown when a build is missing
+	ErrNoBuildsFound = errors.New("no COSA builds found")
+)
+
+// buildsJSON represents the JSON that records the builds
+// TODO: this should be generated by a schema
+type buildsJSON struct {
+	SchemaVersion string `json:"schema-version"`
+	Builds        []struct {
+		ID     string   `json:"id"`
+		Arches []string `json:"arches"`
+	} `json:"builds"`
+	TimeStamp string `json:"timestamp"`
+}
+
+func getBuilds(dir string) (*buildsJSON, error) {
+	path := filepath.Join(dir, "builds", CosaBuildsJSON)
+	d, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, ErrNoBuildsFound
+	}
+	b := &buildsJSON{}
+	if err := json.Unmarshal(d, b); err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+// getLatest returns the latest build for the arch.
+func (b *buildsJSON) getLatest(arch string) (string, bool) {
+	for _, b := range b.Builds {
+		for _, a := range b.Arches {
+			if a == arch {
+				return b.ID, true
+			}
+		}
+	}
+	return "", false
+}