Skip to content

Commit

Permalink
Refactor artifacts logic
Browse files Browse the repository at this point in the history
In particular this makes the distinction between the artifacts dir and
the run dir (but doesn't change the logic yet), but also allows us to
download and run ginkgo without requiring it to be in the PATH (if
ARTIFACTS is not set).
  • Loading branch information
justinsb committed Feb 19, 2021
1 parent 401a2f2 commit 7f5e6db
Show file tree
Hide file tree
Showing 9 changed files with 127 additions and 66 deletions.
4 changes: 2 additions & 2 deletions kubetest2-gce/deployer/deployer.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,8 +92,8 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) {
Strategy: "make",
},
},
kubeconfigPath: filepath.Join(opts.ArtifactsDir(), "kubetest2-kubeconfig"),
logsDir: filepath.Join(opts.ArtifactsDir(), "cluster-logs"),
kubeconfigPath: filepath.Join(opts.RunDir(), "kubetest2-kubeconfig"),
logsDir: filepath.Join(opts.RunDir(), "cluster-logs"),
boskosHeartbeatClose: make(chan struct{}),
instancePrefix: "kubetest2",
network: "default",
Expand Down
2 changes: 1 addition & 1 deletion kubetest2-gke/deployer/deployer.go
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) {
UpOptions: &options.UpOptions{
NumClusters: 1,
},
localLogsDir: filepath.Join(opts.ArtifactsDir(), "logs"),
localLogsDir: filepath.Join(opts.RunDir(), "logs"),
// Leave Version as empty to use the default cluster version.
Version: "",
}
Expand Down
2 changes: 1 addition & 1 deletion kubetest2-kind/deployer/deployer.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func New(opts types.Options) (types.Deployer, *pflag.FlagSet) {
// create a deployer object and set fields that are not flag controlled
d := &deployer{
commonOptions: opts,
logsDir: filepath.Join(opts.ArtifactsDir(), "logs"),
logsDir: filepath.Join(opts.RunDir(), "logs"),
}
// register flags and return
return d, bindFlags(d)
Expand Down
13 changes: 9 additions & 4 deletions pkg/app/app.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,13 +56,16 @@ func RealMain(opts types.Options, d types.Deployer, tester types.Tester) (result
*/
// TODO(bentheelder): signal handling & timeout

// ensure the artifacts dir
if err := os.MkdirAll(opts.ArtifactsDir(), os.ModePerm); err != nil {
klog.Infof("RunDir for this run: %q", opts.RunDir())

// ensure the run dir
if err := os.MkdirAll(opts.RunDir(), os.ModePerm); err != nil {
return err
}

// setup the metadata writer
junitRunner, err := os.Create(
filepath.Join(opts.ArtifactsDir(), "junit_runner.xml"),
filepath.Join(opts.RunDir(), "junit_runner.xml"),
)
if err != nil {
return errors.Wrap(err, "could not create runner output")
Expand Down Expand Up @@ -120,7 +123,9 @@ func RealMain(opts types.Options, d types.Deployer, tester types.Tester) (result
exec.InheritOutput(test)

envsForTester := os.Environ()
envsForTester = append(envsForTester, fmt.Sprintf("%s=%s", "ARTIFACTS", opts.ArtifactsDir()))
// We expose both ARIFACTS and KUBETEST2_RUN_DIR so we can more granular about caching vs output in future.
envsForTester = append(envsForTester, fmt.Sprintf("%s=%s", "ARTIFACTS", opts.RunDir()))
envsForTester = append(envsForTester, fmt.Sprintf("%s=%s", "KUBETEST2_RUN_DIR", opts.RunDir()))
envsForTester = append(envsForTester, fmt.Sprintf("%s=%s", "KUBETEST2_RUN_ID", opts.RunID()))
// If the deployer provides a kubeconfig pass it to the tester
// else assumes that it is handled offline by default methods like
Expand Down
51 changes: 12 additions & 39 deletions pkg/app/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ import (
"github.com/pkg/errors"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"k8s.io/klog"

"sigs.k8s.io/kubetest2/pkg/app/shim"
"sigs.k8s.io/kubetest2/pkg/artifacts"
"sigs.k8s.io/kubetest2/pkg/exec"
"sigs.k8s.io/kubetest2/pkg/types"
)
Expand Down Expand Up @@ -65,6 +65,7 @@ func runE(
opts := &options{}
kubetest2Flags := pflag.NewFlagSet(deployerName, pflag.ContinueOnError)
opts.bindFlags(kubetest2Flags)
artifacts.MustBindFlags(kubetest2Flags)

// NOTE: unknown flags are forwarded to the deployer as arguments
kubetest2Flags.ParseErrorsWhitelist.UnknownFlags = true
Expand Down Expand Up @@ -161,31 +162,10 @@ func runE(
return parseError
}

opts.artifacts = filepath.Join(opts.artifacts, opts.runid)

// run RealMain, which contains all of the logic beyond the CLI boilerplate
return RealMain(opts, deployer, tester)
}

// the default is $ARTIFACTS if set, otherwise ./_artifacts
// constructed as an absolute path to help the ginkgo tester because
// for some reason it needs an absolute path to the kubeconfig
func defaultArtifactsDir() (string, error) {
if path, set := os.LookupEnv("ARTIFACTS"); set {
absPath, err := filepath.Abs(path)
if err != nil {
return "", fmt.Errorf("failed to convert filepath from $ARTIFACTS (%s) to absolute path: %s", path, err)
}
return absPath, nil
}

absPath, err := filepath.Abs("_artifacts")
if err != nil {
return "", fmt.Errorf("when constructing default artifacts dir, failed to get absolute path: %s", err)
}
return absPath, nil
}

// splitArgs splits args into deployerArgs and testerArgs at the first bare `--`
func splitArgs(args []string) ([]string, []string) {
// first split into args and test args
Expand All @@ -204,13 +184,12 @@ func splitArgs(args []string) ([]string, []string) {

// options holds flag values and implements deployer.Options
type options struct {
help bool
build bool
up bool
down bool
test string
artifacts string
runid string
help bool
build bool
up bool
down bool
test string
runid string
}

// bindFlags registers all first class kubetest2 flags
Expand All @@ -229,12 +208,6 @@ func (o *options) bindFlags(flags *pflag.FlagSet) {
defaultRunID = uuid.New().String()
}
flags.StringVar(&o.runid, "run-id", defaultRunID, "unique identifier for a kubetest2 run")

defaultArtifacts, err := defaultArtifactsDir()
if err != nil {
klog.Fatalf("failed to get default artifacts directory: %s", err)
}
flags.StringVar(&o.artifacts, "artifacts", defaultArtifacts, `top-level directory to put artifacts under for each kubetest2 run, defaulting to "${ARTIFACTS:-./_artifacts}". If using the ginkgo tester, this must be an absolute path.`)
}

// assert that options implements deployer options
Expand All @@ -260,14 +233,14 @@ func (o *options) ShouldTest() bool {
return o.test != ""
}

func (o *options) ArtifactsDir() string {
return o.artifacts
}

func (o *options) RunID() string {
return o.runid
}

func (o *options) RunDir() string {
return filepath.Join(artifacts.BaseDir(), o.RunID())
}

// metadata used for CLI usage string
type usage struct {
kubetest2Flags *pflag.FlagSet
Expand Down
79 changes: 79 additions & 0 deletions pkg/artifacts/paths.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
/*
Copyright 2021 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package artifacts

import (
"fmt"
"os"
"path/filepath"

"github.com/spf13/pflag"
"k8s.io/klog"
)

var baseDir string

// BaseDir returns the path to the directory where artifacts should be written
// (including metadata files like junit_runner.xml)
func BaseDir() string {
d := baseDir
if d == "" {
def, err := defaultArtifactsDir()
if err != nil {
klog.Fatalf("failed to get default artifacts directory: %s", err)
}
d = def
}
return d
}

// the default is $ARTIFACTS if set, otherwise ./_artifacts
// constructed as an absolute path to help the ginkgo tester because
// for some reason it needs an absolute path to the kubeconfig
func defaultArtifactsDir() (string, error) {
if path, set := os.LookupEnv("ARTIFACTS"); set {
absPath, err := filepath.Abs(path)
if err != nil {
return "", fmt.Errorf("failed to convert filepath from $ARTIFACTS (%s) to absolute path: %s", path, err)
}
return absPath, nil
}

absPath, err := filepath.Abs("_artifacts")
if err != nil {
return "", fmt.Errorf("when constructing default artifacts dir, failed to get absolute path: %s", err)
}
return absPath, nil
}

// BindFlags binds the artifact related flags.
func BindFlags(flags *pflag.FlagSet) error {
defaultArtifacts, err := defaultArtifactsDir()
if err != nil {
return err
}
flags.StringVar(&baseDir, "artifacts", defaultArtifacts, `top-level directory to put artifacts under for each kubetest2 run, defaulting to "${ARTIFACTS:-./_artifacts}". If using the ginkgo tester, this must be an absolute path.`)
return nil
}

// MustBindFlags calls BindFlags and panics on an error
func MustBindFlags(flags *pflag.FlagSet) {
err := BindFlags(flags)
if err != nil {
klog.Fatalf("failed to get default artifacts directory: %s", err)
}
}
19 changes: 9 additions & 10 deletions pkg/testers/ginkgo/ginkgo.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,10 @@ import (
"github.com/octago/sflags/gen/gpflag"
"k8s.io/klog"

"sigs.k8s.io/kubetest2/pkg/artifacts"
"sigs.k8s.io/kubetest2/pkg/exec"
)

var (
// These paths are set up by AcquireTestPackage()
e2eTestPath = filepath.Join(os.Getenv("ARTIFACTS"), "e2e.test")
binary = filepath.Join(os.Getenv("ARTIFACTS"), "ginkgo")
)

type Tester struct {
FlakeAttempts int `desc:"Make up to this many attempts to run each spec."`
GinkgoArgs string `desc:"Additional arguments supported by the ginkgo binary."`
Expand All @@ -48,6 +43,10 @@ type Tester struct {
TestArgs string `desc:"Additional arguments supported by the e2e test framework (https://godoc.org/k8s.io/kubernetes/test/e2e/framework#TestContextType)."`

kubeconfigPath string

// These paths are set up by AcquireTestPackage()
e2eTestPath string
ginkgoPath string
}

// Test runs the test
Expand All @@ -61,7 +60,7 @@ func (t *Tester) Test() error {
"--ginkgo.flakeAttempts=" + strconv.Itoa(t.FlakeAttempts),
"--ginkgo.skip=" + t.SkipRegex,
"--ginkgo.focus=" + t.FocusRegex,
"--report-dir=" + os.Getenv("ARTIFACTS"),
"--report-dir=" + artifacts.BaseDir(),
}
extraE2EArgs, err := shellquote.Split(t.TestArgs)
if err != nil {
Expand All @@ -76,12 +75,12 @@ func (t *Tester) Test() error {

ginkgoArgs := append(extraGingkoArgs,
"--nodes="+strconv.Itoa(t.Parallel),
e2eTestPath,
t.e2eTestPath,
"--")
ginkgoArgs = append(ginkgoArgs, e2eTestArgs...)

klog.V(0).Infof("Running ginkgo test as %s %+v", binary, ginkgoArgs)
cmd := exec.Command(binary, ginkgoArgs...)
klog.V(0).Infof("Running ginkgo test as %s %+v", t.ginkgoPath, ginkgoArgs)
cmd := exec.Command(t.ginkgoPath, ginkgoArgs...)
exec.InheritOutput(cmd)
return cmd.Run()
}
Expand Down
18 changes: 12 additions & 6 deletions pkg/testers/ginkgo/package.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"strings"

"k8s.io/klog"
"sigs.k8s.io/kubetest2/pkg/artifacts"
"sigs.k8s.io/kubetest2/pkg/exec"
)

Expand Down Expand Up @@ -72,7 +73,12 @@ func (t *Tester) AcquireTestPackage() error {
}

func (t *Tester) extractBinaries(downloadPath string) error {
// finally, search for the test package and extract it
// ensure the artifacts dir
if err := os.MkdirAll(artifacts.BaseDir(), os.ModePerm); err != nil {
return err
}

// Extract files from the test package
f, err := os.Open(downloadPath)
if err != nil {
return fmt.Errorf("failed to open downloaded tar at %s: %s", downloadPath, err)
Expand All @@ -87,17 +93,17 @@ func (t *Tester) extractBinaries(downloadPath string) error {
tarReader := tar.NewReader(gzf)

// Map of paths in archive to destination paths
t.e2eTestPath = filepath.Join(artifacts.BaseDir(), "e2e.test")
t.ginkgoPath = filepath.Join(artifacts.BaseDir(), "ginkgo")
extract := map[string]string{
"kubernetes/test/bin/e2e.test": e2eTestPath,
"kubernetes/test/bin/ginkgo": binary,
"kubernetes/test/bin/e2e.test": t.e2eTestPath,
"kubernetes/test/bin/ginkgo": t.ginkgoPath,
}
extracted := map[string]bool{}

for {
// Put this check before any break condition so we don't
// accidentally incorrectly error
if len(extracted) == len(extract) {
return nil
break
}

header, err := tarReader.Next()
Expand Down
5 changes: 2 additions & 3 deletions pkg/types/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -54,11 +54,10 @@ type Options interface {
ShouldDown() bool
// if this is true, kubetest2 will be calling tester.Test
ShouldTest() bool
// returns the path to the directory where artifacts should be written
// (including metadata files like junit_runner.xml)
ArtifactsDir() string
// RunID returns a unique identifier for a kubetest2 run.
RunID() string
// RunDir returns the directory to put run-specific output files.
RunDir() string
}

// Deployer defines the interface between kubetest and a deployer
Expand Down

0 comments on commit 7f5e6db

Please sign in to comment.