Merge #2412 to simplify test loading and support integration tests

cmd/archive.go

@@ -23,13 +23,9 @@ package cmd
 import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
-
-	"go.k6.io/k6/errext"
-	"go.k6.io/k6/errext/exitcodes"
-	"go.k6.io/k6/lib/metrics"
 )
 
-func getArchiveCmd(globalState *globalState) *cobra.Command { // nolint: funlen
+func getArchiveCmd(gs *globalState) *cobra.Command {
 	archiveOut := "archive.tar"
 	// archiveCmd represents the archive command
 	archiveCmd := &cobra.Command{
@@ -46,60 +42,24 @@ An archive is a fully self-contained test run, and can be executed identically e
   k6 run myarchive.tar`[1:],
 		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
-			src, filesystems, err := readSource(globalState, args[0])
-			if err != nil {
-				return err
-			}
-
-			runtimeOptions, err := getRuntimeOptions(cmd.Flags(), globalState.envVars)
-			if err != nil {
-				return err
-			}
-
-			registry := metrics.NewRegistry()
-			builtinMetrics := metrics.RegisterBuiltinMetrics(registry)
-			r, err := newRunner(
-				globalState.logger, src, globalState.flags.runType,
-				filesystems, runtimeOptions, builtinMetrics, registry,
-			)
-			if err != nil {
-				return err
-			}
-
-			cliOpts, err := getOptions(cmd.Flags())
-			if err != nil {
-				return err
-			}
-			conf, err := getConsolidatedConfig(globalState, Config{Options: cliOpts}, r.GetOptions())
-			if err != nil {
-				return err
-			}
-
-			// Parse the thresholds, only if the --no-threshold flag is not set.
-			// If parsing the threshold expressions failed, consider it as an
-			// invalid configuration error.
-			if !runtimeOptions.NoThresholds.Bool {
-				for _, thresholds := range conf.Options.Thresholds {
-					err = thresholds.Parse()
-					if err != nil {
-						return errext.WithExitCodeIfNone(err, exitcodes.InvalidConfig)
-					}
-				}
-			}
-
-			_, err = deriveAndValidateConfig(conf, r.IsExecutable, globalState.logger)
+			test, err := loadAndConfigureTest(gs, cmd, args, getPartialConfig)
 			if err != nil {
 				return err
 			}
 
-			err = r.SetOptions(conf.Options)
+			// It's important to NOT set the derived options back to the runner
+			// here, only the consolidated ones. Otherwise, if the script used
+			// an execution shortcut option (e.g. `iterations` or `duration`),
+			// we will have multiple conflicting execution options since the
+			// derivation will set `scenarios` as well.
+			err = test.initRunner.SetOptions(test.consolidatedConfig.Options)
 			if err != nil {
 				return err
 			}
 
 			// Archive.
-			arc := r.MakeArchive()
-			f, err := globalState.fs.Create(archiveOut)
+			arc := test.initRunner.MakeArchive()
+			f, err := gs.fs.Create(archiveOut)
 			if err != nil {
 				return err
 			}

cmd/archive_test.go

@@ -6,9 +6,7 @@ import (
 	"testing"
 
 	"github.com/spf13/afero"
-	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
-	"go.k6.io/k6/errext"
 	"go.k6.io/k6/errext/exitcodes"
 )
 
@@ -51,21 +49,10 @@ func TestArchiveThresholds(t *testing.T) {
 				testState.args = append(testState.args, "--no-thresholds")
 			}
 
-			gotErr := newRootCommand(testState.globalState).cmd.Execute()
-
-			assert.Equal(t,
-				testCase.wantErr,
-				gotErr != nil,
-				"archive command error = %v, wantErr %v", gotErr, testCase.wantErr,
-			)
-
 			if testCase.wantErr {
-				var gotErrExt errext.HasExitCode
-				require.ErrorAs(t, gotErr, &gotErrExt)
-				assert.Equalf(t, exitcodes.InvalidConfig, gotErrExt.ExitCode(),
-					"status code must be %d", exitcodes.InvalidConfig,
-				)
+				testState.expectedExitCode = int(exitcodes.InvalidConfig)
 			}
+			newRootCommand(testState.globalState).execute()
 		})
 	}
 }

cmd/cloud.go

@@ -30,7 +30,6 @@ import (
 	"path/filepath"
 	"strconv"
 	"sync"
-	"syscall"
 	"time"
 
 	"github.com/fatih/color"
@@ -42,7 +41,6 @@ import (
 	"go.k6.io/k6/errext/exitcodes"
 	"go.k6.io/k6/lib"
 	"go.k6.io/k6/lib/consts"
-	"go.k6.io/k6/lib/metrics"
 	"go.k6.io/k6/ui/pb"
 )
 
@@ -91,56 +89,23 @@ This will execute the test on the k6 cloud service. Use "k6 login cloud" to auth
 		RunE: func(cmd *cobra.Command, args []string) error {
 			printBanner(globalState)
 
-			logger := globalState.logger
 			progressBar := pb.New(
 				pb.WithConstLeft("Init"),
-				pb.WithConstProgress(0, "Parsing script"),
+				pb.WithConstProgress(0, "Loading test script..."),
 			)
 			printBar(globalState, progressBar)
 
-			// Runner
-			filename := args[0]
-			src, filesystems, err := readSource(globalState, filename)
-			if err != nil {
-				return err
-			}
-
-			runtimeOptions, err := getRuntimeOptions(cmd.Flags(), globalState.envVars)
-			if err != nil {
-				return err
-			}
-
-			modifyAndPrintBar(globalState, progressBar, pb.WithConstProgress(0, "Getting script options"))
-			registry := metrics.NewRegistry()
-			builtinMetrics := metrics.RegisterBuiltinMetrics(registry)
-			r, err := newRunner(logger, src, globalState.flags.runType, filesystems, runtimeOptions, builtinMetrics, registry)
-			if err != nil {
-				return err
-			}
-
-			modifyAndPrintBar(globalState, progressBar, pb.WithConstProgress(0, "Consolidating options"))
-			cliOpts, err := getOptions(cmd.Flags())
-			if err != nil {
-				return err
-			}
-			conf, err := getConsolidatedConfig(globalState, Config{Options: cliOpts}, r.GetOptions())
+			test, err := loadAndConfigureTest(globalState, cmd, args, getPartialConfig)
 			if err != nil {
 				return err
 			}
 
-			// Parse the thresholds, only if the --no-threshold flag is not set.
-			// If parsing the threshold expressions failed, consider it as an
-			// invalid configuration error.
-			if !runtimeOptions.NoThresholds.Bool {
-				for _, thresholds := range conf.Options.Thresholds {
-					err = thresholds.Parse()
-					if err != nil {
-						return errext.WithExitCodeIfNone(err, exitcodes.InvalidConfig)
-					}
-				}
-			}
-
-			derivedConf, err := deriveAndValidateConfig(conf, r.IsExecutable, logger)
+			// It's important to NOT set the derived options back to the runner
+			// here, only the consolidated ones. Otherwise, if the script used
+			// an execution shortcut option (e.g. `iterations` or `duration`),
+			// we will have multiple conflicting execution options since the
+			// derivation will set `scenarios` as well.
+			err = test.initRunner.SetOptions(test.consolidatedConfig.Options)
 			if err != nil {
 				return err
 			}
@@ -149,13 +114,9 @@ This will execute the test on the k6 cloud service. Use "k6 login cloud" to auth
 			// TODO: validate for externally controlled executor (i.e. executors that aren't distributable)
 			// TODO: move those validations to a separate function and reuse validateConfig()?
 
-			err = r.SetOptions(conf.Options)
-			if err != nil {
-				return err
-			}
+			modifyAndPrintBar(globalState, progressBar, pb.WithConstProgress(0, "Building the archive..."))
+			arc := test.initRunner.MakeArchive()
 
-			modifyAndPrintBar(globalState, progressBar, pb.WithConstProgress(0, "Building the archive"))
-			arc := r.MakeArchive()
 			// TODO: Fix this
 			// We reuse cloud.Config for parsing options.ext.loadimpact, but this probably shouldn't be
 			// done, as the idea of options.ext is that they are extensible without touching k6. But in
@@ -174,7 +135,7 @@ This will execute the test on the k6 cloud service. Use "k6 login cloud" to auth
 
 			// Cloud config
 			cloudConfig, err := cloudapi.GetConsolidatedConfig(
-				derivedConf.Collectors["cloud"], globalState.envVars, "", arc.Options.External)
+				test.derivedConfig.Collectors["cloud"], globalState.envVars, "", arc.Options.External)
 			if err != nil {
 				return err
 			}
@@ -205,12 +166,14 @@ This will execute the test on the k6 cloud service. Use "k6 login cloud" to auth
 
 			name := cloudConfig.Name.String
 			if !cloudConfig.Name.Valid || cloudConfig.Name.String == "" {
-				name = filepath.Base(filename)
+				name = filepath.Base(test.sourceRootPath)
 			}
 
 			globalCtx, globalCancel := context.WithCancel(globalState.ctx)
 			defer globalCancel()
 
+			logger := globalState.logger
+
 			// Start cloud test run
 			modifyAndPrintBar(globalState, progressBar, pb.WithConstProgress(0, "Validating script options"))
 			client := cloudapi.NewClient(
@@ -226,13 +189,10 @@ This will execute the test on the k6 cloud service. Use "k6 login cloud" to auth
 			}
 
 			// Trap Interrupts, SIGINTs and SIGTERMs.
-			sigC := make(chan os.Signal, 2)
-			globalState.signalNotify(sigC, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
-			defer globalState.signalStop(sigC)
-			go func() {
-				sig := <-sigC
+			gracefulStop := func(sig os.Signal) {
 				logger.WithField("sig", sig).Print("Stopping cloud test run in response to signal...")
-				// Do this in a separate goroutine so that if it blocks the second signal can stop the execution
+				// Do this in a separate goroutine so that if it blocks, the
+				// second signal can still abort the process execution.
 				go func() {
 					stopErr := client.StopCloudTestRun(refID)
 					if stopErr != nil {
@@ -242,20 +202,21 @@ This will execute the test on the k6 cloud service. Use "k6 login cloud" to auth
 					}
 					globalCancel()
 				}()
-
-				sig = <-sigC
+			}
+			hardStop := func(sig os.Signal) {
 				logger.WithField("sig", sig).Error("Aborting k6 in response to signal, we won't wait for the test to end.")
-				os.Exit(int(exitcodes.ExternalAbort))
-			}()
+			}
+			stopSignalHandling := handleTestAbortSignals(globalState, gracefulStop, hardStop)
+			defer stopSignalHandling()
 
-			et, err := lib.NewExecutionTuple(derivedConf.ExecutionSegment, derivedConf.ExecutionSegmentSequence)
+			et, err := lib.NewExecutionTuple(test.derivedConfig.ExecutionSegment, test.derivedConfig.ExecutionSegmentSequence)
 			if err != nil {
 				return err
 			}
 			testURL := cloudapi.URLForResults(refID, cloudConfig)
-			executionPlan := derivedConf.Scenarios.GetFullExecutionRequirements(et)
+			executionPlan := test.derivedConfig.Scenarios.GetFullExecutionRequirements(et)
 			printExecutionDescription(
-				globalState, "cloud", filename, testURL, derivedConf, et, executionPlan, nil,
+				globalState, "cloud", test.sourceRootPath, testURL, test.derivedConfig, et, executionPlan, nil,
 			)
 
 			modifyAndPrintBar(

cmd/common.go

@@ -21,17 +21,16 @@
 package cmd
 
 import (
-	"archive/tar"
-	"bytes"
 	"fmt"
+	"os"
+	"syscall"
 
-	"github.com/spf13/afero"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"gopkg.in/guregu/null.v3"
 
+	"go.k6.io/k6/errext/exitcodes"
 	"go.k6.io/k6/lib/types"
-	"go.k6.io/k6/loader"
 )
 
 // Panic if the given error is not nil.
@@ -86,28 +85,41 @@ func exactArgsWithMsg(n int, msg string) cobra.PositionalArgs {
 	}
 }
 
-// readSource is a small wrapper around loader.ReadSource returning
-// result of the load and filesystems map
-func readSource(globalState *globalState, filename string) (*loader.SourceData, map[string]afero.Fs, error) {
-	pwd, err := globalState.getwd()
-	if err != nil {
-		return nil, nil, err
+func printToStdout(gs *globalState, s string) {
+	if _, err := fmt.Fprint(gs.stdOut, s); err != nil {
+		gs.logger.Errorf("could not print '%s' to stdout: %s", s, err.Error())
 	}
-
-	filesystems := loader.CreateFilesystems(globalState.fs)
-	src, err := loader.ReadSource(globalState.logger, filename, pwd, filesystems, globalState.stdIn)
-	return src, filesystems, err
 }
 
-func detectType(data []byte) string {
-	if _, err := tar.NewReader(bytes.NewReader(data)).Next(); err == nil {
-		return typeArchive
-	}
-	return typeJS
-}
+// Trap Interrupts, SIGINTs and SIGTERMs and call the given.
+func handleTestAbortSignals(gs *globalState, firstHandler, secondHandler func(os.Signal)) (stop func()) {
+	sigC := make(chan os.Signal, 2)
+	done := make(chan struct{})
+	gs.signalNotify(sigC, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
 
-func printToStdout(gs *globalState, s string) {
-	if _, err := fmt.Fprint(gs.stdOut, s); err != nil {
-		gs.logger.Errorf("could not print '%s' to stdout: %s", s, err.Error())
+	go func() {
+		select {
+		case sig := <-sigC:
+			firstHandler(sig)
+		case <-done:
+			return
+		}
+
+		select {
+		case sig := <-sigC:
+			if secondHandler != nil {
+				secondHandler(sig)
+			}
+			// If we get a second signal, we immediately exit, so something like
+			// https://github.com/k6io/k6/issues/971 never happens again
+			gs.osExit(int(exitcodes.ExternalAbort))
+		case <-done:
+			return
+		}
+	}()
+
+	return func() {
+		close(done)
+		gs.signalStop(sigC)
 	}
 }

cmd/config.go

@@ -91,6 +91,16 @@ func (c Config) Apply(cfg Config) Config {
 	return c
 }
 
+// Returns a Config but only parses the Options inside.
+func getPartialConfig(flags *pflag.FlagSet) (Config, error) {
+	opts, err := getOptions(flags)
+	if err != nil {
+		return Config{}, err
+	}
+
+	return Config{Options: opts}, nil
+}
+
 // Gets configuration from CLI flags.
 func getConfig(flags *pflag.FlagSet) (Config, error) {
 	opts, err := getOptions(flags)