Add test command to run any unit test (#1092)

This commit adds a `test` command that allows the student to run the tests for an exercise without knowing the track-specific test command.
This makes it easier for the student to get started.
For debugging and education purposes, we print the command that is used to run the tests.

CHANGELOG.md

@@ -5,6 +5,7 @@ The exercism CLI follows [semantic versioning](http://semver.org/).
 ----------------
 
 ## Next Release
+* [#1092](https://github.com/exercism/cli/pull/1092) Add `exercism test` command to run the unit tests for nearly any track (inspired by [universal-test-runner](https://github.com/xavdid/universal-test-runner)) - [@xavdid]
 * **Your contribution here**
 
 ## v3.1.0 (2022-10-04)
@@ -489,5 +490,6 @@ All changes by [@msgehard]
 [@sfairchild]: https://github.com/sfairchild
 [@simonjefford]: https://github.com/simonjefford
 [@srt32]: https://github.com/srt32
+[@xavdid]: https://github.com/xavdid
 [@williandrade]: https://github.com/williandrade
 [@zabawaba99]: https://github.com/zabawaba99

cmd/cmd_test.go

@@ -26,12 +26,14 @@ const cfgHomeKey = "EXERCISM_CONFIG_HOME"
 // test, call the command by calling Execute on the App.
 //
 // Example:
-// cmdTest := &CommandTest{
-// 	Cmd:    myCmd,
-// 	InitFn: initMyCmd,
-// 	Args:   []string{"fakeapp", "mycommand", "arg1", "--flag", "value"},
-// 	MockInteractiveResponse: "first-input\nsecond\n",
-// }
+//
+//	cmdTest := &CommandTest{
+//		Cmd:    myCmd,
+//		InitFn: initMyCmd,
+//		Args:   []string{"fakeapp", "mycommand", "arg1", "--flag", "value"},
+//		MockInteractiveResponse: "first-input\nsecond\n",
+//	}
+//
 // cmdTest.Setup(t)
 // defer cmdTest.Teardown(t)
 // ...

cmd/test.go

@@ -0,0 +1,84 @@
+package cmd
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/exercism/cli/workspace"
+	"github.com/spf13/cobra"
+)
+
+var testCmd = &cobra.Command{
+	Use:     "test",
+	Aliases: []string{"t"},
+	Short:   "Run the exercise's tests.",
+	Long: `Run the exercise's tests.
+
+	Run this command in an exercise's root directory.`,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return runTest(args)
+	},
+}
+
+func runTest(args []string) error {
+	track, err := getTrack()
+	if err != nil {
+		return err
+	}
+
+	testConf, ok := workspace.TestConfigurations[track]
+
+	if !ok {
+		return fmt.Errorf("the \"%s\" track does not yet support running tests using the Exercism CLI. Please see HELP.md for testing instructions", track)
+	}
+
+	command, err := testConf.GetTestCommand()
+	if err != nil {
+		return err
+	}
+	cmdParts := strings.Split(command, " ")
+
+	// pass args/flags to this command down to the test handler
+	if len(args) > 0 {
+		cmdParts = append(cmdParts, args...)
+	}
+
+	fmt.Printf("Running tests via `%s`\n\n", strings.Join(cmdParts, " "))
+	exerciseTestCmd := exec.Command(cmdParts[0], cmdParts[1:]...)
+
+	// pipe output directly out, preserving any color
+	exerciseTestCmd.Stdout = os.Stdout
+	exerciseTestCmd.Stderr = os.Stderr
+
+	err = exerciseTestCmd.Run()
+	if err != nil {
+		// unclear what other errors would pop up here, but it pays to be defensive
+		if exitErr, ok := err.(*exec.ExitError); ok {
+			exitCode := exitErr.ExitCode()
+			// if subcommand returned a non-zero exit code, exit with the same
+			os.Exit(exitCode)
+		} else {
+			log.Fatalf("Failed to get error from failed subcommand: %v", err)
+		}
+	}
+	return nil
+}
+
+func getTrack() (string, error) {
+	metadata, err := workspace.NewExerciseMetadata(".")
+	if err != nil {
+		return "", err
+	}
+	if metadata.Track == "" {
+		return "", fmt.Errorf("no track found in exercise metadata")
+	}
+
+	return metadata.Track, nil
+}
+
+func init() {
+	RootCmd.AddCommand(testCmd)
+}

workspace/exercise_config.go

@@ -0,0 +1,57 @@
+package workspace
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"path/filepath"
+)
+
+const configFilename = "config.json"
+
+var configFilepath = filepath.Join(ignoreSubdir, configFilename)
+
+// ExerciseConfig contains exercise metadata.
+// Note: we only use a subset of its fields
+type ExerciseConfig struct {
+	Files struct {
+		Solution []string `json:"solution"`
+		Test     []string `json:"test"`
+	} `json:"files"`
+}
+
+// NewExerciseConfig reads exercise metadata from a file in the given directory.
+func NewExerciseConfig(dir string) (*ExerciseConfig, error) {
+	b, err := ioutil.ReadFile(filepath.Join(dir, configFilepath))
+	if err != nil {
+		return nil, err
+	}
+	var config ExerciseConfig
+	if err := json.Unmarshal(b, &config); err != nil {
+		return nil, err
+	}
+
+	return &config, nil
+}
+
+// GetTestFiles finds returns the names of the file(s) that hold unit tests for this exercise, if any
+func (c *ExerciseConfig) GetSolutionFiles() ([]string, error) {
+	result := c.Files.Solution
+	if result == nil {
+		// solution file(s) key was missing in config json, which is an error when calling this fuction
+		return []string{}, errors.New("no `files.solution` key in your `config.json`. Was it removed by mistake?")
+	}
+
+	return result, nil
+}
+
+// GetTestFiles finds returns the names of the file(s) that hold unit tests for this exercise, if any
+func (c *ExerciseConfig) GetTestFiles() ([]string, error) {
+	result := c.Files.Test
+	if result == nil {
+		// test file(s) key was missing in config json, which is an error when calling this fuction
+		return []string{}, errors.New("no `files.test` key in your `config.json`. Was it removed by mistake?")
+	}
+
+	return result, nil
+}

workspace/exercise_config_test.go

@@ -0,0 +1,98 @@
+package workspace
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestExerciseConfig(t *testing.T) {
+	dir, err := ioutil.TempDir("", "exercise_config")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	err = os.Mkdir(filepath.Join(dir, ".exercism"), os.ModePerm)
+	assert.NoError(t, err)
+
+	f, err := os.Create(filepath.Join(dir, ".exercism", "config.json"))
+	assert.NoError(t, err)
+	defer f.Close()
+
+	_, err = f.WriteString(`{ "blurb": "Learn about the basics of Ruby by following a lasagna recipe.", "authors": ["iHiD", "pvcarrera"], "files": { "solution": ["lasagna.rb"], "test": ["lasagna_test.rb"], "exemplar": [".meta/exemplar.rb"] } } `)
+	assert.NoError(t, err)
+
+	ec, err := NewExerciseConfig(dir)
+	assert.NoError(t, err)
+
+	assert.Equal(t, ec.Files.Solution, []string{"lasagna.rb"})
+	solutionFiles, err := ec.GetSolutionFiles()
+	assert.NoError(t, err)
+	assert.Equal(t, solutionFiles, []string{"lasagna.rb"})
+
+	assert.Equal(t, ec.Files.Test, []string{"lasagna_test.rb"})
+	testFiles, err := ec.GetTestFiles()
+	assert.NoError(t, err)
+	assert.Equal(t, testFiles, []string{"lasagna_test.rb"})
+}
+
+func TestExerciseConfigNoTestKey(t *testing.T) {
+	dir, err := ioutil.TempDir("", "exercise_config")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	err = os.Mkdir(filepath.Join(dir, ".exercism"), os.ModePerm)
+	assert.NoError(t, err)
+
+	f, err := os.Create(filepath.Join(dir, ".exercism", "config.json"))
+	assert.NoError(t, err)
+	defer f.Close()
+
+	_, err = f.WriteString(`{ "blurb": "Learn about the basics of Ruby by following a lasagna recipe.", "authors": ["iHiD", "pvcarrera"], "files": { "exemplar": [".meta/exemplar.rb"] } } `)
+	assert.NoError(t, err)
+
+	ec, err := NewExerciseConfig(dir)
+	assert.NoError(t, err)
+
+	_, err = ec.GetSolutionFiles()
+	assert.Error(t, err, "no `files.solution` key in your `config.json`")
+	_, err = ec.GetTestFiles()
+	assert.Error(t, err, "no `files.test` key in your `config.json`")
+}
+
+func TestMissingExerciseConfig(t *testing.T) {
+	dir, err := ioutil.TempDir("", "exercise_config")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	_, err = NewExerciseConfig(dir)
+	assert.Error(t, err)
+	// any assertions about this error message have to work across all platforms, so be vague
+	// unix: ".exercism/config.json: no such file or directory"
+	// windows: "open .exercism\config.json: The system cannot find the path specified."
+	assert.Contains(t, err.Error(), filepath.Join(".exercism", "config.json:"))
+}
+
+func TestInvalidExerciseConfig(t *testing.T) {
+	dir, err := ioutil.TempDir("", "exercise_config")
+	assert.NoError(t, err)
+	defer os.RemoveAll(dir)
+
+	err = os.Mkdir(filepath.Join(dir, ".exercism"), os.ModePerm)
+	assert.NoError(t, err)
+
+	f, err := os.Create(filepath.Join(dir, ".exercism", "config.json"))
+	assert.NoError(t, err)
+	defer f.Close()
+
+	// invalid JSON
+	_, err = f.WriteString(`{ "blurb": "Learn about the basics of Ruby by following a lasagna recipe.", "authors": ["iHiD", "pvcarr `)
+	assert.NoError(t, err)
+
+	_, err = NewExerciseConfig(dir)
+	assert.Error(t, err)
+	assert.True(t, strings.Contains(err.Error(), "unexpected end of JSON input"))
+}