gmeasure provides BETA support for benchmarking (#447)

gmeasure is a new gomega subpackage intended to provide measurement and benchmarking support for durations and values.  gmeasure replaces Ginkgo V1s deprecated Measure nodes and provides a migration path for users migrating to Ginkgo V2.

gmeasure is organized around an Experiment metaphor. Experiments can record several different Measurements, with each Measurement comprised of multiple data points.  Measurements can hold time.Durations and float64 values and gmeasure includes support measuring the duraiton of callback functions and for sampling functions repeatedly to build an ensemble of data points.  In addition, gmeasure introduces a Stopwatch abtraction for easily measuring and recording durations of code segments.

Once measured, users can readily generate Stats for Measurements to capture their key statistics and these stats can be ranked using a Ranking and associated RankingCriteria.

Experiments can be Cached to disk to speed up subsequent runs.  Experiments are cached by name and version number which makes it easy to manage and bust the cache.

Finally, gmeasure integrates with Ginkgo V2 via the new ReportEntry abstraction.  Experiments, Measurements, and Rankings can all be registered via AddReportEntry.  Doing so generates colorful reports as part of Ginkgo's test output.

gmeasure is currently in beta and will go GA around when Ginkgo V2 goes GA.

gmeasure/cache.go

@@ -0,0 +1,201 @@
+package gmeasure
+
+import (
+	"crypto/md5"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+const CACHE_EXT = ".gmeasure-cache"
+
+/*
+ExperimentCache provides a director-and-file based cache of experiments
+*/
+type ExperimentCache struct {
+	Path string
+}
+
+/*
+NewExperimentCache creates and initializes a new cache.  Path must point to a directory (if path does not exist, NewExperimentCache will create a directory at path).
+
+Cached Experiments are stored as separate files in the cache directory - the filename is a hash of the Experiment name.  Each file contains two JSON-encoded objects - a CachedExperimentHeader that includes the experiment's name and cache version number, and then the Experiment itself.
+*/
+func NewExperimentCache(path string) (ExperimentCache, error) {
+	stat, err := os.Stat(path)
+	if os.IsNotExist(err) {
+		err := os.MkdirAll(path, 0777)
+		if err != nil {
+			return ExperimentCache{}, err
+		}
+	} else if !stat.IsDir() {
+		return ExperimentCache{}, fmt.Errorf("%s is not a directory", path)
+	}
+
+	return ExperimentCache{
+		Path: path,
+	}, nil
+}
+
+/*
+CachedExperimentHeader captures the name of the Cached Experiment and its Version
+*/
+type CachedExperimentHeader struct {
+	Name    string
+	Version int
+}
+
+func (cache ExperimentCache) hashOf(name string) string {
+	return fmt.Sprintf("%x", md5.Sum([]byte(name)))
+}
+
+func (cache ExperimentCache) readHeader(filename string) (CachedExperimentHeader, error) {
+	out := CachedExperimentHeader{}
+	f, err := os.Open(filepath.Join(cache.Path, filename))
+	if err != nil {
+		return out, err
+	}
+	defer f.Close()
+	err = json.NewDecoder(f).Decode(&out)
+	return out, err
+}
+
+/*
+List returns a list of all Cached Experiments found in the cache.
+*/
+func (cache ExperimentCache) List() ([]CachedExperimentHeader, error) {
+	out := []CachedExperimentHeader{}
+	infos, err := ioutil.ReadDir(cache.Path)
+	if err != nil {
+		return out, err
+	}
+	for _, info := range infos {
+		if filepath.Ext(info.Name()) != CACHE_EXT {
+			continue
+		}
+		header, err := cache.readHeader(info.Name())
+		if err != nil {
+			return out, err
+		}
+		out = append(out, header)
+	}
+	return out, nil
+}
+
+/*
+Clear empties out the cache - this will delete any and all detected cache files in the cache directory.  Use with caution!
+*/
+func (cache ExperimentCache) Clear() error {
+	infos, err := ioutil.ReadDir(cache.Path)
+	if err != nil {
+		return err
+	}
+	for _, info := range infos {
+		if filepath.Ext(info.Name()) != CACHE_EXT {
+			continue
+		}
+		err := os.Remove(filepath.Join(cache.Path, info.Name()))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+/*
+Load fetches an experiment from the cache.  Lookup occurs by name.  Load requires that the version numer in the cache is equal to or greater than the passed-in version.
+
+If an experiment with corresponding name and version >= the passed-in version is found, it is unmarshaled and returned.
+
+If no experiment is found, or the cached version is smaller than the passed-in version, Load will return nil.
+
+When paired with Ginkgo you can cache experiments and prevent potentially expensive recomputation with this pattern:
+
+	const EXPERIMENT_VERSION = 1 //bump this to bust the cache and recompute _all_ experiments
+
+    Describe("some experiments", func() {
+    	var cache gmeasure.ExperimentCache
+    	var experiment *gmeasure.Experiment
+
+    	BeforeEach(func() {
+    		cache = gmeasure.NewExperimentCache("./gmeasure-cache")
+    		name := CurrentSpecReport().LeafNodeText
+    		experiment = cache.Load(name, EXPERIMENT_VERSION)
+    		if experiment != nil {
+    			AddReportEntry(experiment)
+    			Skip("cached")
+    		}
+    		experiment = gmeasure.NewExperiment(name)
+			AddReportEntry(experiment)
+    	})
+
+    	It("foo runtime", func() {
+    		experiment.SampleDuration("runtime", func() {
+    			//do stuff
+    		}, gmeasure.SamplingConfig{N:100})
+    	})
+
+    	It("bar runtime", func() {
+    		experiment.SampleDuration("runtime", func() {
+    			//do stuff
+    		}, gmeasure.SamplingConfig{N:100})
+    	})
+
+    	AfterEach(func() {
+    		if !CurrentSpecReport().State.Is(types.SpecStateSkipped) {
+	    		cache.Save(experiment.Name, EXPERIMENT_VERSION, experiment)
+	    	}
+    	})
+    })
+*/
+func (cache ExperimentCache) Load(name string, version int) *Experiment {
+	path := filepath.Join(cache.Path, cache.hashOf(name)+CACHE_EXT)
+	f, err := os.Open(path)
+	if err != nil {
+		return nil
+	}
+	defer f.Close()
+	dec := json.NewDecoder(f)
+	header := CachedExperimentHeader{}
+	dec.Decode(&header)
+	if header.Version < version {
+		return nil
+	}
+	out := NewExperiment("")
+	err = dec.Decode(out)
+	if err != nil {
+		return nil
+	}
+	return out
+}
+
+/*
+Save stores the passed-in experiment to the cache with the passed-in name and version.
+*/
+func (cache ExperimentCache) Save(name string, version int, experiment *Experiment) error {
+	path := filepath.Join(cache.Path, cache.hashOf(name)+CACHE_EXT)
+	f, err := os.Create(path)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	enc := json.NewEncoder(f)
+	err = enc.Encode(CachedExperimentHeader{
+		Name:    name,
+		Version: version,
+	})
+	if err != nil {
+		return err
+	}
+	return enc.Encode(experiment)
+}
+
+/*
+Delete removes the experiment with the passed-in name from the cache
+*/
+func (cache ExperimentCache) Delete(name string) error {
+	path := filepath.Join(cache.Path, cache.hashOf(name)+CACHE_EXT)
+	return os.Remove(path)
+}

gmeasure/cache_test.go

@@ -0,0 +1,109 @@
+package gmeasure_test
+
+import (
+	"fmt"
+	"os"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/onsi/gomega/gmeasure"
+)
+
+var _ = Describe("Cache", func() {
+	var path string
+	var cache gmeasure.ExperimentCache
+	var e1, e2 *gmeasure.Experiment
+
+	BeforeEach(func() {
+		var err error
+		path = fmt.Sprintf("./cache-%d", GinkgoParallelNode())
+		cache, err = gmeasure.NewExperimentCache(path)
+		Ω(err).ShouldNot(HaveOccurred())
+		e1 = gmeasure.NewExperiment("Experiment-1")
+		e1.RecordValue("foo", 32)
+		e2 = gmeasure.NewExperiment("Experiment-2")
+		e2.RecordValue("bar", 64)
+	})
+
+	AfterEach(func() {
+		Ω(os.RemoveAll(path)).Should(Succeed())
+	})
+
+	Describe("when creating a cache that points to a file", func() {
+		It("errors", func() {
+			f, err := os.Create("cache-temp-file")
+			Ω(err).ShouldNot(HaveOccurred())
+			f.Close()
+			cache, err := gmeasure.NewExperimentCache("cache-temp-file")
+			Ω(err).Should(MatchError("cache-temp-file is not a directory"))
+			Ω(cache).Should(BeZero())
+			Ω(os.RemoveAll("cache-temp-file")).Should(Succeed())
+		})
+	})
+
+	Describe("the happy path", func() {
+		It("can save, load, list, delete, and clear the cache", func() {
+			Ω(cache.Save("e1", 1, e1)).Should(Succeed())
+			Ω(cache.Save("e2", 7, e2)).Should(Succeed())
+
+			Ω(cache.Load("e1", 1)).Should(Equal(e1))
+			Ω(cache.Load("e2", 7)).Should(Equal(e2))
+
+			Ω(cache.List()).Should(ConsistOf(
+				gmeasure.CachedExperimentHeader{"e1", 1},
+				gmeasure.CachedExperimentHeader{"e2", 7},
+			))
+
+			Ω(cache.Delete("e2")).Should(Succeed())
+			Ω(cache.Load("e1", 1)).Should(Equal(e1))
+			Ω(cache.Load("e2", 7)).Should(BeNil())
+			Ω(cache.List()).Should(ConsistOf(
+				gmeasure.CachedExperimentHeader{"e1", 1},
+			))
+
+			Ω(cache.Clear()).Should(Succeed())
+			Ω(cache.List()).Should(BeEmpty())
+			Ω(cache.Load("e1", 1)).Should(BeNil())
+			Ω(cache.Load("e2", 7)).Should(BeNil())
+		})
+	})
+
+	Context("with an empty cache", func() {
+		It("should list nothing", func() {
+			Ω(cache.List()).Should(BeEmpty())
+		})
+
+		It("should not error when clearing", func() {
+			Ω(cache.Clear()).Should(Succeed())
+		})
+
+		It("returs nil when loading a non-existing experiment", func() {
+			Ω(cache.Load("floop", 17)).Should(BeNil())
+		})
+	})
+
+	Describe("version management", func() {
+		BeforeEach(func() {
+			Ω(cache.Save("e1", 7, e1)).Should(Succeed())
+		})
+
+		Context("when the cached version is older than the requested version", func() {
+			It("returns nil", func() {
+				Ω(cache.Load("e1", 8)).Should(BeNil())
+			})
+		})
+
+		Context("when the cached version equals the requested version", func() {
+			It("returns the cached version", func() {
+				Ω(cache.Load("e1", 7)).Should(Equal(e1))
+			})
+		})
+
+		Context("when the cached version is newer than the requested version", func() {
+			It("returns the cached version", func() {
+				Ω(cache.Load("e1", 6)).Should(Equal(e1))
+			})
+		})
+	})
+
+})

gmeasure/enum_support.go

@@ -0,0 +1,43 @@
+package gmeasure
+
+import "encoding/json"
+
+type enumSupport struct {
+	toString map[uint]string
+	toEnum   map[string]uint
+	maxEnum  uint
+}
+
+func newEnumSupport(toString map[uint]string) enumSupport {
+	toEnum, maxEnum := map[string]uint{}, uint(0)
+	for k, v := range toString {
+		toEnum[v] = k
+		if maxEnum < k {
+			maxEnum = k
+		}
+	}
+	return enumSupport{toString: toString, toEnum: toEnum, maxEnum: maxEnum}
+}
+
+func (es enumSupport) String(e uint) string {
+	if e > es.maxEnum {
+		return es.toString[0]
+	}
+	return es.toString[e]
+}
+
+func (es enumSupport) UnmarshalJSON(b []byte) (uint, error) {
+	var dec string
+	if err := json.Unmarshal(b, &dec); err != nil {
+		return 0, err
+	}
+	out := es.toEnum[dec] // if we miss we get 0 which is what we want anyway
+	return out, nil
+}
+
+func (es enumSupport) MarshalJSON(e uint) ([]byte, error) {
+	if e == 0 || e > es.maxEnum {
+		return json.Marshal(nil)
+	}
+	return json.Marshal(es.toString[e])
+}