Merge pull request #8 from spenczar/floating_point_weirdos

Handle Inf and NaN means

serde.go

@@ -75,6 +75,12 @@ func unmarshalBinary(d *TDigest, p []byte) error {
  if c.count < 0 {
  return fmt.Errorf("data corruption detected: negative count: %d", c.count)
  }
+ if math.IsNaN(c.mean) {
+ return fmt.Errorf("data corruption detected: NaN mean not permitted")
+ }
+ if math.IsInf(c.mean, 0) {
+ return fmt.Errorf("data corruption detected: Inf mean not permitted")
+ }
  if i > 0 {
  prev := d.centroids[i-1]
  if c.mean < prev.mean {

serde_test.go

@@ -160,6 +160,39 @@ func TestUnmarshalErrors(t *testing.T) {
  },
  errors.New("data corruption detected: centroid 1 has lower mean (1) than preceding centroid 0 (2)"),
  ))
+ t.Run("nan mean", testcase(
+ []byte{
+ 0x80, 0x0c,
+ 0x01, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x40,
+ 0x01, 0x00, 0x00, 0x00,
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
+ },
+ errors.New("data corruption detected: NaN mean not permitted"),
+ ))
+ t.Run("+inf mean", testcase(
+ []byte{
+ 0x80, 0x0c,
+ 0x01, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x40,
+ 0x01, 0x00, 0x00, 0x00,
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0x7F,
+ },
+ errors.New("data corruption detected: Inf mean not permitted"),
+ ))
+ t.Run("-inf mean", testcase(
+ []byte{
+ 0x80, 0x0c,
+ 0x01, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x40,
+ 0x01, 0x00, 0x00, 0x00,
+ 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF0, 0xFF,
+ },
+ errors.New("data corruption detected: Inf mean not permitted"),
+ ))
  t.Run("total size overflow", testcase(
  []byte{
  0x80, 0x0c,

tdigest.go

@@ -225,7 +225,12 @@ func (d *TDigest) addNewCentroid(mean float64, weight int64) {
 // Add will add a value to the TDigest, updating all quantiles. A
 // weight can be specified; use weight of 1 if you don't care about
 // weighting your dataset.
+//
+// Add will ignore input values of NaN or Inf.
 func (d *TDigest) Add(val float64, weight int) {
+ if math.IsNaN(val) || math.IsInf(val, 0) {
+ return
+ }
  d.add(val, int64(weight))
 }
 

tdigest_test.go

@@ -269,6 +269,9 @@ func TestAddValue(t *testing.T) {
  {2.0, 1, []*centroid{{0, 1}, {1, 1}, {2, 1}}},
  {3.0, 1, []*centroid{{0, 1}, {1, 1}, {2.5, 2}}},
  {4.0, 1, []*centroid{{0, 1}, {1, 1}, {2.5, 2}, {4, 1}}},
+ {math.NaN(), 1, []*centroid{{0, 1}, {1, 1}, {2.5, 2}, {4, 1}}},
+ {math.Inf(-1), 1, []*centroid{{0, 1}, {1, 1}, {2.5, 2}, {4, 1}}},
+ {math.Inf(+1), 1, []*centroid{{0, 1}, {1, 1}, {2.5, 2}, {4, 1}}},
  }
 
  d := NewWithCompression(1)