diff --git a/array.go b/array.go index f8dfc839..1ffaea55 100644 --- a/array.go +++ b/array.go @@ -576,11 +576,15 @@ var ( // usage of the whole array func (a *arrayObject) estimateMemUsage(ctx *MemUsageContext) (estimate uint64, newEstimate uint64, err error) { var samplesVisited, memUsage, newMemUsage uint64 - sampleSize := len(a.values) / 10 + arrayLen := len(a.values) + if arrayLen == 0 { + return memUsage, newMemUsage, nil + } + sampleSize := arrayLen / 10 // grabbing one sample every "sampleSize" to provide consistent // memory usage across function executions - for i := 0; i < len(a.values); i += sampleSize { + for i := 0; i < arrayLen; i += sampleSize { if a.values[i] == nil { continue } @@ -590,8 +594,8 @@ func (a *arrayObject) estimateMemUsage(ctx *MemUsageContext) (estimate uint64, n memUsage += inc newMemUsage += newInc // average * number of a.values - estimate = uint64((float32(memUsage) / float32(samplesVisited)) * float32(len(a.values))) - newEstimate = uint64((float32(newMemUsage) / float32(samplesVisited)) * float32(len(a.values))) + estimate = uint64((float32(memUsage) / float32(samplesVisited)) * float32(arrayLen)) + newEstimate = uint64((float32(newMemUsage) / float32(samplesVisited)) * float32(arrayLen)) if err != nil { return estimate, newEstimate, err } diff --git a/array_test.go b/array_test.go index 9aec5498..7619566e 100644 --- a/array_test.go +++ b/array_test.go @@ -186,6 +186,18 @@ func TestArrayObjectMemUsage(t *testing.T) { (4+SizeString)*4, errExpected: nil, }, + { + name: "empty array with negative threshold", + mu: NewMemUsageContext(vm, 88, 100, -1, 50, TestNativeMemUsageChecker{}), + ao: &arrayObject{ + values: []Value{}, + }, + // array overhead + array baseObject + values slice overhead + expectedMem: SizeEmptyStruct + SizeEmptyStruct + SizeEmptySlice, + // array overhead + array baseObject + values slice overhead + expectedNewMem: SizeEmptyStruct + SizeEmptyStruct + SizeEmptySlice, + errExpected: nil, + }, { name: "array limit function undefined throws error", mu: &MemUsageContext{ diff --git a/object.go b/object.go index 7d2b5041..c54a3d17 100644 --- a/object.go +++ b/object.go @@ -1916,6 +1916,9 @@ func computeMemUsageEstimate(memUsage, samplesVisited uint64, totalProps int) ui func (o *baseObject) estimateMemUsage(ctx *MemUsageContext) (estimate uint64, newEstimate uint64, err error) { var samplesVisited, memUsage, newMemUsage uint64 totalProps := len(o.propNames) + if totalProps == 0 { + return memUsage, newMemUsage, nil + } sampleSize := totalProps / 10 // grabbing one sample every "sampleSize" to provide consistent diff --git a/object_gomap.go b/object_gomap.go index c8d67d27..25e8a2a9 100644 --- a/object_gomap.go +++ b/object_gomap.go @@ -157,23 +157,71 @@ func (o *objectGoMapSimple) equal(other objectImpl) bool { return false } +// estimateMemUsage helps calculating mem usage for large objects. +// It will sample the object and use those samples to estimate the +// mem usage. +func (o *objectGoMapSimple) estimateMemUsage(ctx *MemUsageContext) (estimate uint64, newEstimate uint64, err error) { + var samplesVisited, memUsage, newMemUsage uint64 + counter := 0 + totalProps := len(o.data) + if totalProps == 0 { + return memUsage, newMemUsage, nil + } + sampleSize := totalProps / 10 + + // grabbing one sample every "sampleSize" to provide consistent + // memory usage across function executions + for key := range o.data { + counter++ + if counter%sampleSize == 0 { + memUsage += uint64(len(key)) + SizeString + newMemUsage += uint64(len(key)) + SizeString + v := o._getStr(key) + if v == nil { + continue + } + + inc, newInc, err := v.MemUsage(ctx) + samplesVisited += 1 + memUsage += inc + newMemUsage += newInc + if err != nil { + return computeMemUsageEstimate(memUsage, samplesVisited, totalProps), computeMemUsageEstimate(newMemUsage, samplesVisited, totalProps), err + } + } + } + + return computeMemUsageEstimate(memUsage, samplesVisited, totalProps), computeMemUsageEstimate(newMemUsage, samplesVisited, totalProps), nil +} + func (o *objectGoMapSimple) MemUsage(ctx *MemUsageContext) (uint64, uint64, error) { - mem, newMem, err := o.baseObject.MemUsage(ctx) + memUsage, newMemUsage, err := o.baseObject.MemUsage(ctx) if err != nil { return 0, 0, err } + + if ctx.ObjectPropsLenExceedsThreshold(len(o.data)) { + inc, newInc, err := o.estimateMemUsage(ctx) + memUsage += inc + newMemUsage += newInc + if err != nil { + return memUsage, newMemUsage, err + } + return memUsage, newMemUsage, nil + } + for key := range o.data { - mem += uint64(len(key)) + SizeString - newMem += uint64(len(key)) + SizeString - memValue, newMemValue, err := o._getStr(key).MemUsage(ctx) - mem += memValue - newMem += newMemValue + memUsage += uint64(len(key)) + SizeString + newMemUsage += uint64(len(key)) + SizeString + incr, newIncr, err := o._getStr(key).MemUsage(ctx) + memUsage += incr + newMemUsage += newIncr if err != nil { - return mem, newMem, err + return memUsage, newMemUsage, err } - if exceeded := ctx.MemUsageLimitExceeded(mem); exceeded { - return mem, newMem, nil + if exceeded := ctx.MemUsageLimitExceeded(memUsage); exceeded { + return memUsage, newMemUsage, nil } } - return mem, newMem, nil + return memUsage, newMemUsage, nil } diff --git a/object_gomap_test.go b/object_gomap_test.go index 67ba840a..b84993ca 100644 --- a/object_gomap_test.go +++ b/object_gomap_test.go @@ -348,12 +348,13 @@ func TestGoMapMemUsage(t *testing.T) { } tests := []struct { - name string - val *objectGoMapSimple - memLimit uint64 - expectedMem uint64 - expectedNewMem uint64 - errExpected error + name string + val *objectGoMapSimple + memLimit uint64 + estimateThreshold int + expectedMem uint64 + expectedNewMem uint64 + errExpected error }{ { name: "should account for each key value pair given a non-empty object", @@ -366,7 +367,8 @@ func TestGoMapMemUsage(t *testing.T) { "test1": valueInt(99), }, }, - memLimit: 100, + memLimit: 100, + estimateThreshold: 100, // baseObject overhead + len("testN") with string overhead + value expectedMem: SizeEmptyStruct + ((5+SizeString)+SizeInt)*2, // baseObject overhead + len("testN") with string overhead + value @@ -384,7 +386,8 @@ func TestGoMapMemUsage(t *testing.T) { "test1": 99, }, }, - memLimit: 100, + memLimit: 100, + estimateThreshold: 100, // baseObject overhead + len("testN") with string overhead + value expectedMem: SizeEmptyStruct + ((5+SizeString)+SizeInt)*2, // baseObject overhead + len("testN") with string overhead + value @@ -401,7 +404,8 @@ func TestGoMapMemUsage(t *testing.T) { "test": nil, }, }, - memLimit: 100, + memLimit: 100, + estimateThreshold: 100, // overhead + len("test") with string overhead + null expectedMem: SizeEmptyStruct + (4 + SizeString) + SizeEmptyStruct, // overhead + len("test") with string overhead + null @@ -418,7 +422,8 @@ func TestGoMapMemUsage(t *testing.T) { "test": nestedMap, }, }, - memLimit: 100, + memLimit: 100, + estimateThreshold: 100, // overhead + len("testN") with string overhead + (Object prototype with overhead + values with string overhead) expectedMem: SizeEmptyStruct + (4 + SizeString) + nestedMapMemUsage, // overhead + len("testN") with string overhead + (Object prototype with overhead + values with string overhead) @@ -435,7 +440,8 @@ func TestGoMapMemUsage(t *testing.T) { "test": &nestedMap, }, }, - memLimit: 100, + memLimit: 100, + estimateThreshold: 100, // overhead + len("testN") with string overhead + nested overhead expectedMem: SizeEmptyStruct + (4 + SizeString) + SizeEmptyStruct, // overhead + len("testN") with string overhead + nested overhead @@ -457,18 +463,64 @@ func TestGoMapMemUsage(t *testing.T) { "test5": valueInt(99), }, }, - memLimit: 0, + memLimit: 0, + estimateThreshold: 100, // baseObject overhead + len("testN") with string overhead + value expectedMem: SizeEmptyStruct + ((5 + SizeString) + SizeInt), // baseObject overhead + len("testN") with string overhead + value expectedNewMem: SizeEmptyStruct + ((5 + SizeString) + SizeInt), errExpected: nil, }, + { + name: "should estimate mem usage when exceeding object props len threshold", + val: &objectGoMapSimple{ + baseObject: baseObject{ + val: &Object{runtime: vm}, + }, + data: map[string]interface{}{ + "test0": valueInt(99), + "test1": valueInt(99), + "test2": valueInt(99), + "test3": valueInt(99), + "test4": valueInt(99), + "test5": valueInt(99), + "test6": valueInt(99), + "test7": valueInt(99), + "test8": valueInt(99), + "test9": valueInt(99), + "testa": valueInt(99), + "testb": valueInt(99), + }, + }, + memLimit: 0, + estimateThreshold: 10, + // baseObject overhead + len("testN") with string overhead + value + expectedMem: SizeEmptyStruct + ((5+SizeString)+SizeInt)*12, + // baseObject overhead + len("testN") with string overhead + value + expectedNewMem: SizeEmptyStruct + ((5+SizeString)+SizeInt)*12, + errExpected: nil, + }, + { + name: "should estimate mem usage given empty object nad negative threshold", + val: &objectGoMapSimple{ + baseObject: baseObject{ + val: &Object{runtime: vm}, + }, + data: map[string]interface{}{}, + }, + memLimit: 0, + estimateThreshold: -1, + // baseObject overhead + expectedMem: SizeEmptyStruct, + // baseObject overhead + expectedNewMem: SizeEmptyStruct, + errExpected: nil, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - total, newTotal, err := tc.val.MemUsage(NewMemUsageContext(vm, 100, tc.memLimit, 100, 100, nil)) + total, newTotal, err := tc.val.MemUsage(NewMemUsageContext(vm, 100, tc.memLimit, 100, tc.estimateThreshold, nil)) if err != tc.errExpected { t.Fatalf("Unexpected error. Actual: %v Expected: %v", err, tc.errExpected) } diff --git a/object_goslice.go b/object_goslice.go index a4328ea1..c32af2ba 100644 --- a/object_goslice.go +++ b/object_goslice.go @@ -350,24 +350,72 @@ func (o *objectGoSlice) swap(i int, j int) { (*o.data)[i], (*o.data)[j] = (*o.data)[j], (*o.data)[i] } +func (o *objectGoSlice) estimateMemUsage(ctx *MemUsageContext) (estimate uint64, newEstimate uint64, err error) { + var samplesVisited, memUsage, newMemUsage uint64 + counter := 0 + sliceLen := len(*o.data) + if sliceLen == 0 { + return memUsage, newMemUsage, nil + } + sampleSize := sliceLen / 10 + + // grabbing one sample every "sampleSize" to provide consistent + // memory usage across function executions + for _, datum := range *o.data { + counter++ + if counter%sampleSize == 0 { + if datum == nil { + continue + } + + inc, newInc, err := o.val.runtime.ToValue(datum).MemUsage(ctx) + samplesVisited += 1 + memUsage += inc + newMemUsage += newInc + // average * number of a.values + estimate = uint64((float32(memUsage) / float32(samplesVisited)) * float32(sliceLen)) + newEstimate = uint64((float32(newMemUsage) / float32(samplesVisited)) * float32(sliceLen)) + if err != nil { + return estimate, newEstimate, err + } + if exceeded := ctx.MemUsageLimitExceeded(estimate); exceeded { + return estimate, newEstimate, nil + } + } + } + + return estimate, newEstimate, nil +} + func (o *objectGoSlice) MemUsage(ctx *MemUsageContext) (uint64, uint64, error) { - mem, newMem, err := o.baseObject.MemUsage(ctx) + memUsage, newMemUsage, err := o.baseObject.MemUsage(ctx) if err != nil { return 0, 0, err } if o.data == nil { - return mem, newMem, nil + return memUsage, newMemUsage, nil + } + + if ctx.ArrayLenExceedsThreshold(len(*o.data)) { + inc, newInc, err := o.estimateMemUsage(ctx) + memUsage += inc + newMemUsage += newInc + if err != nil { + return memUsage, newMemUsage, err + } + return memUsage, newMemUsage, nil } + for _, datum := range *o.data { - memValue, newMemValue, err := o.val.runtime.ToValue(datum).MemUsage(ctx) - mem += memValue - newMem += newMemValue + inc, newInc, err := o.val.runtime.ToValue(datum).MemUsage(ctx) + memUsage += inc + newMemUsage += newInc if err != nil { - return mem, newMem, err + return memUsage, newMemUsage, err } - if exceeded := ctx.MemUsageLimitExceeded(mem); exceeded { - return mem, newMem, nil + if exceeded := ctx.MemUsageLimitExceeded(memUsage); exceeded { + return memUsage, newMemUsage, nil } } - return mem, newMem, nil + return memUsage, newMemUsage, nil } diff --git a/object_goslice_test.go b/object_goslice_test.go index 56fe43f2..eeb6417c 100644 --- a/object_goslice_test.go +++ b/object_goslice_test.go @@ -275,12 +275,13 @@ func TestGoSliceToString(t *testing.T) { func TestGoSliceMemUsage(t *testing.T) { vm := New() tests := []struct { - name string - val *objectGoSlice - memLimit uint64 - expectedMem uint64 - expectedNewMem uint64 - errExpected error + name string + val *objectGoSlice + memLimit uint64 + estimateThreshold int + expectedMem uint64 + expectedNewMem uint64 + errExpected error }{ { name: "should account for each value given a non-empty slice", @@ -293,7 +294,8 @@ func TestGoSliceMemUsage(t *testing.T) { valueInt(99), }, }, - memLimit: memUsageLimit, + memLimit: memUsageLimit, + estimateThreshold: 10, // overhead + values expectedMem: SizeEmptyStruct + SizeInt*2, // overhead + values @@ -307,7 +309,8 @@ func TestGoSliceMemUsage(t *testing.T) { val: &Object{runtime: vm}, }, }, - memLimit: memUsageLimit, + memLimit: memUsageLimit, + estimateThreshold: 10, // overhead expectedMem: SizeEmptyStruct, // overhead @@ -325,7 +328,8 @@ func TestGoSliceMemUsage(t *testing.T) { 99, }, }, - memLimit: memUsageLimit, + memLimit: memUsageLimit, + estimateThreshold: 10, // overhead + values expectedMem: SizeEmptyStruct + SizeInt*2, // overhead + values @@ -340,7 +344,8 @@ func TestGoSliceMemUsage(t *testing.T) { }, data: &[]interface{}{nil}, }, - memLimit: memUsageLimit, + memLimit: memUsageLimit, + estimateThreshold: 10, // overhead + null expectedMem: SizeEmptyStruct + SizeEmptyStruct, // overhead + null @@ -360,7 +365,8 @@ func TestGoSliceMemUsage(t *testing.T) { }, }, }, - memLimit: memUsageLimit, + memLimit: memUsageLimit, + estimateThreshold: 10, // default + default since we don't account for objectGoSlice in (*Object).MemUsage expectedMem: SizeEmptyStruct + SizeEmptyStruct, // overhead + (value + len("length") with string overhead + "length".value + prototype + ints) @@ -380,7 +386,8 @@ func TestGoSliceMemUsage(t *testing.T) { }, }, }, - memLimit: memUsageLimit, + memLimit: memUsageLimit, + estimateThreshold: 10, // default + default since we don't account for objectGoSlice in (*Object).MemUsage expectedMem: SizeEmptyStruct + SizeEmptyStruct, // overhead + (value + len("length") with string overhead + "length".value + prototype + ints) @@ -398,18 +405,63 @@ func TestGoSliceMemUsage(t *testing.T) { valueInt(99), }, }, - memLimit: 0, + memLimit: 0, + estimateThreshold: 10, // overhead + values expectedMem: SizeEmptyStruct + SizeInt, // overhead + values expectedNewMem: SizeEmptyStruct + SizeInt, errExpected: nil, }, + { + name: "should estimate mem usage when over array len threshold", + val: &objectGoSlice{ + baseObject: baseObject{ + val: &Object{runtime: vm}, + }, + data: &[]interface{}{ + valueInt(99), + valueInt(99), + valueInt(99), + valueInt(99), + valueInt(99), + valueInt(99), + valueInt(99), + valueInt(99), + valueInt(99), + valueInt(99), + valueInt(99), + }, + }, + memLimit: 0, + estimateThreshold: 10, + // overhead + values + expectedMem: SizeEmptyStruct + SizeInt*11, + // overhead + values + expectedNewMem: SizeEmptyStruct + SizeInt*11, + errExpected: nil, + }, + { + name: "should estimate mem usage given an empty slice and negative threshold", + val: &objectGoSlice{ + baseObject: baseObject{ + val: &Object{runtime: vm}, + }, + data: &[]interface{}{}, + }, + memLimit: 0, + estimateThreshold: -1, + // overhead + expectedMem: SizeEmptyStruct, + // overhead + expectedNewMem: SizeEmptyStruct, + errExpected: nil, + }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - total, newTotal, err := tc.val.MemUsage(NewMemUsageContext(vm, 100, tc.memLimit, 100, 100, nil)) + total, newTotal, err := tc.val.MemUsage(NewMemUsageContext(vm, 100, tc.memLimit, tc.estimateThreshold, 100, nil)) if err != tc.errExpected { t.Fatalf("Unexpected error. Actual: %v Expected: %v", err, tc.errExpected) } diff --git a/object_test.go b/object_test.go index 25c12d4d..615ff1f0 100644 --- a/object_test.go +++ b/object_test.go @@ -755,6 +755,20 @@ func TestBaseObjectMemUsage(t *testing.T) { expectedNewMem: SizeEmptyStruct + ((5 + SizeString) + SizeInt), errExpected: nil, }, + { + name: "should estimate memory given an empty object and negative threshold", + threshold: -1, + memLimit: 0, + val: &baseObject{ + propNames: []unistring.String{}, + values: map[unistring.String]Value{}, + }, + // overhead + expectedMem: SizeEmptyStruct, + // overhead + expectedNewMem: SizeEmptyStruct, + errExpected: nil, + }, } for _, tc := range tests { @@ -808,7 +822,7 @@ func TestPrimitiveValueObjectMemUsage(t *testing.T) { errExpected: nil, }, { - name: "should account for overehead and each key value pair given a primitive value object with non-empty object", + name: "should account for overhead and each key value pair given a primitive value object with non-empty object", val: &primitiveValueObject{baseObject: baseObject{propNames: []unistring.String{"test"}, values: map[unistring.String]Value{"test": valueInt(99)}}}, // baseObject overhead + len("test") with string overhead + value expectedMem: SizeEmptyStruct + (4 + SizeString) + SizeInt,