From b5d5c862365ec04d6bf7ca98757b6253184e0522 Mon Sep 17 00:00:00 2001 From: Philip Conrad Date: Tue, 10 May 2022 10:51:31 -0400 Subject: [PATCH] topdown: Add units.parse builtin. This function works on all base decimal and binary SI units of the set: m, K/Ki, M/Mi, G/Gi, T/Ti, P/Pi, and E/Ei Note: Unlike `units.parse_bytes`, this function is case sensitive. Fixes open-policy-agent#1802. Signed-off-by: Philip Conrad --- ast/builtins.go | 13 + capabilities.json | 14 + docs/content/policy-reference.md | 1 + .../units/test-parse-units-comparisons.yaml | 111 +++++ .../units/test-parse-units-errors.yaml | 78 ++++ .../testdata/units/test-parse-units.yaml | 431 ++++++++++++++++++ topdown/parse_bytes.go | 16 +- topdown/parse_units.go | 115 +++++ 8 files changed, 771 insertions(+), 8 deletions(-) create mode 100644 test/cases/testdata/units/test-parse-units-comparisons.yaml create mode 100644 test/cases/testdata/units/test-parse-units-errors.yaml create mode 100644 test/cases/testdata/units/test-parse-units.yaml create mode 100644 topdown/parse_units.go diff --git a/ast/builtins.go b/ast/builtins.go index 4ce91b0978..9ab00c5b18 100644 --- a/ast/builtins.go +++ b/ast/builtins.go @@ -255,6 +255,7 @@ var DefaultBuiltins = [...]*Builtin{ GlobQuoteMeta, // Units + UnitsParse, UnitsParseBytes, // UUIDs @@ -1123,6 +1124,18 @@ var NumbersRange = &Builtin{ * Units */ +// UnitsParse converts strings like 10G, 5K, 4M, 1500m and the like into a +// number. This number can be a non-integer, such as 1.5, 0.22, etc. +var UnitsParse = &Builtin{ + Name: "units.parse", + Decl: types.NewFunction( + types.Args( + types.S, + ), + types.N, + ), +} + // UnitsParseBytes converts strings like 10GB, 5K, 4mb, and the like into an // integer number of bytes. var UnitsParseBytes = &Builtin{ diff --git a/capabilities.json b/capabilities.json index 2f9e6e6409..9ffd8b97b4 100644 --- a/capabilities.json +++ b/capabilities.json @@ -3627,6 +3627,20 @@ "type": "function" } }, + { + "name": "units.parse", + "decl": { + "args": [ + { + "type": "string" + } + ], + "result": { + "type": "number" + }, + "type": "function" + } + }, { "name": "units.parse_bytes", "decl": { diff --git a/docs/content/policy-reference.md b/docs/content/policy-reference.md index 73e00b78b6..8629e80fb6 100644 --- a/docs/content/policy-reference.md +++ b/docs/content/policy-reference.md @@ -459,6 +459,7 @@ The following table shows examples of how ``glob.match`` works: | Built-in | Description | Wasm Support | | ------- |-------------|---------------| +| ``output := units.parse(x)`` | ``output`` is ``x`` converted to a number with support for standard metric decimal and binary SI units (e.g., K, Ki, M, Mi, G, Gi etc.) m, K, M, G, T, P, and E are treated as decimal units and Ki, Mi, Gi, Ti, Pi, and Ei are treated as binary units. Note that 'm' and 'M' are case-sensitive, to allow distinguishing between "milli" and "mega" units respectively. Other units are case-insensitive. | ``SDK-dependent`` | | ``output := units.parse_bytes(x)`` | ``output`` is ``x`` converted to a number with support for standard byte units (e.g., KB, KiB, etc.) KB, MB, GB, and TB are treated as decimal units and KiB, MiB, GiB, and TiB are treated as binary units. The bytes symbol (b/B) in the unit is optional and omitting it wil give the same result (e.g. Mi and MiB) | ``SDK-dependent`` | ### Types diff --git a/test/cases/testdata/units/test-parse-units-comparisons.yaml b/test/cases/testdata/units/test-parse-units-comparisons.yaml new file mode 100644 index 0000000000..2032e7eeb0 --- /dev/null +++ b/test/cases/testdata/units/test-parse-units-comparisons.yaml @@ -0,0 +1,111 @@ +cases: +- data: + modules: + - | + package test + p { + units.parse("8k") > units.parse("7k") + } + note: parse/comparison + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + p { + units.parse("8g") > units.parse("8m") + } + note: parse/comparison + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + p { + units.parse("1234k") < units.parse("1g") + } + note: parse/comparison + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + p { + units.parse("1024") == units.parse("1Ki") + } + note: parse/comparison + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + p { + units.parse("2Mi") == units.parse("2097152") + } + note: parse/comparison + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + p { + units.parse("3Mi") > units.parse("3M") + } + note: parse/comparison + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + p { + units.parse("2Mi") == units.parse("2Mi") + } + note: parse/comparison + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + p { + units.parse("4Mi") > units.parse("4M") + } + note: parse/comparison + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + p { + units.parse("4.1Mi") > units.parse("4Mi") + } + note: parse/comparison + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + p { + units.parse("128Gi") == units.parse("137438953472") + } + note: parse/comparison + query: data.test.p = x + want_result: + - x: true diff --git a/test/cases/testdata/units/test-parse-units-errors.yaml b/test/cases/testdata/units/test-parse-units-errors.yaml new file mode 100644 index 0000000000..5d1cefa403 --- /dev/null +++ b/test/cases/testdata/units/test-parse-units-errors.yaml @@ -0,0 +1,78 @@ +cases: +- data: + modules: + - | + package test + p { + units.parse("") + } + note: parse/failure + query: data.test.p = x + want_error: "units.parse error: no amount provided" + strict_error: true +- data: + modules: + - | + package test + p { + units.parse("G") + } + note: parse/failure + query: data.test.p = x + want_error: "units.parse error: no amount provided" + strict_error: true +- data: + modules: + - | + package test + p { + units.parse("foo") + } + note: parse/failure + query: data.test.p = x + want_error: "units.parse error: no amount provided" + strict_error: true +- data: + modules: + - | + package test + p { + units.parse("0.0.0") + } + note: parse/failure + query: data.test.p = x + want_error: "units.parse error: could not parse amount to a number" + strict_error: true +- data: + modules: + - | + package test + p { + units.parse(".5.2") + } + note: parse/failure + query: data.test.p = x + want_error: "units.parse error: could not parse amount to a number" + strict_error: true +- data: + modules: + - | + package test + p { + units.parse("100 k") + } + note: parse/failure + query: data.test.p = x + want_error: "units.parse error: spaces not allowed in resource strings" + strict_error: true +- data: + modules: + - | + package test + p { + units.parse(" 327Mi ") + } + note: parse/failure + query: data.test.p = x + want_error: "units.parse error: spaces not allowed in resource strings" + strict_error: true diff --git a/test/cases/testdata/units/test-parse-units.yaml b/test/cases/testdata/units/test-parse-units.yaml new file mode 100644 index 0000000000..e01c42a5be --- /dev/null +++ b/test/cases/testdata/units/test-parse-units.yaml @@ -0,0 +1,431 @@ +cases: +- data: + modules: + - | + package test + p { + units.parse("\"100TI\"") == 109951162777600 + } + note: parse/removes quotes and lowercases string + query: data.test.p = x + want_result: + - x: true +- data: + modules: + - | + package test + p { + units.parse("0") == 0 + } + note: parse/zero + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("0.0") == 0 + } + note: parse/zero float + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse(".0") == 0 + } + note: parse/zero bare float + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("12345") == 12345 + } + note: parse/raw number + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("10K") == 10000 + } + note: parse/10 kilo uppercase + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("10KI") == 10240 + } + note: parse/10 Ki uppercase + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("10k") == 10000 + } + note: parse/10 K lowercase + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("10Ki") == 10240 + } + note: parse/10 Ki mixed case + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("200M") == 200000000 + } + note: parse/200 mega + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("300Gi") == 322122547200 + } + note: parse/300 Gi + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("1.1K") == 1100 + } + note: parse/1.1 K floating point + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("1.1Ki") == 1126.4 + } + note: parse/1.1 Ki floating point, not rounded + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse(".5K") == 500 + } + note: parse/.5 K bare floating point + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100k") == 100000 + } + note: parse/100 kilo as k + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100K") == 100000 + } + note: parse/100 kilo as K + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100ki") == 102400 + } + note: parse/100 kibi as ki + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100Ki") == 102400 + } + note: parse/100 kibi as Ki + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + round(units.parse("100m") * 1000) == 100 + } + note: parse/100 milli as m + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100M") == 100000000 + } + note: parse/100 mega as M + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100mi") == 104857600 + } + note: parse/100 mebi as mi + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100Mi") == 104857600 + } + note: parse/100 mebi as Mi + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100g") == 100000000000 + } + note: parse/100 giga as g + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100gi") == 107374182400 + } + note: parse/100 gibi as gi + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100t") == 100000000000000 + } + note: parse/100 tera as t + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100T") == 100000000000000 + } + note: parse/100 tera as T + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100ti") == 109951162777600 + } + note: parse/100 tebi as ti + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100Ti") == 109951162777600 + } + note: parse/100 tebi as Ti + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100p") == 100000000000000000 + } + note: parse/100 peta as p + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100P") == 100000000000000000 + } + note: parse/100 peta as P + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100pi") == 112589990684262400 + } + note: parse/100 pebi as pi + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("100Pi") == 112589990684262400 + } + note: parse/100 pebi as Pi + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("10e") == 10000000000000000000 + } + note: parse/10 eta as e + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("10E") == 10000000000000000000 + } + note: parse/10 eta as E + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("10ei") == 11529215046068469760 + } + note: parse/10 ebi as ei + query: data.test.p = x + want_result: + - x: true + +- data: + modules: + - | + package test + p { + units.parse("10Ei") == 11529215046068469760 + } + note: parse/10 ebi as Ei + query: data.test.p = x + want_result: + - x: true diff --git a/topdown/parse_bytes.go b/topdown/parse_bytes.go index 6e58d2327c..abafdf0681 100644 --- a/topdown/parse_bytes.go +++ b/topdown/parse_bytes.go @@ -35,14 +35,14 @@ func parseNumBytesError(msg string) error { return fmt.Errorf("%s error: %s", ast.UnitsParseBytes.Name, msg) } -func errUnitNotRecognized(unit string) error { +func errBytesUnitNotRecognized(unit string) error { return parseNumBytesError(fmt.Sprintf("byte unit %s not recognized", unit)) } var ( - errNoAmount = parseNumBytesError("no byte amount provided") - errNumConv = parseNumBytesError("could not parse byte amount to a number") - errIncludesSpaces = parseNumBytesError("spaces not allowed in resource strings") + errBytesValueNoAmount = parseNumBytesError("no byte amount provided") + errBytesValueNumConv = parseNumBytesError("could not parse byte amount to a number") + errBytesValueIncludesSpaces = parseNumBytesError("spaces not allowed in resource strings") ) func builtinNumBytes(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error { @@ -56,12 +56,12 @@ func builtinNumBytes(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.T s := formatString(raw) if strings.Contains(s, " ") { - return errIncludesSpaces + return errBytesValueIncludesSpaces } num, unit := extractNumAndUnit(s) if num == "" { - return errNoAmount + return errBytesValueNoAmount } switch unit { @@ -92,12 +92,12 @@ func builtinNumBytes(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.T case "eib", "ei": m.SetUint64(ei) default: - return errUnitNotRecognized(unit) + return errBytesUnitNotRecognized(unit) } numFloat, ok := new(big.Float).SetString(num) if !ok { - return errNumConv + return errBytesValueNumConv } var total big.Int diff --git a/topdown/parse_units.go b/topdown/parse_units.go new file mode 100644 index 0000000000..800ee30014 --- /dev/null +++ b/topdown/parse_units.go @@ -0,0 +1,115 @@ +// Copyright 2022 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "fmt" + "math/big" + "strings" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +// Binary Si unit constants are borrowed from topdown/parse_bytes +const milli float64 = 0.001 +const ( + k uint64 = 1000 + m = k * 1000 + g = m * 1000 + t = g * 1000 + p = t * 1000 + e = p * 1000 +) + +func parseUnitsError(msg string) error { + return fmt.Errorf("%s error: %s", ast.UnitsParse.Name, msg) +} + +func errUnitNotRecognized(unit string) error { + return parseUnitsError(fmt.Sprintf("unit %s not recognized", unit)) +} + +var ( + errNoAmount = parseUnitsError("no amount provided") + errNumConv = parseUnitsError("could not parse amount to a number") + errIncludesSpaces = parseUnitsError("spaces not allowed in resource strings") +) + +// Accepts both normal SI and binary SI units. +func builtinUnits(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error { + var x big.Float + + raw, err := builtins.StringOperand(operands[0].Value, 1) + if err != nil { + return err + } + + // We remove escaped quotes from strings here to retain parity with units.parse_bytes. + s := string(raw) + s = strings.Replace(s, "\"", "", -1) + + if strings.Contains(s, " ") { + return errIncludesSpaces + } + + num, unit := extractNumAndUnit(s) + if num == "" { + return errNoAmount + + } + + // Unlike in units.parse_bytes, we only lowercase after the first letter, + // so that we can distinguish between 'm' and 'M'. + if len(unit) > 1 { + lower := strings.ToLower(unit[1:]) + unit = unit[:1] + lower + } + + switch unit { + case "m": + x.SetFloat64(milli) + case "": + x.SetUint64(none) + case "k", "K": + x.SetUint64(k) + case "ki", "Ki": + x.SetUint64(ki) + case "M": + x.SetUint64(m) + case "mi", "Mi": + x.SetUint64(mi) + case "g", "G": + x.SetUint64(g) + case "gi", "Gi": + x.SetUint64(gi) + case "t", "T": + x.SetUint64(t) + case "ti", "Ti": + x.SetUint64(ti) + case "p", "P": + x.SetUint64(p) + case "pi", "Pi": + x.SetUint64(pi) + case "e", "E": + x.SetUint64(e) + case "ei", "Ei": + x.SetUint64(ei) + default: + return errUnitNotRecognized(unit) + } + + numFloat, ok := new(big.Float).SetString(num) + if !ok { + return errNumConv + } + + numFloat.Mul(numFloat, &x) + return iter(ast.NewTerm(builtins.FloatToNumber(numFloat))) +} + +func init() { + RegisterBuiltinFunc(ast.UnitsParse.Name, builtinUnits) +}