From dfc427ba4cdc31f449f93d6d479b44477cedc97a Mon Sep 17 00:00:00 2001 From: Eugene Apenkin Date: Fri, 17 May 2024 17:48:48 +0400 Subject: [PATCH] decimal: implement binary marshaling --- .github/workflows/go.yml | 105 ++++++++++----------- CHANGELOG.md | 6 ++ decimal.go | 118 ++++++++++++++++++++++++ decimal_test.go | 191 ++++++++++++++++++++++++++++++++++++++- doc_test.go | 36 +++++++- 5 files changed, 403 insertions(+), 53 deletions(-) diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 4b5316a..94ace92 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -12,79 +12,82 @@ jobs: os: [ubuntu-latest, windows-latest, macos-latest] runs-on: ${{ matrix.os }} steps: + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + cache: false - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: ${{ matrix.go-version }} - cache: false + - name: Check out code + uses: actions/checkout@v4 - - name: Check out code - uses: actions/checkout@v4 + - name: Verify code formatting + run: gofmt -s -w . && git diff --exit-code - - name: Verify code formatting - run: gofmt -s -w . && git diff --exit-code + - name: Verify dependency consistency + run: go get -u -t . && go mod tidy && git diff --exit-code - - name: Verify dependency consistency - run: go get -u -t . && go mod tidy && git diff --exit-code + - name: Verify generated code + run: go generate ./... && git diff --exit-code - - name: Verify generated code - run: go generate ./... && git diff --exit-code + - name: Verify potential issues + uses: golangci/golangci-lint-action@v4 - - name: Verify potential issues - uses: golangci/golangci-lint-action@v4 + - name: Run tests with coverage + run: go test -race -shuffle=on -coverprofile="coverage.txt" -covermode=atomic ./... - - name: Run tests with coverage - run: go test -race -shuffle=on -coverprofile="coverage.txt" -covermode=atomic ./... - - - name: Upload test coverage - if: matrix.os == 'ubuntu-latest' && matrix.go-version == 'stable' - uses: codecov/codecov-action@v3 + - name: Upload test coverage + if: matrix.os == 'ubuntu-latest' && matrix.go-version == 'stable' + uses: codecov/codecov-action@v3 fuzz: needs: test runs-on: ubuntu-latest steps: + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version: stable + cache: false + + - name: Check out code + uses: actions/checkout@v4 - - name: Setup Go - uses: actions/setup-go@v5 - with: - go-version: stable - cache: false + - name: Run fuzzing for string parsing + run: go test -fuzztime 20s -fuzz ^FuzzParse$ - - name: Check out code - uses: actions/checkout@v4 + - name: Run fuzzing for binary parsing + run: go test -fuzztime 20s -fuzz ^FuzzBCD$ - - name: Run fuzzing for string parsing - run: go test -fuzztime 20s -fuzz ^FuzzParse$ + - name: Run fuzzing for string conversion + run: go test -fuzztime 20s -fuzz ^FuzzDecimal_String$ - - name: Run fuzzing for string conversion - run: go test -fuzztime 20s -fuzz ^FuzzDecimal_String$ + - name: Run fuzzing for binary conversion + run: go test -fuzztime 20s -fuzz ^FuzzDecimal_BCD$ - - name: Run fuzzing for float64 conversion - run: go test -fuzztime 20s -fuzz ^FuzzDecimal_Float64$ + - name: Run fuzzing for float64 conversion + run: go test -fuzztime 20s -fuzz ^FuzzDecimal_Float64$ - - name: Run fuzzing for int64 conversion - run: go test -fuzztime 20s -fuzz ^FuzzDecimal_Int64$ + - name: Run fuzzing for int64 conversion + run: go test -fuzztime 20s -fuzz ^FuzzDecimal_Int64$ - - name: Run fuzzing for addition - run: go test -fuzztime 20s -fuzz ^FuzzDecimal_Add$ + - name: Run fuzzing for addition + run: go test -fuzztime 20s -fuzz ^FuzzDecimal_Add$ - - name: Run fuzzing for multiplication - run: go test -fuzztime 20s -fuzz ^FuzzDecimal_Mul$ + - name: Run fuzzing for multiplication + run: go test -fuzztime 20s -fuzz ^FuzzDecimal_Mul$ - - name: Run fuzzing for fused multiply-addition - run: go test -fuzztime 60s -fuzz ^FuzzDecimal_FMA$ + - name: Run fuzzing for fused multiply-addition + run: go test -fuzztime 60s -fuzz ^FuzzDecimal_FMA$ - - name: Run fuzzing for division - run: go test -fuzztime 20s -fuzz ^FuzzDecimal_Quo$ + - name: Run fuzzing for division + run: go test -fuzztime 20s -fuzz ^FuzzDecimal_Quo$ - - name: Run fuzzing for integer division and remainder - run: go test -fuzztime 20s -fuzz ^FuzzDecimal_QuoRem$ + - name: Run fuzzing for integer division and remainder + run: go test -fuzztime 20s -fuzz ^FuzzDecimal_QuoRem$ - - name: Run fuzzing for comparison - run: go test -fuzztime 20s -fuzz ^FuzzDecimal_Cmp$ + - name: Run fuzzing for comparison + run: go test -fuzztime 20s -fuzz ^FuzzDecimal_Cmp$ - - name: Run fuzzing for comparison and subtraction - run: go test -fuzztime 20s -fuzz ^FuzzDecimal_CmpSub$ - \ No newline at end of file + - name: Run fuzzing for comparison and subtraction + run: go test -fuzztime 20s -fuzz ^FuzzDecimal_CmpSub$ diff --git a/CHANGELOG.md b/CHANGELOG.md index 7ce6613..6c47d82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [0.1.25] - 2024-05-17 + +### Added + +- Implemented binary marshaling. + ## [0.1.24] - 2024-05-05 ### Changed diff --git a/decimal.go b/decimal.go index 8e3cf30..10ec3e7 100644 --- a/decimal.go +++ b/decimal.go @@ -521,6 +521,108 @@ func (d Decimal) String() string { return string(buf[pos+1:]) } +// parseBCD converts a [packed BCD] representation to a decimal. +// +// [packed BCD]: https://en.wikipedia.org/wiki/Binary-coded_decimal#Packed_BCD +func parseBCD(b []byte) (Decimal, error) { + var pos int + width := len(b) + + // Coefficient and sign + var neg bool + var coef fint + var ok bool + for pos < width { + hi := b[pos] >> 4 + lo := b[pos] & 0x0f + + if hi > 9 { + return Decimal{}, fmt.Errorf("parsing \"%x\": invalid high nibble: %w", b[pos], errInvalidDecimal) + } + coef, ok = coef.fsa(1, hi) + if !ok { + return Decimal{}, errDecimalOverflow + } + + if lo > 9 { + if lo == 0x0d { + neg = true + } else if lo != 0x0c { + return Decimal{}, fmt.Errorf("parsing \"%x\": invalid low nibble: %w", b[pos], errInvalidDecimal) + } + pos++ + break + } + coef, ok = coef.fsa(1, lo) + if !ok { + return Decimal{}, errDecimalOverflow + } + pos++ + } + + // Scale + var scale int + var hasScale bool + if pos < width { + hi := b[pos] >> 4 + lo := b[pos] & 0x0f + hasScale = true + + if hi > 1 { + return Decimal{}, fmt.Errorf("parsing \"%x\": invalid high nibble: %w", b[pos], errInvalidDecimal) + } + scale = int(hi) * 10 + + if lo > 9 { + return Decimal{}, fmt.Errorf("parsing \"%x\": invalid low nibble: %w", b[pos], errInvalidDecimal) + } + scale += int(lo) + + pos++ + } + + if pos != width { + return Decimal{}, fmt.Errorf("invalid byte \"%x\": %w", b[pos], errInvalidDecimal) + } + if !hasScale { + return Decimal{}, fmt.Errorf("no scale: %w", errInvalidDecimal) + } + + return newSafe(neg, coef, scale) +} + +// bcd returns a [packed BCD] representation of a decimal. +// +// [packed BCD]: https://en.wikipedia.org/wiki/Binary-coded_decimal#Packed_BCD +func (d Decimal) bcd() []byte { + var buf [11]byte + pos := len(buf) - 1 + coef := d.Coef() + scale := d.Scale() + + // Scale + buf[pos] = byte(scale/10)<<4 | byte(scale%10) + pos-- + + // Sign and first digit + if d.IsNeg() { + buf[pos] = byte(coef%10)<<4 | 0x0d + } else { + buf[pos] = byte(coef%10)<<4 | 0x0c + } + pos-- + coef /= 10 + + // Coefficient + for coef > 0 { + buf[pos] = byte(coef/10%10)<<4 | byte(coef%10) + pos-- + coef /= 100 + } + + return buf[pos+1:] +} + // Float64 returns the nearest binary floating-point number rounded // using [rounding half to even] (banker's rounding). // See also constructor [NewFromFloat64]. @@ -602,6 +704,22 @@ func (d Decimal) MarshalText() ([]byte, error) { return []byte(d.String()), nil } +// UnmarshalBinary implements the [encoding.BinaryUnmarshaler] interface. +// +// [encoding.BinaryUnmarshaler]: https://pkg.go.dev/encoding#BinaryUnmarshaler +func (d *Decimal) UnmarshalBinary(data []byte) error { + var err error + *d, err = parseBCD(data) + return err +} + +// MarshalBinary implements the [encoding.BinaryMarshaler] interface. +// +// [encoding.BinaryMarshaler]: https://pkg.go.dev/encoding#BinaryMarshaler +func (d Decimal) MarshalBinary() ([]byte, error) { + return d.bcd(), nil +} + // Scan implements the [sql.Scanner] interface. // See also constructor [Parse]. // diff --git a/decimal_test.go b/decimal_test.go index 0565883..9a38594 100644 --- a/decimal_test.go +++ b/decimal_test.go @@ -1,6 +1,7 @@ package decimal import ( + "bytes" "database/sql" "database/sql/driver" "encoding" @@ -30,7 +31,9 @@ func TestDecimal_Size(t *testing.T) { } func TestDecimal_Interfaces(t *testing.T) { - var d any = Decimal{} + var d any + + d = Decimal{} _, ok := d.(fmt.Stringer) if !ok { t.Errorf("%T does not implement fmt.Stringer", d) @@ -43,6 +46,10 @@ func TestDecimal_Interfaces(t *testing.T) { if !ok { t.Errorf("%T does not implement encoding.TextMarshaler", d) } + _, ok = d.(encoding.BinaryMarshaler) + if !ok { + t.Errorf("%T does not implement encoding.BinaryMarshaler", d) + } _, ok = d.(driver.Valuer) if !ok { t.Errorf("%T does not implement driver.Valuer", d) @@ -53,6 +60,10 @@ func TestDecimal_Interfaces(t *testing.T) { if !ok { t.Errorf("%T does not implement encoding.TextUnmarshaler", d) } + _, ok = d.(encoding.BinaryUnmarshaler) + if !ok { + t.Errorf("%T does not implement encoding.BinaryUnmarshaler", d) + } _, ok = d.(sql.Scanner) if !ok { t.Errorf("%T does not implement sql.Scanner", d) @@ -523,6 +534,136 @@ func TestDecimal_String(t *testing.T) { }) } +func TestParseBCD(t *testing.T) { + t.Run("success", func(t *testing.T) { + tests := []struct { + bcd []byte + want string + }{ + {[]byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9d, 0x00}, "-9999999999999999999"}, + {[]byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9d, 0x01}, "-999999999999999999.9"}, + {[]byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9d, 0x02}, "-99999999999999999.99"}, + {[]byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9d, 0x03}, "-9999999999999999.999"}, + {[]byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9d, 0x19}, "-0.9999999999999999999"}, + {[]byte{0x1d, 0x00}, "-1"}, + {[]byte{0x1d, 0x01}, "-0.1"}, + {[]byte{0x1d, 0x02}, "-0.01"}, + {[]byte{0x1d, 0x19}, "-0.0000000000000000001"}, + {[]byte{0x0c, 0x00}, "0"}, + {[]byte{0x0c, 0x01}, "0.0"}, + {[]byte{0x0c, 0x02}, "0.00"}, + {[]byte{0x0c, 0x19}, "0.0000000000000000000"}, + {[]byte{0x1c, 0x00}, "1"}, + {[]byte{0x1c, 0x01}, "0.1"}, + {[]byte{0x1c, 0x02}, "0.01"}, + {[]byte{0x1c, 0x19}, "0.0000000000000000001"}, + {[]byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9c, 0x00}, "9999999999999999999"}, + {[]byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9c, 0x01}, "999999999999999999.9"}, + {[]byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9c, 0x02}, "99999999999999999.99"}, + {[]byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9c, 0x03}, "9999999999999999.999"}, + {[]byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9c, 0x19}, "0.9999999999999999999"}, + + // Exported constants + {[]byte{0x1d, 0x00}, NegOne.String()}, + {[]byte{0x0c, 0x00}, Zero.String()}, + {[]byte{0x1c, 0x00}, One.String()}, + {[]byte{0x2c, 0x00}, Two.String()}, + {[]byte{0x01, 0x0c, 0x00}, Ten.String()}, + {[]byte{0x10, 0x0c, 0x00}, Hundred.String()}, + {[]byte{0x01, 0x00, 0x0c, 0x00}, Thousand.String()}, + {[]byte{0x27, 0x18, 0x28, 0x18, 0x28, 0x45, 0x90, 0x45, 0x23, 0x5c, 0x18}, E.String()}, + {[]byte{0x31, 0x41, 0x59, 0x26, 0x53, 0x58, 0x97, 0x93, 0x23, 0x8c, 0x18}, Pi.String()}, + } + for _, tt := range tests { + got, err := parseBCD(tt.bcd) + if err != nil { + t.Errorf("parseBCD(% x) failed: %v", tt.bcd, err) + continue + } + want := MustParse(tt.want) + if got != want { + t.Errorf("parseBCD(% x) = %q, want %q", tt.bcd, got, want) + } + } + }) + + t.Run("error", func(t *testing.T) { + tests := map[string][]byte{ + "empty": {}, + "invalid nibble 1": {0x0f}, + "invalid nibble 2": {0xf0}, + "invalid nibble 3": {0x0c, 0x0f}, + "invalid nibble 4": {0x0c, 0xf0}, + "decimal overflow 1": {0x09, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9d, 0x00}, + "decimal overflow 2": {0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9d, 0x00}, + "no sign": {0x00}, + "scale overflow": {0x0c, 0x00, 0x00}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + _, err := parseBCD(tt) + if err == nil { + t.Errorf("parseBCD(% x) did not fail", tt) + } + }) + } + }) +} + +func TestDecimal_BCD(t *testing.T) { + t.Run("success", func(t *testing.T) { + tests := []struct { + d string + want []byte + }{ + {"-9999999999999999999", []byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9d, 0x00}}, + {"-999999999999999999.9", []byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9d, 0x01}}, + {"-99999999999999999.99", []byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9d, 0x02}}, + {"-9999999999999999.999", []byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9d, 0x03}}, + {"-0.9999999999999999999", []byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9d, 0x19}}, + {"-1", []byte{0x1d, 0x00}}, + {"-0.1", []byte{0x1d, 0x01}}, + {"-0.01", []byte{0x1d, 0x02}}, + {"-0.0000000000000000001", []byte{0x1d, 0x19}}, + {"0", []byte{0x0c, 0x00}}, + {"0.0", []byte{0x0c, 0x01}}, + {"0.00", []byte{0x0c, 0x02}}, + {"0.0000000000000000000", []byte{0x0c, 0x19}}, + {"1", []byte{0x1c, 0x00}}, + {"0.1", []byte{0x1c, 0x01}}, + {"0.01", []byte{0x1c, 0x02}}, + {"0.0000000000000000001", []byte{0x1c, 0x19}}, + {"9999999999999999999", []byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9c, 0x00}}, + {"999999999999999999.9", []byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9c, 0x01}}, + {"99999999999999999.99", []byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9c, 0x02}}, + {"9999999999999999.999", []byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9c, 0x03}}, + {"0.9999999999999999999", []byte{0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x99, 0x9c, 0x19}}, + + // Exported constants + {NegOne.String(), []byte{0x1d, 0x00}}, + {Zero.String(), []byte{0x0c, 0x00}}, + {One.String(), []byte{0x1c, 0x00}}, + {Two.String(), []byte{0x2c, 0x00}}, + {Ten.String(), []byte{0x01, 0x0c, 0x00}}, + {Hundred.String(), []byte{0x10, 0x0c, 0x00}}, + {Thousand.String(), []byte{0x01, 0x00, 0x0c, 0x00}}, + {E.String(), []byte{0x27, 0x18, 0x28, 0x18, 0x28, 0x45, 0x90, 0x45, 0x23, 0x5c, 0x18}}, + {Pi.String(), []byte{0x31, 0x41, 0x59, 0x26, 0x53, 0x58, 0x97, 0x93, 0x23, 0x8c, 0x18}}, + } + for _, tt := range tests { + d, err := Parse(tt.d) + if err != nil { + t.Errorf("Parse(%q) failed: %v", tt.d, err) + continue + } + got := d.bcd() + if !bytes.Equal(got, tt.want) { + t.Errorf("Parse(%q).bcd() = % x, want % x", tt.d, got, tt.want) + } + } + }) +} + func TestDecimal_Float64(t *testing.T) { tests := []struct { d string @@ -2885,6 +3026,25 @@ func FuzzParse(f *testing.F) { ) } +func FuzzBCD(f *testing.F) { + for _, c := range corpus { + d, err := newSafe(c.neg, fint(c.coef), c.scale) + if err != nil { + continue + } + f.Add(d.bcd()) + } + + f.Fuzz( + func(t *testing.T, bcd []byte) { + _, err := parseBCD(bcd) + if err != nil { + t.Skip() + } + }, + ) +} + func FuzzDecimal_String(f *testing.F) { for _, d := range corpus { f.Add(d.neg, d.scale, d.coef) @@ -2904,6 +3064,7 @@ func FuzzDecimal_String(f *testing.F) { t.Errorf("Parse(%q) failed: %v", s, err) return } + if got.CmpTotal(want) != 0 { t.Errorf("Parse(%q) = %v, want %v", s, got, want) return @@ -2912,6 +3073,34 @@ func FuzzDecimal_String(f *testing.F) { ) } +func FuzzDecimal_BCD(f *testing.F) { + for _, d := range corpus { + f.Add(d.neg, d.scale, d.coef) + } + + f.Fuzz( + func(t *testing.T, neg bool, scale int, coef uint64) { + want, err := newSafe(neg, fint(coef), scale) + if err != nil { + t.Skip() + return + } + + s := want.bcd() + got, err := parseBCD(s) + if err != nil { + t.Errorf("parseBCD(% x) failed: %v", s, err) + return + } + + if got.CmpTotal(want) != 0 { + t.Errorf("parseBCD(% x) = %v, want %v", s, got, want) + return + } + }, + ) +} + func FuzzDecimal_Int64(f *testing.F) { for _, d := range corpus { for s := 0; s <= MaxScale; s++ { diff --git a/doc_test.go b/doc_test.go index 4c24a9e..c3d83ff 100644 --- a/doc_test.go +++ b/doc_test.go @@ -256,6 +256,37 @@ func ExampleDecimal_String() { // Output: 1234567890.123456789 } +func unmarshalBytes(b []byte) (decimal.Decimal, error) { + var d decimal.Decimal + err := d.UnmarshalBinary(b) + return d, err +} + +func marshalBytes(s string) ([]byte, error) { + d, err := decimal.Parse(s) + if err != nil { + panic(err) + } + bcd, err := d.MarshalBinary() + if err != nil { + panic(err) + } + return bcd, nil +} + +func ExampleDecimal_UnmarshalBinary() { + fmt.Println(unmarshalBytes([]byte{0x56, 0x7c, 0x02})) + // Output: + // 5.67 +} + +func ExampleDecimal_MarshalBinary() { + bcd, err := marshalBytes("5.67") + fmt.Printf("% x %v\n", bcd, err) + // Output: + // 56 7c 02 +} + func ExampleDecimal_Float64() { d := decimal.MustParse("0.1") e := decimal.MustParse("123.456") @@ -291,7 +322,10 @@ type Object struct { func unmarshalJSON(s string) (Object, error) { var v Object err := json.Unmarshal([]byte(s), &v) - return v, err + if err != nil { + return Object{}, err + } + return v, nil } func marshalJSON(s string) (string, error) {