Skip to content

Commit

Permalink
fit: Added support for "invalid" value checking.
Browse files Browse the repository at this point in the history 
Added support for array values.
Added documentation and tests.
  • Loading branch information
@mlofjard
mlofjard committed Feb 10, 2024
1 parent 46dbf5b commit 54c6f0c
Show file tree
Hide file tree
Showing 18 changed files with 141,174 additions and 1,800 deletions.
14 changes: 14 additions & 0 deletions format/fit/README.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
### Generated files
- format/fit/mappers/messages_generated.go
- format/fit/mappers/types_generated.go

### How to generate them if needed

1. Download the Fit SDK from: https://developer.garmin.com/fit/download/
2. Install NodeJS and NPM
3. Go to the `format/fit/testdata/generator` folder.
4. Run `npm install` if it's your first time
5. Run `node index.js t /PathToSDK/Profile.xlsx > ../../mappers/types_generated.go`
6. Run `node index.js m /PathToSDK/Profile.xlsx > ../../mappers/messages_generated.go`
7. Edit `messages_generated.go` and remove the incorrect "Scale" from line ~461
8. Correct spelling of farenheit->fahrenheit and bondary->boundary to please Go linter
193 changes: 91 additions & 102 deletions format/fit/fit.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ import (
"github.com/wader/fq/pkg/decode"
"github.com/wader/fq/pkg/interp"
"github.com/wader/fq/pkg/scalar"
"golang.org/x/text/encoding"
)

//go:embed fit.md
var fitFS embed.FS

func init() {
Expand All @@ -23,6 +23,7 @@ func init() {
Groups: []*decode.Group{format.Probe},
DecodeFn: decodeFIT,
})

interp.RegisterFS(fitFS)
}

Expand All @@ -36,27 +37,27 @@ func calcCRC(bytes []byte) uint16 {
crc = 0
for i := 0; i < len(bytes); i++ {
// compute checksum of lower four bits of byte
var byte = bytes[i]
var checkByte = bytes[i]
var tmp = fitCRCTable[crc&0xF]
crc = (crc >> 4) & 0x0FFF
crc = crc ^ tmp ^ fitCRCTable[byte&0xF]
crc = crc ^ tmp ^ fitCRCTable[checkByte&0xF]
tmp = fitCRCTable[crc&0xF]
crc = (crc >> 4) & 0x0FFF
crc = crc ^ tmp ^ fitCRCTable[(byte>>4)&0xF]
crc = crc ^ tmp ^ fitCRCTable[(checkByte>>4)&0xF]
}

return crc
}

type fitContext struct {
dataSize int
headerSize int
dataSize uint64
headerSize uint64
}

type dataRecordContext struct {
compressed bool
data bool
localMessageType int
localMessageType uint64
hasDeveloperFields bool
}

Expand All @@ -70,24 +71,13 @@ type fileDescriptionContext struct {
nativeMsgNo uint64
}

type fieldDef struct {
name string
typ string
format string
unit string
scale float64
offset int64
size int
}

type devFieldDefMap map[uint64]map[uint64]fieldDef
type localFieldDefMap map[uint64]map[uint64]fieldDef
type devFieldDefMap map[uint64]map[uint64]mappers.FieldDef
type localFieldDefMap map[uint64]map[uint64]mappers.FieldDef
type localMsgIsDevDef map[uint64]bool

func fitDecodeFileHeader(d *decode.D, fc *fitContext) {
frameStart := d.Pos()

// d.FieldStruct("ident", func(d *decode.D) {
headerSize := d.FieldU8("headerSize")
d.FieldU8("protocolVersion")
d.FieldU16("profileVersion")
Expand All @@ -98,28 +88,27 @@ func fitDecodeFileHeader(d *decode.D, fc *fitContext) {
headerCRC := calcCRC(d.BytesRange(frameStart, int(headerSize)-2))
d.FieldU16("crc", d.UintValidate(uint64(headerCRC)))
}
fc.headerSize = int(headerSize)
fc.dataSize = int(dataSize)
fc.headerSize = headerSize
fc.dataSize = dataSize
}

func fitDecodeDataRecordHeader(d *decode.D, drc *dataRecordContext) {
drc.compressed = d.FieldBool("normalHeader", scalar.BoolMapDescription{false: "Normal header",
true: "Compressed header"})
headerType := d.FieldU1("headerType", scalar.UintMapDescription{0: "Normal header", 1: "Compressed header"})
drc.compressed = headerType == 1
if drc.compressed {
localMessageType := d.FieldU2("localMessageType")
d.FieldU32("timeOffset")
drc.localMessageType = int(localMessageType)
drc.localMessageType = localMessageType
drc.data = true
} else {
mTypeIsDef := d.FieldBool("messageType", scalar.BoolMap{true: {Sym: 1, Description: "Definition message"},
false: {Sym: 0, Description: "Data message"}})
mTypeIsDef := d.FieldU1("messageType", scalar.UintMapDescription{0: "Data message", 1: "Definition message"})
hasDeveloperFields := d.FieldBool("hasDeveloperFields")
d.FieldBool("reserved")
localMessageType := d.FieldU4("localMessageType")

drc.hasDeveloperFields = hasDeveloperFields
drc.localMessageType = int(localMessageType)
drc.data = !mTypeIsDef
drc.localMessageType = localMessageType
drc.data = mTypeIsDef == 0
}
}

Expand All @@ -136,14 +125,14 @@ func fitDecodeDefinitionMessage(d *decode.D, drc *dataRecordContext, lmfd localF
}
messageNo := d.FieldU16("globalMessageNumber", mappers.TypeDefMap["mesg_num"])
if messageNo == 206 { // developer field_description
isDevMap[uint64(drc.localMessageType)] = true
isDevMap[drc.localMessageType] = true
} else {
isDevMap[uint64(drc.localMessageType)] = false
isDevMap[drc.localMessageType] = false
}
numFields := d.FieldU8("fields")
lmfd[uint64(drc.localMessageType)] = make(map[uint64]fieldDef, numFields)
lmfd[drc.localMessageType] = make(map[uint64]mappers.FieldDef, numFields)
d.FieldArray("fieldDefinitions", func(d *decode.D) {
for i := 0; i < int(numFields); i++ {
for i := uint64(0); i < numFields; i++ {
d.FieldStruct("fieldDefinition", func(d *decode.D) {
fieldDefNo := d.FieldU8("fieldDefNo", mappers.FieldDefMap[messageNo])
size := d.FieldU8("size")
Expand All @@ -153,10 +142,10 @@ func fitDecodeDefinitionMessage(d *decode.D, drc *dataRecordContext, lmfd localF
fDefLookup, isSet := mappers.FieldDefMap[messageNo][fieldDefNo]
if isSet {
var foundName = fDefLookup.Name
lmfd[uint64(drc.localMessageType)][uint64(i)] = fieldDef{name: foundName, typ: typ, size: int(size), format: fDefLookup.Type, unit: fDefLookup.Unit, scale: fDefLookup.Scale, offset: fDefLookup.Offset}
lmfd[drc.localMessageType][i] = mappers.FieldDef{Name: foundName, Type: typ, Size: size, Format: fDefLookup.Type, Unit: fDefLookup.Unit, Scale: fDefLookup.Scale, Offset: fDefLookup.Offset}
} else {
var foundName = fmt.Sprintf("UNKOWN_%d", fieldDefNo)
lmfd[uint64(drc.localMessageType)][uint64(i)] = fieldDef{name: foundName, typ: typ, size: int(size), format: "unknown"}
var foundName = fmt.Sprintf("UNKNOWN_%d", fieldDefNo)
lmfd[drc.localMessageType][i] = mappers.FieldDef{Name: foundName, Type: typ, Size: size, Format: "unknown"}
}
})
}
Expand All @@ -171,18 +160,14 @@ func fitDecodeDefinitionMessage(d *decode.D, drc *dataRecordContext, lmfd localF
size := d.FieldU8("size")
devDataIdx := d.FieldU8("devDataIdx")

//baseType := d.FieldU8("baseType", mappers.TypeDefMap["fit_base_type"])

//var typ = mappers.TypeDefMap["fit_base_type"][baseType].Name
typ := dmfd[devDataIdx][fieldDefNo].typ
//fDefLookup, isSet := mappers.FieldDefMap[messageNo][fieldDefNo]
fDefLookup, isSet := dmfd[devDataIdx][fieldDefNo]

if isSet {
var foundName = fDefLookup.name
lmfd[uint64(drc.localMessageType)][uint64(i)] = fieldDef{name: foundName, typ: typ, size: int(size), unit: fDefLookup.unit, scale: fDefLookup.scale, offset: fDefLookup.offset}
var foundName = fDefLookup.Name
lmfd[drc.localMessageType][i] = mappers.FieldDef{Name: foundName, Type: fDefLookup.Type, Size: size, Unit: fDefLookup.Unit, Scale: fDefLookup.Scale, Offset: fDefLookup.Offset}
} else {
var foundName = fmt.Sprintf("UNKOWN_%d", fieldDefNo)
lmfd[uint64(drc.localMessageType)][uint64(i)] = fieldDef{name: foundName, typ: typ, size: int(size), format: "unknown"}
var foundName = fmt.Sprintf("UNKNOWN_%d", fieldDefNo)
lmfd[drc.localMessageType][i] = mappers.FieldDef{Name: foundName, Type: "UNKNOWN", Size: size, Format: "unknown"}
}
})
}
Expand All @@ -191,25 +176,31 @@ func fitDecodeDefinitionMessage(d *decode.D, drc *dataRecordContext, lmfd localF

}

func ensureDevFieldMap(dmfd devFieldDefMap, devIdx uint64, fieldDefNo uint64) {
func ensureDevFieldMap(dmfd devFieldDefMap, devIdx uint64) {
_, devIsSet := dmfd[devIdx]

if !devIsSet {
dmfd[devIdx] = make(map[uint64]fieldDef)
dmfd[devIdx] = make(map[uint64]mappers.FieldDef)
}
}

func fieldUint(fieldFn func(string, ...scalar.UintMapper) uint64, d *decode.D, fdc *fileDescriptionContext, expectedSize int, fDef fieldDef, uintFormatter scalar.UintFn) {
func fieldUint(fieldFn func(string, ...scalar.UintMapper) uint64, expectedSize uint64, fDef mappers.FieldDef, uintFormatter scalar.UintFn, fdc *fileDescriptionContext) {
var val uint64
if fDef.size != expectedSize {
d.FieldStr(fDef.name, fDef.size, encoding.Nop)

if fDef.Size != expectedSize {
arrayCount := fDef.Size / expectedSize
for i := uint64(0); i < arrayCount; i++ {
fieldFn(fmt.Sprintf("%s_%d", fDef.Name, i), uintFormatter)
}
} else {
if uintFormatter != nil {
val = fieldFn(fDef.name, uintFormatter)
val = fieldFn(fDef.Name, uintFormatter)
} else {
val = fieldFn(fDef.name)
val = fieldFn(fDef.Name)
}

switch fDef.name {
// Save developer field definitions
switch fDef.Name {
case "developer_data_index":
fdc.devIdx = val
case "field_definition_number":
Expand All @@ -224,35 +215,37 @@ func fieldUint(fieldFn func(string, ...scalar.UintMapper) uint64, d *decode.D, f
}
}

func fieldSint(fieldFn func(string, ...scalar.SintMapper) int64, d *decode.D, fdc *fileDescriptionContext, expectedSize int, fDef fieldDef, sintFormatter scalar.SintFn) {
//var val uint64
if fDef.size != expectedSize {
d.FieldStr(fDef.name, fDef.size, encoding.Nop)
func fieldSint(fieldFn func(string, ...scalar.SintMapper) int64, expectedSize uint64, fDef mappers.FieldDef, sintFormatter scalar.SintFn) {
if fDef.Size != expectedSize {
arrayCount := fDef.Size / expectedSize
for i := uint64(0); i < arrayCount; i++ {
fieldFn(fmt.Sprintf("%s_%d", fDef.Name, i), sintFormatter)
}
} else {
if sintFormatter != nil {
fieldFn(fDef.name, sintFormatter)
fieldFn(fDef.Name, sintFormatter)
} else {
fieldFn(fDef.name)
fieldFn(fDef.Name)
}
//setDevFieldUint(isDevDep, dmfd, fDef.name, val);
}
}

func fieldFloat(fieldFn func(string, ...scalar.FltMapper) float64, d *decode.D, fdc *fileDescriptionContext, expectedSize int, fDef fieldDef) {
//var val uint64
if fDef.size != expectedSize {
d.FieldStr(fDef.name, fDef.size, encoding.Nop)
func fieldFloat(fieldFn func(string, ...scalar.FltMapper) float64, expectedSize uint64, fDef mappers.FieldDef, floatFormatter scalar.FltFn) {
if fDef.Size != expectedSize {
arrayCount := fDef.Size / expectedSize
for i := uint64(0); i < arrayCount; i++ {
fieldFn(fmt.Sprintf("%s_%d", fDef.Name, i), floatFormatter)
}
} else {
fieldFn(fDef.name)

//setDevFieldUint(isDevDep, dmfd, fDef.name, val);
fieldFn(fDef.Name)
}
}

func fieldString(d *decode.D, fdc *fileDescriptionContext, fDef fieldDef) {
val := d.FieldUTF8NullFixedLen(fDef.name, fDef.size)
func fieldString(d *decode.D, fDef mappers.FieldDef, fdc *fileDescriptionContext) {
val := d.FieldUTF8NullFixedLen(fDef.Name, int(fDef.Size), scalar.StrMapSymStr{"": "[invalid]"})

switch fDef.name {
// Save developer field definitions
switch fDef.Name {
case "field_name":
fdc.name = val
case "units":
Expand All @@ -262,55 +255,54 @@ func fieldString(d *decode.D, fdc *fileDescriptionContext, fDef fieldDef) {

func fitDecodeDataMessage(d *decode.D, drc *dataRecordContext, lmfd localFieldDefMap, dmfd devFieldDefMap, isDevMap localMsgIsDevDef) {
var fdc fileDescriptionContext
keys := make([]int, len(lmfd[uint64(drc.localMessageType)]))
keys := make([]int, len(lmfd[drc.localMessageType]))
i := 0
for k := range lmfd[uint64(drc.localMessageType)] {
for k := range lmfd[drc.localMessageType] {
keys[i] = int(k)
i++
}
sort.Ints(keys)

isDevDep := isDevMap[uint64(drc.localMessageType)]
isDevDep := isDevMap[drc.localMessageType]

for _, k := range keys {
fDef := lmfd[uint64(drc.localMessageType)][uint64(k)]
fDef := lmfd[drc.localMessageType][uint64(k)]

var uintFormatter = mappers.GetUintFormatter(fDef.format, fDef.unit, fDef.scale, fDef.offset)
var sintFormatter = mappers.GetSintFormatter(fDef.format, fDef.unit, fDef.scale, fDef.offset)
var uintFormatter = mappers.GetUintFormatter(fDef)
var sintFormatter = mappers.GetSintFormatter(fDef)
var floatFormatter = mappers.GetFloatFormatter(fDef)

switch fDef.typ {
// case "byte":
// d.FieldStr(fDef.name, fDef.size, encoding.Nop)
switch fDef.Type {
case "enum", "uint8", "uint8z", "byte":
fieldUint(d.FieldU8, d, &fdc, 1, fDef, uintFormatter)
fieldUint(d.FieldU8, 1, fDef, uintFormatter, &fdc)
case "uint16", "uint16z":
fieldUint(d.FieldU16, 2, fDef, uintFormatter, &fdc)
case "uint32", "uint32z":
fieldUint(d.FieldU32, 4, fDef, uintFormatter, &fdc)
case "uint64", "uint64z":
fieldUint(d.FieldU64, 8, fDef, uintFormatter, &fdc)
case "sint8":
fieldSint(d.FieldS8, d, &fdc, 1, fDef, sintFormatter)
fieldSint(d.FieldS8, 1, fDef, sintFormatter)
case "sint16":
fieldSint(d.FieldS16, d, &fdc, 2, fDef, sintFormatter)
case "uint16", "uint16z":
fieldUint(d.FieldU16, d, &fdc, 2, fDef, uintFormatter)
fieldSint(d.FieldS16, 2, fDef, sintFormatter)
case "sint32":
fieldSint(d.FieldS32, d, &fdc, 4, fDef, sintFormatter)
case "uint32", "uint32z":
fieldUint(d.FieldU32, d, &fdc, 4, fDef, uintFormatter)
fieldSint(d.FieldS32, 4, fDef, sintFormatter)
case "sint64":
fieldSint(d.FieldS64, 8, fDef, sintFormatter)
case "float32":
fieldFloat(d.FieldF32, d, &fdc, 4, fDef)
fieldFloat(d.FieldF32, 4, fDef, floatFormatter)
case "float64":
fieldFloat(d.FieldF64, d, &fdc, 8, fDef)
case "sint64":
fieldSint(d.FieldS64, d, &fdc, 4, fDef, sintFormatter)
case "uint64", "uint64z":
fieldUint(d.FieldU64, d, &fdc, 8, fDef, uintFormatter)
fieldFloat(d.FieldF64, 8, fDef, floatFormatter)
case "string":
fieldString(d, &fdc, fDef)
fieldString(d, fDef, &fdc)
default:
d.Fatalf("Unknown type %s", fDef.typ)
d.Fatalf("Unknown type %s", fDef.Type)
}
}

if isDevDep {
ensureDevFieldMap(dmfd, fdc.devIdx, fdc.fDefNo)
dmfd[fdc.devIdx][fdc.fDefNo] = fieldDef{name: fdc.name, typ: fdc.typ, unit: fdc.unit, scale: 0, offset: 0}
ensureDevFieldMap(dmfd, fdc.devIdx)
dmfd[fdc.devIdx][fdc.fDefNo] = mappers.FieldDef{Name: fdc.name, Type: fdc.typ, Unit: fdc.unit, Scale: 0, Offset: 0}
}
}

Expand All @@ -322,12 +314,9 @@ func decodeFIT(d *decode.D) any {
var dmfd devFieldDefMap = make(devFieldDefMap)
var isDevMap localMsgIsDevDef = make(localMsgIsDevDef)

//decodeBSONDocument(d)
d.FieldStruct("header", func(d *decode.D) { fitDecodeFileHeader(d, &fc) })

d.FieldArray("dataRecords", func(d *decode.D) {
// // headerPos := d.Pos()

for d.Pos() < int64((fc.headerSize+fc.dataSize)*8) {
d.FieldStruct("dataRecord", func(d *decode.D) {
var drc dataRecordContext
Expand All @@ -344,9 +333,9 @@ func decodeFIT(d *decode.D) any {

var fileCRC uint16
if fc.headerSize == 12 {
fileCRC = calcCRC(d.BytesRange(0, fc.dataSize+fc.headerSize)) // 12 byte header - CRC whole file except the CRC itself
fileCRC = calcCRC(d.BytesRange(0, int(fc.dataSize+fc.headerSize))) // 12 byte header - CRC whole file except the CRC itself
} else {
fileCRC = calcCRC(d.BytesRange(14*8, fc.dataSize)) // 14 byte header - CRC everything below header except the CRC itself
fileCRC = calcCRC(d.BytesRange(14*8, int(fc.dataSize))) // 14 byte header - CRC everything below header except the CRC itself
}
d.FieldU16("crc", d.UintValidate(uint64(fileCRC)))

Expand Down
Loading

0 comments on commit 54c6f0c

Please sign in to comment.