From cf1d13fa905e5a0996f59fcd6ab5fc3cd97f06dc Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 12:01:33 +0200 Subject: [PATCH 01/14] [FIX] Added example and moved test data --- example/example.go | 129 ++++++++++++++++++++++ {test_data => example/test_data}/TEST.DBF | Bin {test_data => example/test_data}/TEST.FPT | Bin 3 files changed, 129 insertions(+) create mode 100644 example/example.go rename {test_data => example/test_data}/TEST.DBF (100%) rename {test_data => example/test_data}/TEST.FPT (100%) diff --git a/example/example.go b/example/example.go new file mode 100644 index 0000000..e0c3070 --- /dev/null +++ b/example/example.go @@ -0,0 +1,129 @@ +package main + +import ( + "fmt" + "time" + + "github.com/Valentin-Kaiser/go-dbase/dbase" +) + +type Test struct { + ID int32 `json:"ID"` + Niveau int32 `json:"NIVEAU"` + Date time.Time `json:"DATUM"` + TIJD string `json:"TIJD"` + SOORT float64 `json:"SOORT"` + ID_NR int32 `json:"ID_NR"` + UserNR int32 `json:"USERNR"` + CompanyName string `json:"COMP_NAME"` + CompanyOS string `json:"COMP_OS"` + Melding string `json:"MELDING"` + Number int64 `json:"NUMBER"` + Float float64 `json:"FLOAT"` + Bool bool `json:"BOOL"` +} + +func main() { + // Open the example database file. + dbf, err := dbase.Open("./test_data/TEST.DBF", new(dbase.Win1250Converter)) + if err != nil { + panic(err) + } + defer dbf.Close() + + // Print all database column infos. + for _, column := range dbf.Columns() { + fmt.Printf("Name: %v - Type: %v \n", column.Name(), column.Type()) + } + + // Read the complete first row. + row, err := dbf.Row() + if err != nil { + panic(err) + } + + // Print all the columns in their Go values as slice. + fmt.Printf("%+v", row.Values()) + + // Go back to start. + dbf.Skip(0) + + // Loop through all rows using rowPointer in DBF struct. + for !dbf.EOF() { + fmt.Printf("EOF: %v - Pointer: %v \n", dbf.EOF(), dbf.Pointer()) + + // This reads the complete row. + row, err := dbf.Row() + if err != nil { + panic(err) + } + + // Increase the pointer. + dbf.Skip(1) + + // Skip deleted rows. + if row.Deleted { + continue + } + + // Get value by column position + _, err = row.Value(0) + if err != nil { + panic(err) + } + + // Get value by column name + _, err = row.Value(dbf.ColumnPos("COMP_NAME")) + if err != nil { + panic(err) + } + + // Enable space trimming per default + dbf.SetTrimspacesDefault(true) + // Disable space trimming for the company name + dbf.SetColumnModification(dbf.ColumnPos("COMP_NAME"), false, "", nil) + // Add a column modification to switch the names of "NUMBER" and "Float" to match the data types + dbf.SetColumnModification(dbf.ColumnPos("NUMBER"), true, "FLOAT", nil) + dbf.SetColumnModification(dbf.ColumnPos("FLOAT"), true, "NUMBER", nil) + + // Read the row into a struct. + t := &Test{} + err = row.ToStruct(t) + if err != nil { + panic(err) + } + + fmt.Printf("Company: %v", t.CompanyName) + } + + // Read only the third column of rows 1, 2 and 3 + for _, row := range []uint32{1, 2, 3} { + err := dbf.GoTo(row) + if err != nil { + panic(err) + } + + // Check if the row is deleted + deleted, err := dbf.Deleted() + if err != nil { + panic(err) + } + if deleted { + fmt.Printf("Row %v deleted \n", row) + continue + } + + // Read the entire row + r, err := dbf.Row() + if err != nil { + panic(err) + } + + // Read the seventh column + column, err := r.Value(7) + if err != nil { + panic(err) + } + fmt.Printf("Row %v column 7: %v \n", row, column) + } +} diff --git a/test_data/TEST.DBF b/example/test_data/TEST.DBF similarity index 100% rename from test_data/TEST.DBF rename to example/test_data/TEST.DBF diff --git a/test_data/TEST.FPT b/example/test_data/TEST.FPT similarity index 100% rename from test_data/TEST.FPT rename to example/test_data/TEST.FPT From f18ece18df70a026427bb391e0bab794a50bdb1d Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 12:02:06 +0200 Subject: [PATCH 02/14] [FIX] Skip method no longer returns an error --- dbase/io.go | 9 +++------ dbase/table.go | 8 +++----- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/dbase/io.go b/dbase/io.go index 9627c9b..39a4ea1 100644 --- a/dbase/io.go +++ b/dbase/io.go @@ -288,19 +288,16 @@ func (dbf *DBF) GoTo(rowNumber uint32) error { } // Skip adds offset to the internal row pointer -// Returns EOF error if at end of file and positions the pointer at lastRow+1 -// Returns BOF error is the row pointer would be become negative and positions the pointer at 0 +// If at end of file positions the pointer at lastRow+1 +// If the row pointer would be become negative positions the pointer at 0 // Does not skip deleted rows -func (dbf *DBF) Skip(offset int64) error { +func (dbf *DBF) Skip(offset int64) { newval := int64(dbf.table.rowPointer) + offset if newval >= int64(dbf.dbaseHeader.RowsCount) { dbf.table.rowPointer = dbf.dbaseHeader.RowsCount - return fmt.Errorf("dbase-io-skip-1:FAILED:%v", ERROR_EOF.AsError()) } if newval < 0 { dbf.table.rowPointer = 0 - return fmt.Errorf("dbase-io-skip-2:FAILED:%v", ERROR_BOF.AsError()) } dbf.table.rowPointer = uint32(newval) - return nil } diff --git a/dbase/table.go b/dbase/table.go index 526a908..d6ba16b 100644 --- a/dbase/table.go +++ b/dbase/table.go @@ -211,10 +211,8 @@ func (dbf *DBF) Rows(skipInvalid bool) ([]*Row, error) { return nil, fmt.Errorf("dbase-table-rows-1:FAILED:%v", err) } - err = dbf.Skip(1) - if err != nil { - return nil, fmt.Errorf("dbase-table-rows-2:FAILED:%v", err) - } + // Increment the row pointer + dbf.Skip(1) // skip deleted rows if row.Deleted { @@ -237,7 +235,7 @@ func (dbf *DBF) Row() (*Row, error) { return dbf.BytesToRow(data) } -// Column gets a column value by column pos (index) +// Value gets a column value by column pos (index) func (r *Row) Value(pos int) (interface{}, error) { if pos < 0 || len(r.Data) < pos { return 0, fmt.Errorf("dbase-table-column-1:FAILED:%v", ERROR_INVALID.AsError()) From f4fedd486fa65c193ec06e7552c48664f108d0ba Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 12:51:54 +0200 Subject: [PATCH 03/14] [ADD] .golangci-lint.yml --- .golangci.yml | 49 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .golangci.yml diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..094e872 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,49 @@ +run: + timeout: 60s + +linters: + # start with everything + enable-all: true + + disable: + # deprecated + - golint + - interfacer + - maligned + - scopelint + - deadcode + - varcheck + - structcheck + + # too annoying + - cyclop + - exhaustive + - exhaustivestruct + - exhaustruct + - forbidigo + - funlen + - gochecknoglobals + - godot + - goerr113 + - gofumpt + - gomnd + - lll + - nakedret + - nestif + - nlreturn + - tagliatelle + - varnamelen + - wsl + - nosnakecase + - ifshort + - gci + + # disabled because of generics + - rowserrcheck + - sqlclosecheck + - structcheck + - wastedassign + +linters-settings: + wsl: + allow-cuddle-declarations: true From 09746a85d60baad48bcadeb04461a27cebffcb41 Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 12:52:07 +0200 Subject: [PATCH 04/14] [FIX] Renamed constants --- dbase/constants.go | 58 +++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 32 deletions(-) diff --git a/dbase/constants.go b/dbase/constants.go index 31f2821..76a1fff 100644 --- a/dbase/constants.go +++ b/dbase/constants.go @@ -1,42 +1,36 @@ package dbase -import "fmt" - -type DBaseError string +type Error string const ( - // returned when the end of a dBase database file is reached - ERROR_EOF DBaseError = "EOF" - // returned when the row pointer is attempted to be moved before the first row - ERROR_BOF DBaseError = "BOF" - // returned when the read of a row or column did not finish - ERROR_INCOMPLETE DBaseError = "INCOMPLETE" - // returned when an invalid column position is used (x<1 or x>number of columns) - ERROR_INVALID DBaseError = "INVALID" - // returned when a file operation is attempted on a non existent file - ERROR_NO_DBF_FILE DBaseError = "FPT_FILE_NOT_FOUND" - ERROR_NO_FPT_FILE DBaseError = "DBF_FILE_NOT_FOUND" - ERROR_INVALID_ENCODING DBaseError = "INVALID_ENCODING" + // Returned when the end of a dBase database file is reached + EOF Error = "EOF" + // Returned when the row pointer is attempted to be moved before the first row + BOF Error = "BOF" + // Returned when the read of a row or column did not finish + Incomplete Error = "INCOMPLETE" + // Returned when a file operation is attempted on a non existent file + NoFPT Error = "FPT_FILE_NOT_FOUND" + NoDBF Error = "DBF_FILE_NOT_FOUND" + // Returned when an invalid column position is used (x<1 or x>number of columns) + InvalidPosition Error = "INVALID_Position" + InvalidEncoding Error = "INVALID_ENCODING" // Supported file types - FOXPRO byte = 0x30 - FOXPRO_AUTOINCREMENT byte = 0x31 + FoxPro byte = 0x30 + FoxProAutoincrement byte = 0x31 // Relevant byte marker - NULL byte = 0x00 - BLANK byte = 0x20 - END_OF_COLUMN byte = 0x0D - ACTIVE = BLANK - DELETED = 0x2A - EOF_MARKER byte = 0x1A + Null byte = 0x00 + Blank byte = 0x20 + ColumnEnd byte = 0x0D + Active = Blank + Deleted = 0x2A + EOFMarker byte = 0x1A - // dBase Table flags - STRUCTURAL byte = 0x01 - MEMO byte = 0x02 - STRUCTURAL_MEMO byte = 0x03 - DATABASE byte = 0x04 + // DBase Table flags + Structural byte = 0x01 + Memo byte = 0x02 + StructuralMemo byte = 0x03 + Database byte = 0x04 ) - -func (re DBaseError) AsError() error { - return fmt.Errorf(string(re)) -} From eeda0d8311666a252521a2159f1138a5802824f5 Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 12:52:29 +0200 Subject: [PATCH 05/14] [FIX] Conversion error handling and optimization --- dbase/conversion.go | 58 +++++++++++++++++++++++++++------------------ 1 file changed, 35 insertions(+), 23 deletions(-) diff --git a/dbase/conversion.go b/dbase/conversion.go index 1a47be5..468a934 100644 --- a/dbase/conversion.go +++ b/dbase/conversion.go @@ -8,8 +8,8 @@ import ( "time" ) -// convert year, month and day to a julian day number -// julian day number -> days since 01-01-4712 BC +// Convert year, month and day to a julian day number +// Julian day number -> days since 01-01-4712 BC func YMD2JD(y, m, d int) int { return d - 32075 + 1461*(y+4800+(m-14)/12)/4 + @@ -17,12 +17,12 @@ func YMD2JD(y, m, d int) int { 3*((y+4900+(m-14)/12)/100)/4 } -// convert julian day number to year, month and day -// julian day number -> days since 01-01-4712 BC +// Convert julian day number to year, month and day +// Julian day number -> days since 01-01-4712 BC func JD2YMD(date int) (int, int, int) { l := date + 68569 n := 4 * l / 146097 - l = l - (146097*n+3)/4 + l -= (146097*n + 3) / 4 y := 4000 * (l + 1) / 1461001 l = l - 1461*y/4 + 31 m := 80 * l / 2447 @@ -33,23 +33,24 @@ func JD2YMD(date int) (int, int, int) { return y, m, d } -// convert julian day number to golang time.Time -// julian day number -> days since 01-01-4712 BC +// Convert julian day number to golang time.Time +// Julian day number -> days since 01-01-4712 BC func JDToDate(number int) (time.Time, error) { y, m, d := JD2YMD(number) ys := strconv.Itoa(y) ms := strconv.Itoa(m) ds := strconv.Itoa(d) - if m < 10 { ms = "0" + ms } - if d < 10 { ds = "0" + ds } - - return time.Parse("2006-01-02", ys+"-"+ms+"-"+ds) + t, err := time.Parse("2006-01-02", ys+"-"+ms+"-"+ds) + if err != nil { + return t, fmt.Errorf("dbase-conversion-jdtodate-1:FAILED:%w", err) + } + return t, nil } /** @@ -58,57 +59,68 @@ func JDToDate(number int) (time.Time, error) { * ################################################################ */ +// parseDate parses a date string from a byte slice and returns a time.Time func (dbf *DBF) parseDate(raw []byte) (time.Time, error) { if string(raw) == strings.Repeat(" ", 8) { return time.Time{}, nil } - return time.Parse("20060102", string(raw)) + t, err := time.Parse("20060102", string(raw)) + if err != nil { + return t, fmt.Errorf("dbase-interpreter-parsedate-1:FAILED:%w", err) + } + return t, nil } +// parseDateTIme parses a date and time string from a byte slice and returns a time.Time func (dbf *DBF) parseDateTime(raw []byte) (time.Time, error) { if len(raw) != 8 { - return time.Time{}, fmt.Errorf("dbase-conversion-parsedate-1:FAILED:%v", ERROR_INVALID.AsError()) + return time.Time{}, fmt.Errorf("dbase-conversion-parsedate-1:FAILED:%v", InvalidPosition) } julDat := int(binary.LittleEndian.Uint32(raw[:4])) mSec := int(binary.LittleEndian.Uint32(raw[4:])) - // Determine year, month, day y, m, d := JD2YMD(julDat) if y < 0 || y > 9999 { return time.Time{}, nil } - // Calculate whole seconds and use the remainder as nanosecond resolution nSec := mSec / 1000 - mSec = mSec - (nSec * 1000) - + mSec -= (nSec * 1000) // Create time using ymd and nanosecond timestamp return time.Date(y, time.Month(m), d, 0, 0, nSec, mSec*int(time.Millisecond), time.UTC), nil } -// parses a string as byte array to int64 +// parseNumericInt parses a string as byte array to int64 func (dbf *DBF) parseNumericInt(raw []byte) (int64, error) { trimmed := strings.TrimSpace(string(raw)) if len(trimmed) == 0 { return int64(0), nil } - return strconv.ParseInt(trimmed, 10, 64) + i, err := strconv.ParseInt(trimmed, 10, 64) + if err != nil { + return i, fmt.Errorf("dbase-conversion-parseint-1:FAILED:%w", err) + } + return i, nil } -// parses a string as byte array to float64 +// parseFloat parses a string as byte array to float64 func (dbf *DBF) parseFloat(raw []byte) (float64, error) { trimmed := strings.TrimSpace(string(raw)) if len(trimmed) == 0 { - return float64(0.0), nil + return float64(0), nil + } + f, err := strconv.ParseFloat(strings.TrimSpace(trimmed), 64) + if err != nil { + return f, fmt.Errorf("dbase-conversion-parsefloat-1:FAILED:%w", err) } - return strconv.ParseFloat(strings.TrimSpace(string(trimmed)), 64) + return f, nil } // toUTF8String converts a byte slice to a UTF8 string using the converter func (dbf *DBF) toUTF8String(raw []byte) (string, error) { utf8, err := dbf.convert.Decode(raw) if err != nil { - return string(raw), fmt.Errorf("dbase-conversion-toutf8string-1:FAILED:%v", err) + return string(raw), fmt.Errorf("dbase-conversion-toutf8string-1:FAILED:%w", err) } return string(utf8), nil } From 4250ce53e9cb7340f9c038ef29222bb7f215ead8 Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 12:52:40 +0200 Subject: [PATCH 06/14] [FIX] Encoding error handling --- dbase/encoding.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/dbase/encoding.go b/dbase/encoding.go index 822b78f..145afe5 100644 --- a/dbase/encoding.go +++ b/dbase/encoding.go @@ -11,7 +11,7 @@ import ( "golang.org/x/text/transform" ) -// Converter is the interface as passed to Open +// EncodingConverter is the interface as passed to Open type EncodingConverter interface { Decode(in []byte) ([]byte, error) Encode(in []byte) ([]byte, error) @@ -28,7 +28,7 @@ func (d *Win1250Converter) Decode(in []byte) ([]byte, error) { r := transform.NewReader(bytes.NewReader(in), charmap.Windows1250.NewDecoder()) data, err := io.ReadAll(r) if err != nil { - return nil, fmt.Errorf("dbase-encoding-decode-1:FAILED:%v", err) + return nil, fmt.Errorf("dbase-encoding-decode-1:FAILED:%w", err) } return data, nil } @@ -39,8 +39,7 @@ func (d *Win1250Converter) Encode(in []byte) ([]byte, error) { enc := charmap.Windows1250.NewEncoder() nDst, _, err := enc.Transform(out, in, false) if err != nil { - return nil, fmt.Errorf("dbase-encoding-encode-1:FAILED:%v", err) + return nil, fmt.Errorf("dbase-encoding-encode-1:FAILED:%w", err) } - return out[:nDst], nil } From 455759daf7ee65d8d913653fff987ad05c2d213e Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 12:53:05 +0200 Subject: [PATCH 07/14] [FIX] Interpreter error handling and optimization --- dbase/interpreter.go | 58 +++++++++++++++++++++----------------------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/dbase/interpreter.go b/dbase/interpreter.go index 42657d4..e89fa29 100644 --- a/dbase/interpreter.go +++ b/dbase/interpreter.go @@ -3,24 +3,23 @@ // // The supported column types with their return Go types are: // -// Column Type >> Column Type Name >> Golang type +// Column Type >> Column Type Name >> Golang type // -// B >> Double >> float64 -// C >> Character >> string -// D >> Date >> time.Time -// F >> Float >> float64 -// I >> Integer >> int32 -// L >> Logical >> bool -// M >> Memo >> string -// M >> Memo (Binary) >> []byte -// N >> Numeric (0 decimals) >> int64 -// N >> Numeric (with decimals) >> float64 -// T >> DateTime >> time.Time -// Y >> Currency >> float64 +// B >> Double >> float64 +// C >> Character >> string +// D >> Date >> time.Time +// F >> Float >> float64 +// I >> Integer >> int32 +// L >> Logical >> bool +// M >> Memo >> string +// M >> Memo (Binary) >> []byte +// N >> Numeric (0 decimals) >> int64 +// N >> Numeric (with decimals) >> float64 +// T >> DateTime >> time.Time +// Y >> Currency >> float64 // // This module contains the functions to convert a dbase database entry as byte array into a row struct // with the columns converted into the corresponding data types. -// package dbase import ( @@ -39,14 +38,13 @@ func (dbf *DBF) DataToValue(raw []byte, column *Column) (interface{}, error) { if len(raw) != int(column.Length) { return nil, fmt.Errorf("dbase-interpreter-datatovalue-1:FAILED:invalid length %v Bytes != %v Bytes", len(raw), column.Length) } - switch column.Type() { case "M": // M values contain the address in the FPT file from where to read data memo, isText, err := dbf.parseMemo(raw) if isText { if err != nil { - return string(memo), fmt.Errorf("dbase-interpreter-datatovalue-2:FAILED:%v", err) + return string(memo), fmt.Errorf("dbase-interpreter-datatovalue-2:FAILED:%w", err) } return string(memo), nil } @@ -55,7 +53,7 @@ func (dbf *DBF) DataToValue(raw []byte, column *Column) (interface{}, error) { // C values are stored as strings, the returned string is not trimmed str, err := dbf.toUTF8String(raw) if err != nil { - return str, fmt.Errorf("dbase-interpreter-datatovalue-4:FAILED:%v", err) + return str, fmt.Errorf("dbase-interpreter-datatovalue-4:FAILED:%w", err) } return str, nil case "I": @@ -68,7 +66,7 @@ func (dbf *DBF) DataToValue(raw []byte, column *Column) (interface{}, error) { // D values are stored as string in format YYYYMMDD, convert to time.Time date, err := dbf.parseDate(raw) if err != nil { - return date, fmt.Errorf("dbase-interpreter-datatovalue-5:FAILED:%v", err) + return date, fmt.Errorf("dbase-interpreter-datatovalue-5:FAILED:%w", err) } return date, nil case "T": @@ -78,7 +76,7 @@ func (dbf *DBF) DataToValue(raw []byte, column *Column) (interface{}, error) { // Above info from http://fox.wikis.com/wc.dll?Wiki~DateTime dateTime, err := dbf.parseDateTime(raw) if err != nil { - return dateTime, fmt.Errorf("dbase-interpreter-datatovalue-6:FAILED:%v", err) + return dateTime, fmt.Errorf("dbase-interpreter-datatovalue-6:FAILED:%w", err) } return dateTime, nil case "L": @@ -89,13 +87,13 @@ func (dbf *DBF) DataToValue(raw []byte, column *Column) (interface{}, error) { return raw, nil case "Y": // Y values are currency values stored as ints with 4 decimal places - return float64(float64(binary.LittleEndian.Uint64(raw)) / 10000), nil + return float64(binary.LittleEndian.Uint64(raw)) / 10000, nil case "N": // N values are stored as string values, if no decimals return as int64, if decimals treat as float64 if column.Decimals == 0 { i, err := dbf.parseNumericInt(raw) if err != nil { - return i, fmt.Errorf("dbase-interpreter-datatovalue-7:FAILED:%v", err) + return i, fmt.Errorf("dbase-interpreter-datatovalue-7:FAILED:%w", err) } return i, nil } @@ -104,7 +102,7 @@ func (dbf *DBF) DataToValue(raw []byte, column *Column) (interface{}, error) { // F values are stored as string values f, err := dbf.parseFloat(raw) if err != nil { - return f, fmt.Errorf("dbase-interpreter-datatovalue-8:FAILED:%v", err) + return f, fmt.Errorf("dbase-interpreter-datatovalue-8:FAILED:%w", err) } return f, nil default: @@ -114,22 +112,20 @@ func (dbf *DBF) DataToValue(raw []byte, column *Column) (interface{}, error) { // Returns if the row at internal row pointer is deleted func (dbf *DBF) Deleted() (bool, error) { - if dbf.table.rowPointer >= dbf.dbaseHeader.RowsCount { - return false, fmt.Errorf("dbase-interpreter-deleted-1:FAILED:%v", ERROR_EOF.AsError()) + if dbf.table.rowPointer >= dbf.header.RowsCount { + return false, fmt.Errorf("dbase-interpreter-deleted-1:FAILED:%v", EOF) } - - _, err := syscall.Seek(syscall.Handle(*dbf.dbaseFileHandle), int64(dbf.dbaseHeader.FirstRow)+(int64(dbf.table.rowPointer)*int64(dbf.dbaseHeader.RowLength)), 0) + _, err := syscall.Seek(syscall.Handle(*dbf.dbaseFileHandle), int64(dbf.header.FirstRow)+(int64(dbf.table.rowPointer)*int64(dbf.header.RowLength)), 0) if err != nil { - return false, fmt.Errorf("dbase-interpreter-deleted-2:FAILED:%v", err) + return false, fmt.Errorf("dbase-interpreter-deleted-2:FAILED:%w", err) } - buf := make([]byte, 1) read, err := syscall.Read(syscall.Handle(*dbf.dbaseFileHandle), buf) if err != nil { - return false, fmt.Errorf("dbase-interpreter-deleted-3:FAILED:%v", err) + return false, fmt.Errorf("dbase-interpreter-deleted-3:FAILED:%w", err) } if read != 1 { - return false, fmt.Errorf("dbase-interpreter-deleted-4:FAILED:%v", ERROR_INCOMPLETE.AsError()) + return false, fmt.Errorf("dbase-interpreter-deleted-4:FAILED:%v", Incomplete) } - return buf[0] == DELETED, nil + return buf[0] == Deleted, nil } From 87b8e143a50a7645b42a6faaa05f13e4197174dd Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 12:53:44 +0200 Subject: [PATCH 08/14] [FIX] Renamed DBaseHeader to header, improved errors and optimized --- dbase/table.go | 113 ++++++++++++++++++++----------------------------- 1 file changed, 47 insertions(+), 66 deletions(-) diff --git a/dbase/table.go b/dbase/table.go index d6ba16b..f1d2ea0 100644 --- a/dbase/table.go +++ b/dbase/table.go @@ -11,7 +11,7 @@ import ( // Containing DBF header information like dBase FileType, last change and rows count. // https://docs.microsoft.com/en-us/previous-versions/visualstudio/foxpro/st4a0s68(v=vs.80)#table-header-record-structure -type DBaseHeader struct { +type Header struct { FileType byte // File type flag Year uint8 // Last update year (0-99) Month uint8 // Last update month @@ -49,6 +49,7 @@ type Column struct { Reserved [8]byte // Reserved } +// ColumnModification contains the modification to change values or name of columns type ColumnModification struct { Trimspaces bool Convert func(interface{}) interface{} @@ -71,19 +72,19 @@ type Row struct { // Parses the year, month and day to time.Time. // Note: the year is stored in 2 digits, 15 is 2015 -func (h *DBaseHeader) Modified() time.Time { +func (h *Header) Modified() time.Time { return time.Date(2000+int(h.Year), time.Month(h.Month), int(h.Day), 0, 0, 0, 0, time.Local) } // Returns the calculated number of columns from the header info alone (without the need to read the columninfo from the header). // This is the fastest way to determine the number of rows in the file. // Note: when OpenFile is used the columns have already been parsed so it is better to call DBF.ColumnsCount in that case. -func (h *DBaseHeader) ColumnsCount() uint16 { - return uint16((h.FirstRow - 296) / 32) +func (h *Header) ColumnsCount() uint16 { + return (h.FirstRow - 296) / 32 } // Returns the calculated file size based on the header info -func (h *DBaseHeader) FileSize() int64 { +func (h *Header) FileSize() int64 { return 296 + int64(h.ColumnsCount()*32) + int64(h.RowsCount*uint32(h.RowLength)) } @@ -95,7 +96,7 @@ func (h *DBaseHeader) FileSize() int64 { // Returns if the internal row pointer is at end of file func (dbf *DBF) EOF() bool { - return dbf.table.rowPointer >= dbf.dbaseHeader.RowsCount + return dbf.table.rowPointer >= dbf.header.RowsCount } // Returns if the internal row pointer is before first row @@ -109,13 +110,13 @@ func (dbf *DBF) Pointer() uint32 { } // Returns the dBase database file header struct for inspecting -func (dbf *DBF) Header() *DBaseHeader { - return dbf.dbaseHeader +func (dbf *DBF) Header() *Header { + return dbf.header } // returns the number of rows func (dbf *DBF) RowsCount() uint32 { - return dbf.dbaseHeader.RowsCount + return dbf.header.RowsCount } // Returns all columns infos @@ -153,7 +154,6 @@ func (dbf *DBF) SetColumnModification(position int, trimspaces bool, key string, if position < 0 || position >= len(dbf.table.columns) { return } - dbf.table.columnMods[position] = &ColumnModification{ Trimspaces: trimspaces, Convert: convert, @@ -173,7 +173,7 @@ func (dbf *DBF) GetColumnModification(position int) *ColumnModification { func (dbf *DBF) Value(columnposition int) (interface{}, error) { data, err := dbf.readColumn(dbf.table.rowPointer, columnposition) if err != nil { - return nil, fmt.Errorf("dbase-table-value-1:FAILED:%v", err) + return nil, fmt.Errorf("dbase-table-value-1:FAILED:%w", err) } // columnposition is valid or readColumn would have returned an error return dbf.DataToValue(data, dbf.table.columns[columnposition]) @@ -208,20 +208,16 @@ func (dbf *DBF) Rows(skipInvalid bool) ([]*Row, error) { // This reads the complete row row, err := dbf.Row() if err != nil && !skipInvalid { - return nil, fmt.Errorf("dbase-table-rows-1:FAILED:%v", err) + return nil, fmt.Errorf("dbase-table-rows-1:FAILED:%w", err) } - // Increment the row pointer dbf.Skip(1) - // skip deleted rows if row.Deleted { continue } - rows = append(rows, row) } - return rows, nil } @@ -229,44 +225,40 @@ func (dbf *DBF) Rows(skipInvalid bool) ([]*Row, error) { func (dbf *DBF) Row() (*Row, error) { data, err := dbf.readRow(dbf.table.rowPointer) if err != nil { - return nil, fmt.Errorf("dbase-table-get-row-1:FAILED:%v", err) + return nil, fmt.Errorf("dbase-table-get-row-1:FAILED:%w", err) } - return dbf.BytesToRow(data) } // Value gets a column value by column pos (index) -func (r *Row) Value(pos int) (interface{}, error) { - if pos < 0 || len(r.Data) < pos { - return 0, fmt.Errorf("dbase-table-column-1:FAILED:%v", ERROR_INVALID.AsError()) +func (row *Row) Value(pos int) (interface{}, error) { + if pos < 0 || len(row.Data) < pos { + return 0, fmt.Errorf("dbase-table-column-1:FAILED:%v", InvalidPosition) } - return r.Data[pos], nil + return row.Data[pos], nil } // Values gets all columns as a slice -func (r *Row) Values() []interface{} { - return r.Data +func (row *Row) Values() []interface{} { + return row.Data } // Reads raw row data of one row at rowPosition func (dbf *DBF) readRow(rowPosition uint32) ([]byte, error) { - if rowPosition >= dbf.dbaseHeader.RowsCount { - return nil, fmt.Errorf("dbase-table-read-row-1:FAILED:%v", ERROR_EOF.AsError()) + if rowPosition >= dbf.header.RowsCount { + return nil, fmt.Errorf("dbase-table-read-row-1:FAILED:%v", EOF) } - buf := make([]byte, dbf.dbaseHeader.RowLength) - - _, err := syscall.Seek(syscall.Handle(*dbf.dbaseFileHandle), int64(dbf.dbaseHeader.FirstRow)+(int64(rowPosition)*int64(dbf.dbaseHeader.RowLength)), 0) + buf := make([]byte, dbf.header.RowLength) + _, err := syscall.Seek(syscall.Handle(*dbf.dbaseFileHandle), int64(dbf.header.FirstRow)+(int64(rowPosition)*int64(dbf.header.RowLength)), 0) if err != nil { - return buf, fmt.Errorf("dbase-table-read-row-2:FAILED:%v", err) + return buf, fmt.Errorf("dbase-table-read-row-2:FAILED:%w", err) } - read, err := syscall.Read(syscall.Handle(*dbf.dbaseFileHandle), buf) if err != nil { - return buf, fmt.Errorf("dbase-table-read-row-3:FAILED:%v", err) + return buf, fmt.Errorf("dbase-table-read-row-3:FAILED:%w", err) } - - if read != int(dbf.dbaseHeader.RowLength) { - return buf, fmt.Errorf("dbase-table-read-row-1:FAILED:%v", ERROR_INCOMPLETE.AsError()) + if read != int(dbf.header.RowLength) { + return buf, fmt.Errorf("dbase-table-read-row-1:FAILED:%v", Incomplete) } return buf, nil } @@ -277,29 +269,25 @@ func (dbf *DBF) BytesToRow(data []byte) (*Row, error) { rec := &Row{} rec.DBF = dbf rec.Data = make([]interface{}, dbf.ColumnsCount()) - - if len(data) < int(dbf.dbaseHeader.RowLength) { - return nil, fmt.Errorf("dbase-table-bytestorow-1:FAILED:invalid row data size %v Bytes < %v Bytes", len(data), int(dbf.dbaseHeader.RowLength)) + if len(data) < int(dbf.header.RowLength) { + return nil, fmt.Errorf("dbase-table-bytestorow-1:FAILED:invalid row data size %v Bytes < %v Bytes", len(data), int(dbf.header.RowLength)) } - // a row should start with te delete flag, a space ACTIVE(0x20) or DELETED(0x2A) - rec.Deleted = data[0] == DELETED - if !rec.Deleted && data[0] != ACTIVE { + rec.Deleted = data[0] == Deleted + if !rec.Deleted && data[0] != Active { return nil, fmt.Errorf("dbase-table-bytestorow-2:FAILED:invalid row data, no delete flag found at beginning of row") } - // deleted flag already read offset := uint16(1) for i := 0; i < len(rec.Data); i++ { columninfo := dbf.table.columns[i] val, err := dbf.DataToValue(data[offset:offset+uint16(columninfo.Length)], dbf.table.columns[i]) if err != nil { - return rec, fmt.Errorf("dbase-table-bytestorow-3:FAILED:%v", err) + return rec, fmt.Errorf("dbase-table-bytestorow-3:FAILED:%w", err) } rec.Data[i] = val offset += uint16(columninfo.Length) } - return rec, nil } @@ -312,12 +300,10 @@ func (dbf *DBF) BytesToRow(data []byte) (*Row, error) { // Returns all rows as a slice of maps. func (dbf *DBF) RowsToMap(skipInvalid bool) ([]map[string]interface{}, error) { out := make([]map[string]interface{}, 0) - rows, err := dbf.Rows(skipInvalid) if err != nil { return nil, err } - for _, row := range rows { rmap, err := row.ToMap() if err != nil { @@ -326,7 +312,6 @@ func (dbf *DBF) RowsToMap(skipInvalid bool) ([]map[string]interface{}, error) { out = append(out, rmap) } - return out, nil } @@ -335,23 +320,24 @@ func (dbf *DBF) RowsToMap(skipInvalid bool) ([]map[string]interface{}, error) { func (dbf *DBF) RowsToJSON(skipInvalid bool) ([]byte, error) { rows, err := dbf.RowsToMap(skipInvalid) if err != nil { - return nil, fmt.Errorf("dbase-table-to-json-1:FAILED:%v", err) + return nil, fmt.Errorf("dbase-table-rows-to-json-1:FAILED:%w", err) } - mapRows := make([]map[string]interface{}, 0) for _, row := range rows { for k, v := range row { if dbf.table.columnMods[dbf.ColumnPos(k)].Trimspaces { if str, ok := v.(string); ok { - row[k] = strings.TrimSpace(str) } } } mapRows = append(mapRows, row) } - - return json.Marshal(mapRows) + j, err := json.Marshal(mapRows) + if err != nil { + return j, fmt.Errorf("dbase-table-rows-to-json-2:FAILED:%w", err) + } + return j, nil } // Returns all rows as a slice of struct. @@ -363,21 +349,17 @@ func (dbf *DBF) RowsToJSON(skipInvalid bool) ([]byte, error) { // If trimspaces is true we trim spaces from string values (this is slower because of an extra reflect operation and all strings in the row map are re-assigned) func (dbf *DBF) RowsToStruct(v interface{}, skipInvalid bool) ([]interface{}, error) { out := make([]interface{}, 0) - rows, err := dbf.Rows(skipInvalid) if err != nil { return nil, err } - for _, row := range rows { err := row.ToStruct(v) if err != nil { return nil, err } - out = append(out, v) } - return out, nil } @@ -387,7 +369,7 @@ func (row *Row) ToMap() (map[string]interface{}, error) { for i, cn := range row.DBF.ColumnNames() { val, err := row.Value(i) if err != nil { - return out, fmt.Errorf("dbase-table-to-map-1:FAILED:error on column %s (column %d): %s", cn, i, err) + return out, fmt.Errorf("dbase-table-to-map-1:FAILED:error on column %s (column %d): %w", cn, i, err) } colMod := row.DBF.table.columnMods[i] if colMod != nil { @@ -396,17 +378,14 @@ func (row *Row) ToMap() (map[string]interface{}, error) { val = strings.TrimSpace(str) } } - if colMod.Convert != nil { val = colMod.Convert(val) } - if len(colMod.ExternalKey) != 0 { out[colMod.ExternalKey] = val continue } } - out[cn] = val } return out, nil @@ -417,9 +396,13 @@ func (row *Row) ToMap() (map[string]interface{}, error) { func (row *Row) ToJSON() ([]byte, error) { m, err := row.ToMap() if err != nil { - return nil, fmt.Errorf("dbase-table-to-json-1:FAILED:%v", err) + return nil, fmt.Errorf("dbase-table-row-to-json-1:FAILED:%w", err) } - return json.Marshal(m) + j, err := json.Marshal(m) + if err != nil { + return j, fmt.Errorf("dbase-table-row-to-json-2:FAILED:%w", err) + } + return j, nil } // Parses the row from map to JSON-encoded data and stores the result in the value pointed to by v. @@ -429,13 +412,11 @@ func (row *Row) ToJSON() ([]byte, error) { func (row *Row) ToStruct(v interface{}) error { jsonRow, err := row.ToJSON() if err != nil { - return fmt.Errorf("dbase-table-to-struct-1:FAILED:%v", err) + return fmt.Errorf("dbase-table-to-struct-1:FAILED:%w", err) } - err = json.Unmarshal(jsonRow, v) if err != nil { - return fmt.Errorf("dbase-table-to-struct-2:FAILED:%v", err) + return fmt.Errorf("dbase-table-to-struct-2:FAILED:%w", err) } - return nil } From f95f1b9ae21f065aab598100339d975369682090 Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 12:53:56 +0200 Subject: [PATCH 09/14] [FIX] Applied changed header name --- dbase/io.go | 151 +++++++++++++++++++++------------------------------- 1 file changed, 62 insertions(+), 89 deletions(-) diff --git a/dbase/io.go b/dbase/io.go index 39a4ea1..becbb40 100644 --- a/dbase/io.go +++ b/dbase/io.go @@ -10,16 +10,17 @@ import ( syscall "golang.org/x/sys/windows" ) +// DBF is the main struct to handle a dBase file. type DBF struct { - // the used converter instance passed by opening a file + // The used converter instance passed by opening a file convert EncodingConverter - // dBase and memo file syscall handle pointer + // DBase and memo file syscall handle pointer dbaseFileHandle *syscall.Handle memoFileHandle *syscall.Handle - // dBase and memo file header containing relevant information - dbaseHeader *DBaseHeader - memoHeader *MemoHeader - // containing the columns and internal row pointer + // DBase and memo file header containing relevant information + header *Header + memoHeader *MemoHeader + // Containing the columns and internal row pointer table *Table } @@ -33,24 +34,20 @@ type DBF struct { // To close the embedded file handle(s) call DBF.Close(). func Open(filename string, conv EncodingConverter) (*DBF, error) { filename = filepath.Clean(filename) - - // open file in non blocking mode with syscall + // Open file in non blocking mode with syscall fd, err := syscall.Open(filename, syscall.O_RDWR|syscall.O_CLOEXEC|syscall.O_NONBLOCK, 0644) if err != nil { - return nil, fmt.Errorf("dbase-io-open-1:FAILED:%v", err) + return nil, fmt.Errorf("dbase-io-open-1:FAILED:%w", err) } - dbf, err := prepareDBF(fd, conv) if err != nil { - return nil, fmt.Errorf("dbase-io-open-2:FAILED:%v", err) + return nil, fmt.Errorf("dbase-io-open-2:FAILED:%w", err) } - dbf.dbaseFileHandle = &fd - // Check if there is an FPT according to the header. // If there is we will try to open it in the same dir (using the same filename and case). // If the FPT file does not exist an error is returned. - if (dbf.dbaseHeader.TableFlags & MEMO) != 0 { + if (dbf.header.TableFlags & Memo) != 0 { ext := filepath.Ext(filename) fptExt := ".fpt" if strings.ToUpper(ext) == ext { @@ -58,17 +55,14 @@ func Open(filename string, conv EncodingConverter) (*DBF, error) { } fd, err := syscall.Open(strings.TrimSuffix(filename, ext)+fptExt, syscall.O_RDWR|syscall.O_CLOEXEC|syscall.O_NONBLOCK, 0644) if err != nil { - return nil, fmt.Errorf("dbase-io-open-3:FAILED:%v", err) + return nil, fmt.Errorf("dbase-io-open-3:FAILED:%w", err) } - err = dbf.prepareMemo(fd) if err != nil { - return nil, fmt.Errorf("dbase-io-open-4:FAILED:%v", err) + return nil, fmt.Errorf("dbase-io-open-4:FAILED:%w", err) } - dbf.memoFileHandle = &fd } - return dbf, nil } @@ -77,17 +71,15 @@ func (dbf *DBF) Close() error { if dbf.dbaseFileHandle != nil { err := syscall.Close(*dbf.dbaseFileHandle) if err != nil { - return fmt.Errorf("dbase-io-close-1:FAILED:Closing DBF failed with error: %v", err) + return fmt.Errorf("dbase-io-close-1:FAILED:Closing DBF failed with error: %w", err) } } - if dbf.memoFileHandle != nil { err := syscall.Close(*dbf.memoFileHandle) if err != nil { - return fmt.Errorf("dbase-io-close-2:FAILED:Closing FPT failed with error: %v", err) + return fmt.Errorf("dbase-io-close-2:FAILED:Closing FPT failed with error: %w", err) } } - return nil } @@ -102,22 +94,18 @@ func (dbf *DBF) Close() error { func prepareDBF(fd syscall.Handle, conv EncodingConverter) (*DBF, error) { header, err := readDBFHeader(fd) if err != nil { - return nil, fmt.Errorf("dbase-io-preparedbf-1:FAILED:%v", err) + return nil, fmt.Errorf("dbase-io-preparedbf-1:FAILED:%w", err) } - - // check if the fileversion flag is expected, expand validFileVersion if needed + // Check if the fileversion flag is expected, expand validFileVersion if needed if err := validateFileVersion(header.FileType); err != nil { - return nil, fmt.Errorf("dbase-io-preparedbf-2:FAILED:%v", err) + return nil, fmt.Errorf("dbase-io-preparedbf-2:FAILED:%w", err) } - - // read columninfo columns, err := readColumnInfos(fd) if err != nil { - return nil, fmt.Errorf("dbase-io-preparedbf-3:FAILED:%v", err) + return nil, fmt.Errorf("dbase-io-preparedbf-3:FAILED:%w", err) } - dbf := &DBF{ - dbaseHeader: header, + header: header, dbaseFileHandle: &fd, table: &Table{ columns: columns, @@ -128,51 +116,45 @@ func prepareDBF(fd syscall.Handle, conv EncodingConverter) (*DBF, error) { return dbf, nil } -func readDBFHeader(fd syscall.Handle) (*DBaseHeader, error) { - h := &DBaseHeader{} - if _, err := syscall.Seek(syscall.Handle(fd), 0, 0); err != nil { - return nil, fmt.Errorf("dbase-io-readdbfheader-1:FAILED:%v", err) +// Reads the DBF header from the file handle. +func readDBFHeader(fd syscall.Handle) (*Header, error) { + h := &Header{} + if _, err := syscall.Seek(fd, 0, 0); err != nil { + return nil, fmt.Errorf("dbase-io-readdbfheader-1:FAILED:%w", err) } - b := make([]byte, 1024) - n, err := syscall.Read(syscall.Handle(fd), b) + n, err := syscall.Read(fd, b) if err != nil { - return nil, fmt.Errorf("dbase-io-readdbfheader-2:FAILED:%v", err) + return nil, fmt.Errorf("dbase-io-readdbfheader-2:FAILED:%w", err) } - - // integers in table files are stored with the least significant byte first. + // LittleEndian - Integers in table files are stored with the least significant byte first. err = binary.Read(bytes.NewReader(b[:n]), binary.LittleEndian, h) if err != nil { - return nil, fmt.Errorf("dbase-io-readdbfheader-3:FAILED:%v", err) + return nil, fmt.Errorf("dbase-io-readdbfheader-3:FAILED:%w", err) } return h, nil } // Reads raw column data of one column at columnPosition at rowPosition func (dbf *DBF) readColumn(rowPosition uint32, columnPosition int) ([]byte, error) { - if rowPosition >= dbf.dbaseHeader.RowsCount { - return nil, fmt.Errorf("dbase-io-readcolumn-1:FAILED:%v", ERROR_EOF.AsError()) + if rowPosition >= dbf.header.RowsCount { + return nil, fmt.Errorf("dbase-io-readcolumn-1:FAILED:%v", EOF) } - if columnPosition < 0 || columnPosition > int(dbf.ColumnsCount()) { - return nil, fmt.Errorf("dbase-io-readcolumn-2:FAILED:%v", ERROR_INVALID.AsError()) + return nil, fmt.Errorf("dbase-io-readcolumn-2:FAILED:%v", InvalidPosition) } - buf := make([]byte, dbf.table.columns[columnPosition].Length) - pos := int64(dbf.dbaseHeader.FirstRow) + (int64(rowPosition) * int64(dbf.dbaseHeader.RowLength)) + int64(dbf.table.columns[columnPosition].Position) - - _, err := syscall.Seek(syscall.Handle(*dbf.dbaseFileHandle), pos, 0) + pos := int64(dbf.header.FirstRow) + (int64(rowPosition) * int64(dbf.header.RowLength)) + int64(dbf.table.columns[columnPosition].Position) + _, err := syscall.Seek(*dbf.dbaseFileHandle, pos, 0) if err != nil { - return buf, fmt.Errorf("dbase-io-readcolumn-3:FAILED:%v", err) + return buf, fmt.Errorf("dbase-io-readcolumn-3:FAILED:%w", err) } - - read, err := syscall.Read(syscall.Handle(*dbf.dbaseFileHandle), buf) + read, err := syscall.Read(*dbf.dbaseFileHandle, buf) if err != nil { - return buf, fmt.Errorf("dbase-io-readcolumn-4:FAILED:%v", err) + return buf, fmt.Errorf("dbase-io-readcolumn-4:FAILED:%w", err) } - if read != int(dbf.table.columns[columnPosition].Length) { - return buf, fmt.Errorf("dbase-io-readcolumn-5:FAILED:%v", ERROR_INCOMPLETE.AsError()) + return buf, fmt.Errorf("dbase-io-readcolumn-5:FAILED:%v", Incomplete) } return buf, nil } @@ -180,38 +162,33 @@ func (dbf *DBF) readColumn(rowPosition uint32, columnPosition int) ([]byte, erro // Reads column infos from DBF header, starting at pos 32, until it finds the Header row terminator END_OF_COLUMN(0x0D). func readColumnInfos(fd syscall.Handle) ([]*Column, error) { columns := make([]*Column, 0) - offset := int64(32) b := make([]byte, 1) for { // Check if we are at 0x0D by reading one byte ahead - if _, err := syscall.Seek(syscall.Handle(fd), offset, 0); err != nil { - return nil, fmt.Errorf("dbase-io-readcolumninfos-1:FAILED:%v", err) + if _, err := syscall.Seek(fd, offset, 0); err != nil { + return nil, fmt.Errorf("dbase-io-readcolumninfos-1:FAILED:%w", err) } - if _, err := syscall.Read(syscall.Handle(fd), b); err != nil { - return nil, fmt.Errorf("dbase-io-readcolumninfos-2:FAILED:%v", err) + if _, err := syscall.Read(fd, b); err != nil { + return nil, fmt.Errorf("dbase-io-readcolumninfos-2:FAILED:%w", err) } - if b[0] == END_OF_COLUMN { + if b[0] == ColumnEnd { break } - // Position back one byte and read the column - if _, err := syscall.Seek(syscall.Handle(fd), -1, 1); err != nil { - return nil, fmt.Errorf("dbase-io-readcolumninfos-3:FAILED:%v", err) + if _, err := syscall.Seek(fd, -1, 1); err != nil { + return nil, fmt.Errorf("dbase-io-readcolumninfos-3:FAILED:%w", err) } - buf := make([]byte, 2048) - n, err := syscall.Read(syscall.Handle(fd), buf) + n, err := syscall.Read(fd, buf) if err != nil { - return nil, fmt.Errorf("dbase-io-readcolumninfos-4:FAILED:%v", err) + return nil, fmt.Errorf("dbase-io-readcolumninfos-4:FAILED:%w", err) } - column := &Column{} err = binary.Read(bytes.NewReader(buf[:n]), binary.LittleEndian, column) if err != nil { - return nil, fmt.Errorf("dbase-io-readcolumninfos-5:FAILED:%v", err) + return nil, fmt.Errorf("dbase-io-readcolumninfos-5:FAILED:%w", err) } - if column.Name() == "_NullFlags" { offset += 32 continue @@ -228,41 +205,37 @@ func readColumnInfos(fd syscall.Handle) ([]*Column, error) { // the return value is the raw data and true if the data read is text (false is RAW binary data). func (dbf *DBF) readMemo(blockdata []byte) ([]byte, bool, error) { if dbf.memoFileHandle == nil { - return nil, false, fmt.Errorf("dbase-io-readmemo-1:FAILED:%v", ERROR_NO_FPT_FILE.AsError()) + return nil, false, fmt.Errorf("dbase-io-readmemo-1:FAILED:%v", NoFPT) } - // Determine the block number block := binary.LittleEndian.Uint32(blockdata) // The position in the file is blocknumber*blocksize - _, err := syscall.Seek(syscall.Handle(*dbf.memoFileHandle), int64(dbf.memoHeader.BlockSize)*int64(block), 0) + _, err := syscall.Seek(*dbf.memoFileHandle, int64(dbf.memoHeader.BlockSize)*int64(block), 0) if err != nil { - return nil, false, fmt.Errorf("dbase-io-readmemo-2:FAILED:%v", err) + return nil, false, fmt.Errorf("dbase-io-readmemo-2:FAILED:%w", err) } - // Read the memo block header, instead of reading into a struct using binary.Read we just read the two // uints in one buffer and then convert, this saves seconds for large DBF files with many memo columns // as it avoids using the reflection in binary.Read hbuf := make([]byte, 8) - _, err = syscall.Read(syscall.Handle(*dbf.memoFileHandle), hbuf) + _, err = syscall.Read(*dbf.memoFileHandle, hbuf) if err != nil { - return nil, false, fmt.Errorf("dbase-io-readmemo-3:FAILED:%v", err) + return nil, false, fmt.Errorf("dbase-io-readmemo-3:FAILED:%w", err) } - sign := binary.BigEndian.Uint32(hbuf[:4]) leng := binary.BigEndian.Uint32(hbuf[4:]) if leng == 0 { // No data according to block header? Not sure if this should be an error instead return []byte{}, sign == 1, nil } - // Now read the actual data buf := make([]byte, leng) - read, err := syscall.Read(syscall.Handle(*dbf.memoFileHandle), buf) + read, err := syscall.Read(*dbf.memoFileHandle, buf) if err != nil { - return buf, false, fmt.Errorf("dbase-io-readmemo-4:FAILED:%v", err) + return buf, false, fmt.Errorf("dbase-io-readmemo-4:FAILED:%w", err) } if read != int(leng) { - return buf, sign == 1, fmt.Errorf("dbase-io-readmemo-5:FAILED:%v", ERROR_INCOMPLETE.AsError()) + return buf, sign == 1, fmt.Errorf("dbase-io-readmemo-5:FAILED:%v", Incomplete) } return buf, sign == 1, nil } @@ -271,7 +244,7 @@ func validateFileVersion(version byte) error { switch version { default: return fmt.Errorf("dbase-io-validatefileversion-1:FAILED:untested DBF file version: %d (%x hex)", version, version) - case FOXPRO, FOXPRO_AUTOINCREMENT: + case FoxPro, FoxProAutoincrement: return nil } } @@ -279,9 +252,9 @@ func validateFileVersion(version byte) error { // GoTo sets the internal row pointer to row rowNumber // Returns and EOF error if at EOF and positions the pointer at lastRow+1 func (dbf *DBF) GoTo(rowNumber uint32) error { - if rowNumber > dbf.dbaseHeader.RowsCount { - dbf.table.rowPointer = dbf.dbaseHeader.RowsCount - return fmt.Errorf("dbase-io-goto-1:FAILED:go to %v > %v:%v", rowNumber, dbf.dbaseHeader.RowsCount, ERROR_EOF.AsError()) + if rowNumber > dbf.header.RowsCount { + dbf.table.rowPointer = dbf.header.RowsCount + return fmt.Errorf("dbase-io-goto-1:FAILED:go to %v > %v:%v", rowNumber, dbf.header.RowsCount, EOF) } dbf.table.rowPointer = rowNumber return nil @@ -293,8 +266,8 @@ func (dbf *DBF) GoTo(rowNumber uint32) error { // Does not skip deleted rows func (dbf *DBF) Skip(offset int64) { newval := int64(dbf.table.rowPointer) + offset - if newval >= int64(dbf.dbaseHeader.RowsCount) { - dbf.table.rowPointer = dbf.dbaseHeader.RowsCount + if newval >= int64(dbf.header.RowsCount) { + dbf.table.rowPointer = dbf.header.RowsCount } if newval < 0 { dbf.table.rowPointer = 0 From 05d20e0d3ed95e4d90d6332ef1562db4cf6788b4 Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 12:54:09 +0200 Subject: [PATCH 10/14] [FIX] Error handling --- dbase/memo.go | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/dbase/memo.go b/dbase/memo.go index 3228226..3b1b28d 100644 --- a/dbase/memo.go +++ b/dbase/memo.go @@ -25,43 +25,42 @@ type MemoHeader struct { func (dbf *DBF) parseMemo(raw []byte) ([]byte, bool, error) { memo, isText, err := dbf.readMemo(raw) if err != nil { - return []byte{}, false, fmt.Errorf("dbase-table-parse-memo-1:FAILED:%v", err) + return []byte{}, false, fmt.Errorf("dbase-table-parse-memo-1:FAILED:%w", err) } if isText { memo, err = dbf.convert.Decode(memo) if err != nil { - return []byte{}, false, fmt.Errorf("dbase-table-parse-memo-2:FAILED:%v", err) + return []byte{}, false, fmt.Errorf("dbase-table-parse-memo-2:FAILED:%w", err) } } return memo, isText, nil } +// prepareMemo prepares the memo file for reading. func (dbf *DBF) prepareMemo(fd syscall.Handle) error { memoHeader, err := readMemoHeader(fd) if err != nil { - return fmt.Errorf("dbase-table-prepare-memo-1:FAILED:%v", err) + return fmt.Errorf("dbase-table-prepare-memo-1:FAILED:%w", err) } - dbf.memoFileHandle = &fd dbf.memoHeader = memoHeader return nil } +// readMemoHeader reads the memo header from the given file handle. func readMemoHeader(fd syscall.Handle) (*MemoHeader, error) { h := &MemoHeader{} - if _, err := syscall.Seek(syscall.Handle(fd), 0, 0); err != nil { - return nil, fmt.Errorf("dbase-table-read-memo-header-1:FAILED:%v", err) + if _, err := syscall.Seek(fd, 0, 0); err != nil { + return nil, fmt.Errorf("dbase-table-read-memo-header-1:FAILED:%w", err) } - b := make([]byte, 1024) - n, err := syscall.Read(syscall.Handle(fd), b) + n, err := syscall.Read(fd, b) if err != nil { - return nil, fmt.Errorf("dbase-table-read-memo-header-2:FAILED:%v", err) + return nil, fmt.Errorf("dbase-table-read-memo-header-2:FAILED:%w", err) } - err = binary.Read(bytes.NewReader(b[:n]), binary.BigEndian, h) if err != nil { - return nil, fmt.Errorf("dbase-table-read-memo-header-3:FAILED:%v", err) + return nil, fmt.Errorf("dbase-table-read-memo-header-3:FAILED:%w", err) } return h, nil } From 223d0e9ea2fe45589d07a75eea0621be2d563406 Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 12:54:19 +0200 Subject: [PATCH 11/14] [FIX] Variable spelling in example --- example/example.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example/example.go b/example/example.go index e0c3070..794b6a7 100644 --- a/example/example.go +++ b/example/example.go @@ -13,7 +13,7 @@ type Test struct { Date time.Time `json:"DATUM"` TIJD string `json:"TIJD"` SOORT float64 `json:"SOORT"` - ID_NR int32 `json:"ID_NR"` + IDNR int32 `json:"ID_NR"` UserNR int32 `json:"USERNR"` CompanyName string `json:"COMP_NAME"` CompanyOS string `json:"COMP_OS"` From 2110005e4d1ea2c83f4d8655adf6620ce903e5b1 Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 12:55:20 +0200 Subject: [PATCH 12/14] [FIX] Added Microsoft docs link and fixed example in readme --- README.md | 53 +++++++++++++++++++++++++++++------------------------ 1 file changed, 29 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 4d0e7a9..36222db 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,8 @@ The supported column types with their return Go types are: | T | DateTime | time.Time | | Y | Currency | float64 | +If you need more information about dbase data types take a look here: [Microsoft Visual Studio Foxpro](https://learn.microsoft.com/en-us/previous-versions/visualstudio/foxpro/74zkxe2k(v=vs.80)) + # Installation ``` go get github.com/Valentin-Kaiser/go-dbase/dbase @@ -70,7 +72,7 @@ type Test struct { Date time.Time `json:"DATUM"` TIJD string `json:"TIJD"` SOORT float64 `json:"SOORT"` - ID_NR int32 `json:"ID_NR"` + IDNR int32 `json:"ID_NR"` UserNR int32 `json:"USERNR"` CompanyName string `json:"COMP_NAME"` CompanyOS string `json:"COMP_OS"` @@ -81,99 +83,102 @@ type Test struct { } func main() { - // Open file + // Open the example database file. dbf, err := dbase.Open("./test_data/TEST.DBF", new(dbase.Win1250Converter)) if err != nil { panic(err) } defer dbf.Close() - // Print all database column infos + // Print all database column infos. for _, column := range dbf.Columns() { - fmt.Println(column.Name(), column.Type(), column.Decimals) + fmt.Printf("Name: %v - Type: %v \n", column.Name(), column.Type()) } - // Read the complete first row + // Read the complete first row. row, err := dbf.Row() if err != nil { panic(err) } - // Print all the columns in their Go values as slice - fmt.Println(row.Values()) + // Print all the columns in their Go values as slice. + fmt.Printf("%+v", row.Values()) - // Go back to start - err = dbf.Skip(0) - if err != nil { - panic(err) - } + // Go back to start. + dbf.Skip(0) - // Loop through all rows using rowPointer in DBF struct - // Reads the complete row + // Loop through all rows using rowPointer in DBF struct. for !dbf.EOF() { - // This reads the complete row + fmt.Printf("EOF: %v - Pointer: %v \n", dbf.EOF(), dbf.Pointer()) + + // This reads the complete row. row, err := dbf.Row() if err != nil { panic(err) } + // Increase the pointer. dbf.Skip(1) - // skip deleted rows + + // Skip deleted rows. if row.Deleted { continue } - // get value by position + // Get value by column position _, err = row.Value(0) if err != nil { panic(err) } - // get value by name + // Get value by column name _, err = row.Value(dbf.ColumnPos("COMP_NAME")) if err != nil { panic(err) } - // Set space trimming per default + // Enable space trimming per default dbf.SetTrimspacesDefault(true) // Disable space trimming for the company name dbf.SetColumnModification(dbf.ColumnPos("COMP_NAME"), false, "", nil) - // add a column modification to switch the names of "NUMBER" and "Float" to match the data types + // Add a column modification to switch the names of "NUMBER" and "Float" to match the data types dbf.SetColumnModification(dbf.ColumnPos("NUMBER"), true, "FLOAT", nil) dbf.SetColumnModification(dbf.ColumnPos("FLOAT"), true, "NUMBER", nil) + // Read the row into a struct. t := &Test{} err = row.ToStruct(t) if err != nil { panic(err) } - fmt.Printf("%+v \n", t) + + fmt.Printf("Company: %v", t.CompanyName) } // Read only the third column of rows 1, 2 and 3 - rownumbers := []uint32{1, 2, 3} - for _, row := range rownumbers { + for _, row := range []uint32{1, 2, 3} { err := dbf.GoTo(row) if err != nil { panic(err) } + // Check if the row is deleted deleted, err := dbf.Deleted() if err != nil { panic(err) } - if deleted { fmt.Printf("Row %v deleted \n", row) continue } + // Read the entire row r, err := dbf.Row() if err != nil { panic(err) } + // Read the seventh column column, err := r.Value(7) if err != nil { panic(err) From 8cb4cfd77b8592a3e1953d43c1fe9398e99253ce Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 13:01:30 +0200 Subject: [PATCH 13/14] [FIX] Disabled gofmt linter --- .golangci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.golangci.yml b/.golangci.yml index 094e872..4047794 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -37,6 +37,7 @@ linters: - nosnakecase - ifshort - gci + - gofmt # disabled because of generics - rowserrcheck From cec71c52e94056ee3a2f800761394f6c7d9fb256 Mon Sep 17 00:00:00 2001 From: Valentin Kaiser Date: Tue, 27 Sep 2022 13:14:23 +0200 Subject: [PATCH 14/14] [FIX] Disabled annoying linter --- .golangci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.golangci.yml b/.golangci.yml index 4047794..186af7c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -38,6 +38,7 @@ linters: - ifshort - gci - gofmt + - goimports # disabled because of generics - rowserrcheck