fix(swift): lex tag separators properly

token/swift_lexer.go

@@ -1,5 +1,9 @@
 package token
 
+import (
+ "strings"
+)
+
 // NewSwiftLexer returns a SwiftLexer ready for parsing the given input string
 func NewSwiftLexer(name, input string) *SwiftLexer {
  lexer := NewStringLexer(name, input)
@@ -134,8 +138,27 @@ func lexSwiftAlphaNumeric(l *StringLexer) StringLexerStateFn {
 }
 
 func isTagBoundary(s *StringLexer) bool {
+ oneOf := func(fn ...func() bool) bool {
+ for _, f := range fn {
+ if ok := f(); ok {
+ return true
+ }
+ }
+ return false
+ }
  currentPos := s.pos
- isTagBoundary := s.accept(string(carriageReturn)) && s.accept(string(lineFeed)) && s.accept(string(dash)+string(tagIdentifier))
+ isTagBoundary := s.accept(string(carriageReturn)) &&
+ s.accept(string(lineFeed)) && oneOf(
+ func() bool {
+ return strings.HasPrefix(s.input[s.pos:], string(dash)+string(tagIdentifier))
+ },
+ func() bool {
+ return strings.HasPrefix(s.input[s.pos:], string(tagIdentifier))
+ },
+ func() bool {
+ return s.input[s.pos:] == string(dash)
+ },
+ )
  s.pos = currentPos
  return isTagBoundary
 }
@@ -173,10 +196,14 @@ const (
 )
 
 var swiftTokenName = map[Type]string{
- SWIFT_ALPHA: "a",
- SWIFT_CHARACTER: "c",
- SWIFT_DECIMAL: "d",
- SWIFT_NUMERIC: "n",
+ SWIFT_ALPHA: "a",
+ SWIFT_CHARACTER: "c",
+ SWIFT_DECIMAL: "d",
+ SWIFT_NUMERIC: "n",
+ SWIFT_ALPHANUMERIC: "an",
+ SWIFT_DATASET_START: "datasetStart",
+ SWIFT_TAG_SEPARATOR: "tagSeparator",
+ SWIFT_MESSAGE_SEPARATOR: "messageSeparator",
 }
 
 func init() {

token/swift_lexer_test.go

@@ -0,0 +1,50 @@
+package token
+
+import (
+ "testing"
+)
+
+const githubIssue13TestData = "\r\n" +
+ ":20:MT940-1803060458\r\n" +
+ ":21:NONREF\r\n" +
+ ":25:20040000/12345678EUR\r\n" +
+ ":28C:0/13\r\n:60M:C170201EUR1234,56\r\n" +
+ ":61:1702010201DR86,40NMSCNONREF//POS 8888888888\r\n" +
+ ":86:005?20LASTSCHRIFT/BELAST.?888888888888 8888888884REFERE?22NZ HVV \r\n" +
+ "A?23BO?24END-TO-END-REF.:?888888888888 8888888884?26CORE / MANDAT\r\n" +
+ "SREF.:?27VMH008888880001?28GL\xc4UBIGER-ID:?29DE88888888888888888?32H\r\n" +
+ "AMBURGER HOCHBAHN AG?60Ref. IL888888G8888888/6716\r\n" +
+ ":61:1702010201DR24,00NMSCNONREF//POS 3409790600\r\n" +
+ ":86:005?20LASTSCHRIFT/BELAST.?21110865 BEITRAG MITGLIED 888?228888?23\r\n" +
+ "END-TO-END-REF.:?8888888888888ZV888888Z?25CORE / MANDATSREF.:?26EV\r\n" +
+ "-000008888?27GL\xc4UBIGER-ID:?88DE88ZZZ8888888888?29Ref. IL888888G2\r\n" +
+ "145077/1543?32SOME TEST-VEREIN E.V.\r\n" +
+ ":62M:C170203EUR5378,36\r\n-"
+
+func Test_SwiftLexer(t *testing.T) {
+ lexer := NewSwiftLexer("testlexer", githubIssue13TestData)
+
+ var tokens []Token
+
+ for lexer.HasNext() {
+ tokens = append(tokens, lexer.Next())
+ }
+
+ expectedLen := 22
+ if len(tokens) != expectedLen {
+ t.Logf("Expected %d tokens, got %d", expectedLen, len(tokens))
+ t.Fail()
+ }
+
+ var messageSeparatorCount int
+ for _, tk := range tokens {
+ if tk.Type() == SWIFT_MESSAGE_SEPARATOR {
+ messageSeparatorCount++
+ }
+ }
+
+ if messageSeparatorCount != 1 {
+ t.Logf("Expected one message separator, got %d", messageSeparatorCount)
+ t.Fail()
+ }
+}

token/token.go

@@ -156,9 +156,9 @@ func (e elementToken) String() string {
  return e.val
  }
  if len(e.val) > 10 {
- return fmt.Sprintf("%.10q...", e.val)
+ return fmt.Sprintf("%s(%.10q...)", e.typ.String(), e.val)
  }
- return fmt.Sprintf("%q", e.val)
+ return fmt.Sprintf("%s(%q)", e.typ.String(), e.val)
 }
 
 const (