Updates for issue #312:

Adding code to support tokenizing the line:

  1)  Added token attribute to the line object
  2)  Added token.py file
  3)  Adding tests

Currently the tokenizer handles consecutive spaces, comments and characters.

NOTE:  Future commits will handle VHDL symbols.

vsg/line.py

@@ -1,5 +1,6 @@
 
 from vsg import utils
+from vsg import tokens
 
 
 class line():
@@ -26,6 +27,7 @@ def __init__(self, line):
         self.line = line
         self.lineLower = line.lower()
         self.lineNoComment = utils.remove_comment(line)
+        self.tokens = tokens.create(self.line)
 
         self.indentLevel = None
         # Misc attributes

vsg/tests/line/test_token_method.py

@@ -0,0 +1,83 @@
+
+import unittest
+
+from vsg import line
+
+class testTokenMethod(unittest.TestCase):
+
+    def test_token_create_method_with_single_spaces(self):
+        oLine = line.line('contents of line')
+        self.assertTrue(oLine)
+        self.assertEqual('contents of line', oLine.line)
+
+        lTokens = []
+        lTokens.append('contents')
+        lTokens.append(' ')
+        lTokens.append('of')
+        lTokens.append(' ')
+        lTokens.append('line')
+
+        self.assertEqual(lTokens, oLine.tokens)
+
+    def test_token_create_method_with_multiple_spaces(self):
+        oLine = line.line('   contents   of     line')
+        self.assertTrue(oLine)
+        self.assertEqual('   contents   of     line', oLine.line)
+
+        lTokens = []
+        lTokens.append('   ')
+        lTokens.append('contents')
+        lTokens.append('   ')
+        lTokens.append('of')
+        lTokens.append('     ')
+        lTokens.append('line')
+
+        self.assertEqual(lTokens, oLine.tokens)
+
+    def test_token_create_method_with_comment_at_end_of_line_without_spaces_around_dashes(self):
+        oLine = line.line('contents of line--This is a comment')
+        self.assertTrue(oLine)
+        self.assertEqual('contents of line--This is a comment', oLine.line)
+
+        lTokens = []
+        lTokens.append('contents')
+        lTokens.append(' ')
+        lTokens.append('of')
+        lTokens.append(' ')
+        lTokens.append('line')
+        lTokens.append('--This is a comment')
+
+        self.assertEqual(lTokens, oLine.tokens)
+
+    def test_token_create_method_with_comment_at_end_of_line_with_spaces_around_dashes(self):
+        oLine = line.line('contents of line --  This is a comment')
+        self.assertTrue(oLine)
+        self.assertEqual('contents of line --  This is a comment', oLine.line)
+
+        lTokens = []
+        lTokens.append('contents')
+        lTokens.append(' ')
+        lTokens.append('of')
+        lTokens.append(' ')
+        lTokens.append('line')
+        lTokens.append(' ')
+        lTokens.append('--  This is a comment')
+
+        self.assertEqual(lTokens, oLine.tokens)
+
+    def test_token_create_method_with_comment_at_end_of_line_with_spaces_at_the_end_of_line(self):
+        oLine = line.line('contents of line --  This is a comment  ')
+        self.assertTrue(oLine)
+        self.assertEqual('contents of line --  This is a comment  ', oLine.line)
+
+        lTokens = []
+        lTokens.append('contents')
+        lTokens.append(' ')
+        lTokens.append('of')
+        lTokens.append(' ')
+        lTokens.append('line')
+        lTokens.append(' ')
+        lTokens.append('--  This is a comment  ')
+
+        self.assertEqual(lTokens, oLine.tokens)
+

vsg/tokens.py

@@ -0,0 +1,32 @@
+
+def create(sString):
+    '''
+    This function takes a string and returns a list of tokens.
+    '''
+    lReturn = []
+    sToken = ''
+    fCommentFound = False
+    for iIndex, sChar in enumerate(sString):
+        if len(sToken) == 0:
+            sToken = sChar
+        else:
+            if sChar == '-' and sString[iIndex + 1] == '-':
+                fCommentFound = True
+                lReturn.append(sToken)
+                sToken = ''
+            if fCommentFound:
+                sToken += sChar
+                continue
+            if sChar == ' ' and sToken[-1] == ' ':
+                sToken += sChar
+            if not sChar == ' ' and not sToken[-1] == ' ':
+                sToken += sChar
+            if sChar == ' ' and not sToken[-1] == ' ':
+                lReturn.append(sToken)
+                sToken = sChar
+            if not sChar == ' ' and sToken[-1] == ' ':
+                lReturn.append(sToken)
+                sToken = sChar
+
+    lReturn.append(sToken)
+    return lReturn