Merge pull request #24 from vaidik/parser

Parser

.travis.yml

@@ -2,13 +2,10 @@ language: python
 
 matrix:
   include:
-    - name: "Python 2.7 on Xenial Linux"
-      python: 2.6
-      dist: trusty
-    - name: "Python 2.7 on Xenial Linux"
+    - name: "Python 2.7 on Precise"
       python: 2.7
       dist: precise
-    - name: "Python 3.3 on Xenial Linux"
+    - name: "Python 3.3 on Trusty"
       python: 3.3
       dist: trusty
     - name: "Python 3.4 on Xenial Linux"

commentjson/__init__.py

@@ -1,5 +1,6 @@
 from .commentjson import dump
 from .commentjson import dumps
 from .commentjson import JSONLibraryException
+from .commentjson import ParserException
 from .commentjson import load
 from .commentjson import loads

commentjson/commentjson.py

@@ -24,48 +24,105 @@
     import simplejson as json
 
 
-class JSONLibraryException(Exception):
-    ''' Exception raised when the JSON library in use raises an exception i.e.
-    the exception is not caused by `commentjson` and only caused by the JSON
-    library `commentjson` is using.
+import lark
 
-    .. note::
+from lark import Lark
+from lark.reconstruct import Reconstructor
 
-        As of now, ``commentjson`` supports only standard library's ``json``
-        module. It might start supporting other widely-used contributed JSON
-        libraries in the future.
+
+parser = Lark('''
+    ?start: value
+    ?value: object
+          | array
+          | string
+          | SIGNED_NUMBER      -> number
+          | "true"             -> true
+          | "false"            -> false
+          | "null"             -> null
+    array  : "[" [value ("," value)*] "]"
+    object : "{" [pair ("," pair)*] "}"
+    pair   : string ":" value
+    string : ESCAPED_STRING
+
+    COMMENT: /(#|\/\/)[^\\n]*/
+
+    %import common.ESCAPED_STRING
+    %import common.SIGNED_NUMBER
+    %import common.WS
+    %ignore WS
+    %ignore COMMENT
+''', start='start')
+
+serializer = Reconstructor(parser)
+
+
+class BaseException(Exception):
+    ''' Base exception to be implemented and raised while handling exceptions
+    raised by libraries used in `commentjson`.
+
+    Sets message of self in a way that it clearly calls out that the exception
+    was raised by another library, along with the entire stacktrace of the
+    exception raised by the other library.
     '''
 
     def __init__(self, exc):
+        if self.library is None:
+            raise NotImplementedError(
+                'Value of library must be set in the '
+                'inherited exception class.')
+
         tb = traceback.format_exc()
         tb = '\n'.join(' ' * 4 + line_ for line_ in tb.split('\n'))
 
-        json_error = None
+        error = None
         try:
-            json_error = exc.msg
+            error = exc.msg
         except AttributeError:
             try:
-                json_error = exc.message
+                error = exc.message
             except AttributeError:
-                json_error = str(exc)
+                error = str(exc)
 
         self.message = '\n'.join([
             'JSON Library Exception\n',
-            ('Exception thrown by JSON library (json): '
-             '\033[4;37m%s\033[0m\n' % json_error),
+            ('Exception thrown by library (%s): '
+             '\033[4;37m%s\033[0m\n' % (self.library, error)),
             '%s' % tb,
         ])
         Exception.__init__(self, self.message)
 
 
+class ParserException(BaseException):
+    '''Exception raised when the `lark` raises an exception i.e.
+    the exception is not caused by `commentjson` and caused by the use of
+    `lark` in `commentjson`.
+    '''
+
+    library = 'lark'
+
+
+class JSONLibraryException(BaseException):
+    '''Exception raised when the `json` raises an exception i.e.
+    the exception is not caused by `commentjson` and caused by the use of
+    `json` in `commentjson`.
+
+    .. note::
+
+        As of now, ``commentjson`` supports only standard library's ``json``
+        module. It might start supporting other widely-used contributed JSON
+        libraries in the future.
+    '''
+
+    library = 'json'
+
+
 def loads(text, **kwargs):
     ''' Deserialize `text` (a `str` or `unicode` instance containing a JSON
     document with Python or JavaScript like comments) to a Python object.
 
     :param text: serialized JSON string with or without comments.
     :param kwargs: all the arguments that `json.loads <http://docs.python.org/
                    2/library/json.html#json.loads>`_ accepts.
-    :raises: commentjson.JSONLibraryException
     :returns: dict or list.
     '''
 
@@ -84,7 +141,13 @@ def loads(text, **kwargs):
                 lines[index] = re.sub(regex_inline, r'\1', line)
 
     try:
-        return json.loads('\n'.join(lines), **kwargs)
+        parsed = parser.parse(text)
+        final_text = serializer.reconstruct(parsed)
+    except lark.exceptions.UnexpectedCharacters:
+        raise ParserException('Unable to parse text')
+
+    try:
+        return json.loads(final_text, **kwargs)
     except Exception as e:
         raise JSONLibraryException(e)
 

commentjson/tests/array_with_hash-commented.json

@@ -0,0 +1,12 @@
+{
+    "a": ["1", "2"], # comment
+    "b": ["1 #asd", "2"], # comment
+    "c": [], # comment,
+    "d": [ # comment
+        1, # comment
+        "123", # comment
+        "123#4", # comment
+        "#4123",
+        [] # comment
+    ]
+}

commentjson/tests/array_with_hash-uncommented.json

@@ -0,0 +1,12 @@
+{
+    "a": ["1", "2"],
+    "b": ["1 #asd", "2"],
+    "c": [],
+    "d": [
+        1,
+        "123",
+        "123#4",
+        "#4123",
+        []
+    ]
+}

commentjson/tests/inline-commented.json

@@ -1,8 +1,8 @@
 {
     "key1": 1, # testing this
     "key1.2": 1, // testing this
-    "key2": "1", # testing this
-    "key2.2": "1", // testing this
+    "key2": "1", # testing this ending with quote "
+    "key2.2": "1", // testing this ending with quote "
     "key3": 3,
     "key4": 4.3, # testing float 
     "key4.2": 4.3, // testing float 

commentjson/tests/inline_last_quote-commented.json

@@ -0,0 +1,3 @@
+{
+    "a": "b" # this is a quote: "
+}

commentjson/tests/inline_last_quote-uncommented.json

@@ -0,0 +1,3 @@
+{
+    "a": "b"
+}

commentjson/tests/test_commentjson.py

@@ -18,7 +18,9 @@ def setUp(self):
         self.files = ('sample', 'line_comment', 'inline_last_float',
                       'inline_last_int', 'nested_object', 'string_with_hash',
                       'string_with_inline_comment',
-                      'inline_has_special_characters')
+                      'inline_has_special_characters',
+                      'array_with_hash',
+                      'inline_last_quote')
 
         for file_ in self.files:
             fpath = os.path.join(self.path, file_)
@@ -40,6 +42,13 @@ def tearDown(self):
         if os.path.exists(test_file_path):
             os.unlink(test_file_path)
 
+    def test_dumping_parsing_simple_string(self):
+        string = '//'
+        self.assertEqual(commentjson.loads(commentjson.dumps(string)), string)
+
+        string = '#'
+        self.assertEqual(commentjson.loads(commentjson.dumps(string)), string)
+
     def test_dumps(self):
         test_dict = dict(a=1, b=2)
         c_dump = commentjson.dumps(test_dict)
@@ -64,14 +73,15 @@ def test_loads(self):
         for index, test_json_ in iteritems(self.test_json):
             commented = test_json_['commented']
             uncommented = test_json_['uncommented']
-            self.assertEqual(commentjson.loads(commented),
-                json.loads(uncommented))
+            self.assertEqual(
+                commentjson.loads(commented),
+                json.loads(uncommented),
+                'Failed for test: %s' % test_json_['commented'])
 
     def test_loads_with_kwargs(self):
         def test_hook(loaded_dict):
             return {}
         commented = self.test_json['sample']['commented']
-        uncommented = self.test_json['sample']['uncommented']
         test_kwargs = dict(object_hook=test_hook)
 
         c_load = commentjson.loads(commented, **test_kwargs)
@@ -80,14 +90,14 @@ def test_hook(loaded_dict):
         assert c_load == {}
 
     def test_loads_throws_exception(self):
-        self.assertRaises(commentjson.JSONLibraryException, commentjson.loads,
+        self.assertRaises(commentjson.ParserException, commentjson.loads,
                           'Unserializable text')
 
     def test_dump(self):
         test_dict = dict(a=1, b=2)
 
         wfp = open(os.path.join(self.path, 'test.json'), 'w')
-        c_dump = commentjson.dump(test_dict, wfp)
+        commentjson.dump(test_dict, wfp)
         wfp.close()
 
         rfp = open(os.path.join(self.path, 'test.json'), 'r')
@@ -101,13 +111,14 @@ def test_dump_with_kwargs(self):
         test_kwargs = dict(indent=4)
 
         wfp = open(os.path.join(self.path, 'test.json'), 'w')
-        c_dump = commentjson.dump(test_dict, wfp, **test_kwargs)
+        commentjson.dump(test_dict, wfp, **test_kwargs)
         wfp.close()
 
         rfp = open(os.path.join(self.path, 'test.json'), 'r')
         j_dump = json.dumps(test_dict, **test_kwargs)
+        c_dump = rfp.read()
 
-        assert rfp.read(), j_dump
+        assert c_dump == j_dump, c_dump
         rfp.close()
 
     def test_dump_throws_exception(self):
@@ -133,7 +144,7 @@ def test_hook(loaded_dict):
         test_kwargs = dict(object_hook=test_hook)
         rfp = open(os.path.join(self.path, 'sample-commented.json'), 'r')
 
-        assert commentjson.load(rfp, **test_kwargs) == {}
+        self.assertEqual(commentjson.load(rfp, **test_kwargs), {})
         rfp.close()
 
     def test_load_throws_exception(self):

setup.py

@@ -6,9 +6,11 @@
 from setuptools import setup, find_packages
 
 
-__version__ = '.'.join(map(str, (0, 7, 2)))
+__version__ = '.'.join(map(str, (0, 8, 0)))
 
-install_requires = []
+install_requires = [
+    'lark-parser>=0.7.1'
+]
 if sys.version_info <= (2, 6):
     install_requires.append('simplejson')