Handling comments in nginx configs

AUTHORS.rst

@@ -11,3 +11,4 @@ Contributors
 ------------
 
 * Grant Hulegaard <loki.labrys@gmail.com> `@gshulegaard <https://github.com/gshulegaard> <https://gitlab.com/gshulegaard>`_
+* Ivan Poluyanov <i.poluyanov@icloud.com> `@poluyanov <https://github.com/poluyanov>`_

README.rst

@@ -81,6 +81,7 @@ the schema defined below, and dumps the entire thing as a JSON payload.
      --no-catch            only collect first error in file
      --tb-onerror          include tracebacks in config errors
      --single-file         do not include other config files
+     --include-comments    include comments in json
 
 **Privacy and Security**
 

crossplane/__main__.py

@@ -28,14 +28,14 @@ def _dump_payload(obj, fp, indent):
     fp.write(json.dumps(obj, **kwargs) + '\n')
 
 
-def parse(filename, out, indent=None, catch=None, tb_onerror=None, ignore='', single=False):
+def parse(filename, out, indent=None, catch=None, tb_onerror=None, ignore='', single=False, comments=False):
     ignore = ignore.split(',') if ignore else []
 
     def callback(e):
         exc = sys.exc_info() + (10,)
         return ''.join(format_exception(*exc)).rstrip()
 
-    kwargs = {'catch_errors': catch, 'ignore': ignore, 'single': single}
+    kwargs = {'catch_errors': catch, 'ignore': ignore, 'single': single, 'comments': comments}
     if tb_onerror:
         kwargs['onerror'] = callback
 
@@ -170,6 +170,7 @@ def create_subparser(function, help):
     p.add_argument('--no-catch', action='store_false', dest='catch', help='only collect first error in file')
     p.add_argument('--tb-onerror', action='store_true', help='include tracebacks in config errors')
     p.add_argument('--single-file', action='store_true', dest='single', help='do not include other config files')
+    p.add_argument('--include-comments', action='store_true', dest='comments', help='include comments in json')
 
     p = create_subparser(build, 'builds an nginx config from a json payload')
     p.add_argument('filename', help='the file with the config payload')

crossplane/builder.py

@@ -62,12 +62,34 @@ def _enquote(arg):
 
 def build(payload, indent=4, tabs=False):
     padding = '\t' if tabs else ' ' * indent
+    state = {
+        'prev_obj': None,
+        'depth': -1
+    }
 
-    def _build_lines(objs, depth):
-        margin = padding * depth
+    def _put_line(line, obj):
+        margin = padding * state['depth']
+
+        # don't need put \n on first line and after comment
+        if state['prev_obj'] is None:
+            return margin + line
+
+        # trailing comments have to be without \n
+        if obj['directive'] == '#' and obj['line'] == state['prev_obj']['line']:
+            return ' ' + line
+
+        return '\n' + margin + line
+
+    def _build_lines(objs):
+        state['depth'] = state['depth'] + 1
 
         for obj in objs:
             directive = obj['directive']
+
+            if directive == '#':
+                yield _put_line('#' + obj['comment'], obj)
+                continue
+
             args = [_enquote(arg) for arg in obj['args']]
 
             if directive == 'if':
@@ -78,12 +100,18 @@ def _build_lines(objs, depth):
                 line = directive
 
             if obj.get('block') is None:
-                yield margin + line + ';'
+                yield _put_line(line + ';', obj)
             else:
-                yield margin + line + ' {'
-                for line in _build_lines(obj['block'], depth+1):
+                yield _put_line(line + ' {', obj)
+
+                # set prev_obj to propper indentation in block
+                state['prev_obj'] = obj
+                for line in _build_lines(obj['block']):
                     yield line
-                yield margin + '}'
+                yield _put_line('}', obj)
+
+            state['prev_obj'] = obj
+        state['depth'] = state['depth'] - 1
 
-    lines = _build_lines(payload, depth=0)
-    return '\n'.join(lines)
+    lines = _build_lines(payload)
+    return ''.join(lines)

crossplane/lexer.py

@@ -43,10 +43,13 @@ def _lex_file_object(file_obj):
             while char.isspace():
                 char, line = next(it)
 
-        # if starting comment then disregard until EOL
+        # if starting comment
         if not token and char == '#':
-            while not char.endswith('\n'):  # don't escape newlines
-                char, line = next(it)
+            while not char.endswith('\n'):
+                token = token + char
+                char, _ = next(it)
+            yield (token, line)
+            token = ''
             continue
 
         if not token:

crossplane/parser.py

@@ -19,7 +19,7 @@ def _prepare_if_args(stmt):
         args[:] = args[start:end]
 
 
-def parse(filename, onerror=None, catch_errors=True, ignore=(), single=False):
+def parse(filename, onerror=None, catch_errors=True, ignore=(), single=False, comments=False):
     """
     Parses an nginx config file and returns a nested dict payload
     
@@ -28,6 +28,7 @@ def parse(filename, onerror=None, catch_errors=True, ignore=(), single=False):
     :param catch_errors: bool; if False, parse stops after first error
     :param ignore: list or tuple of directives to exclude from the payload
     :param single: bool; if True, including from other files doesn't happen
+    :param comments: bool; if True, including comments to json payload
     :returns: a payload that describes the parsed nginx config
     """
     config_dir = os.path.dirname(filename)
@@ -77,6 +78,19 @@ def _parse(parsing, tokens, ctx=(), consume=False):
                     _parse(parsing, tokens, consume=True)
                 continue
 
+            # if token is comment
+            if token.startswith('#'):
+                if comments:
+                    stmt = {
+                        "directive": "#",
+                        "args": [],
+                        "line": lineno,
+                        "comment": token[1:]
+                    }
+
+                    parsed.append(stmt)
+                continue
+
             # the first token should always(?) be an nginx directive
             stmt = {
                 'directive': token,

tests/configs/with-comments/nginx.conf

@@ -0,0 +1,14 @@
+events {
+    worker_connections 1024;
+}
+#comment
+http {
+    server {
+        listen       127.0.0.1:8080; #listen
+        server_name  default_server;
+        location / { ## this is brace
+            # location /
+            return 200 "foo bar baz";
+        }
+    }
+}

tests/test_build.py

@@ -25,24 +25,24 @@ def assert_equal_payloads(a, b, ignore_keys=()):
         assert a == b
 
 
-def compare_parsed_and_built(conf_dirname, conf_basename, tmpdir):
+def compare_parsed_and_built(conf_dirname, conf_basename, tmpdir, **kwargs):
     original_dirname = os.path.join(here, 'configs', conf_dirname)
     original_path = os.path.join(original_dirname, conf_basename)
-    original_payload = crossplane.parse(original_path)
+    original_payload = crossplane.parse(original_path, **kwargs)
     original_parsed = original_payload['config'][0]['parsed']
 
     build1_config = crossplane.build(original_parsed)
     build1_file = tmpdir.join('build1.conf')
     build1_file.write(build1_config)
-    build1_payload = crossplane.parse(build1_file.strpath)
+    build1_payload = crossplane.parse(build1_file.strpath, **kwargs)
     build1_parsed = build1_payload['config'][0]['parsed']
 
     assert_equal_payloads(original_parsed, build1_parsed, ignore_keys=['line'])
 
     build2_config = crossplane.build(build1_parsed)
     build2_file = tmpdir.join('build2.conf')
     build2_file.write(build2_config)
-    build2_payload = crossplane.parse(build2_file.strpath)
+    build2_payload = crossplane.parse(build2_file.strpath, **kwargs)
     build2_parsed = build2_payload['config'][0]['parsed']
 
     assert build1_config == build2_config
@@ -110,10 +110,126 @@ def test_build_nested_and_multiple_args():
         '}'
     ])
 
+def test_build_with_comments():
+    payload = [
+        {
+           "args" : [],
+           "block" : [
+              {
+                 "args" : [
+                    "1024"
+                 ],
+                 "line" : 2,
+                 "directive" : "worker_connections"
+              }
+           ],
+           "line" : 1,
+           "directive" : "events"
+        },
+        {
+           "directive" : "#",
+           "line" : 4,
+           "comment" : "comment",
+           "args" : []
+        },
+        {
+           "directive" : "http",
+           "block" : [
+              {
+                 "args" : [],
+                 "line" : 6,
+                 "block" : [
+                    {
+                       "args" : [
+                          "127.0.0.1:8080"
+                       ],
+                       "line" : 7,
+                       "directive" : "listen"
+                    },
+                    {
+                       "directive" : "#",
+                       "line" : 7,
+                       "comment" : "listen",
+                       "args" : []
+                    },
+                    {
+                       "args" : [
+                          "default_server"
+                       ],
+                       "directive" : "server_name",
+                       "line" : 8
+                    },
+                    {
+                       "args" : [
+                          "/"
+                       ],
+                       "directive" : "location",
+                       "line" : 9,
+                       "block" : [
+                          {
+                             "args" : [],
+                             "directive" : "#",
+                             "line" : 9,
+                             "comment" : "# this is brace"
+                          },
+                          {
+                             "directive" : "#",
+                             "comment" : " location /",
+                             "line" : 10,
+                             "args" : []
+                          },
+                          {
+                             "directive" : "#",
+                             "comment" : " is here",
+                             "line" : 11,
+                             "args" : []
+                          },
+                          {
+                             "args" : [
+                                "200",
+                                "foo bar baz"
+                             ],
+                             "line" : 11,
+                             "directive" : "return"
+                          }
+                       ]
+                    }
+                 ],
+                 "directive" : "server"
+              }
+           ],
+           "line" : 5,
+           "args" : []
+        }
+     ]
+
+    built = crossplane.build(payload, indent=4, tabs=False)
+
+    assert built == '\n'.join([
+        'events {',
+        '    worker_connections 1024;',
+        '}',
+        '#comment',
+        'http {',
+        '    server {',
+        '        listen 127.0.0.1:8080; #listen',
+        '        server_name default_server;',
+        '        location / { ## this is brace',
+        '            # location /',
+        '            # is here',
+        "            return 200 'foo bar baz';",
+        '        }',
+        '    }',
+        '}'
+    ])
 
 def test_compare_parsed_and_built_simple(tmpdir):
     compare_parsed_and_built('simple', 'nginx.conf', tmpdir)
 
 
 def test_compare_parsed_and_built_messy(tmpdir):
     compare_parsed_and_built('messy', 'nginx.conf', tmpdir)
+
+
+def test_compare_parsed_and_built_messy_with_comments(tmpdir):
+    compare_parsed_and_built('with-comments', 'nginx.conf', tmpdir, comments=True)