diff --git a/AUTHORS.rst b/AUTHORS.rst index d1d5326..691d96a 100644 --- a/AUTHORS.rst +++ b/AUTHORS.rst @@ -11,3 +11,4 @@ Contributors ------------ * Grant Hulegaard `@gshulegaard `_ +* Ivan Poluyanov `@poluyanov `_ \ No newline at end of file diff --git a/README.rst b/README.rst index 09a269a..3bc2e97 100644 --- a/README.rst +++ b/README.rst @@ -81,6 +81,7 @@ the schema defined below, and dumps the entire thing as a JSON payload. --no-catch only collect first error in file --tb-onerror include tracebacks in config errors --single-file do not include other config files + --include-comments include comments in json **Privacy and Security** diff --git a/crossplane/__main__.py b/crossplane/__main__.py index 244f5e3..fbeb7ad 100644 --- a/crossplane/__main__.py +++ b/crossplane/__main__.py @@ -28,14 +28,14 @@ def _dump_payload(obj, fp, indent): fp.write(json.dumps(obj, **kwargs) + '\n') -def parse(filename, out, indent=None, catch=None, tb_onerror=None, ignore='', single=False): +def parse(filename, out, indent=None, catch=None, tb_onerror=None, ignore='', single=False, comments=False): ignore = ignore.split(',') if ignore else [] def callback(e): exc = sys.exc_info() + (10,) return ''.join(format_exception(*exc)).rstrip() - kwargs = {'catch_errors': catch, 'ignore': ignore, 'single': single} + kwargs = {'catch_errors': catch, 'ignore': ignore, 'single': single, 'comments': comments} if tb_onerror: kwargs['onerror'] = callback @@ -170,6 +170,7 @@ def create_subparser(function, help): p.add_argument('--no-catch', action='store_false', dest='catch', help='only collect first error in file') p.add_argument('--tb-onerror', action='store_true', help='include tracebacks in config errors') p.add_argument('--single-file', action='store_true', dest='single', help='do not include other config files') + p.add_argument('--include-comments', action='store_true', dest='comments', help='include comments in json') p = create_subparser(build, 'builds an nginx config from a json payload') p.add_argument('filename', help='the file with the config payload') diff --git a/crossplane/builder.py b/crossplane/builder.py index 3dea54a..17b0d27 100644 --- a/crossplane/builder.py +++ b/crossplane/builder.py @@ -62,12 +62,34 @@ def _enquote(arg): def build(payload, indent=4, tabs=False): padding = '\t' if tabs else ' ' * indent + state = { + 'prev_obj': None, + 'depth': -1 + } - def _build_lines(objs, depth): - margin = padding * depth + def _put_line(line, obj): + margin = padding * state['depth'] + + # don't need put \n on first line and after comment + if state['prev_obj'] is None: + return margin + line + + # trailing comments have to be without \n + if obj['directive'] == '#' and obj['line'] == state['prev_obj']['line']: + return ' ' + line + + return '\n' + margin + line + + def _build_lines(objs): + state['depth'] = state['depth'] + 1 for obj in objs: directive = obj['directive'] + + if directive == '#': + yield _put_line('#' + obj['comment'], obj) + continue + args = [_enquote(arg) for arg in obj['args']] if directive == 'if': @@ -78,12 +100,18 @@ def _build_lines(objs, depth): line = directive if obj.get('block') is None: - yield margin + line + ';' + yield _put_line(line + ';', obj) else: - yield margin + line + ' {' - for line in _build_lines(obj['block'], depth+1): + yield _put_line(line + ' {', obj) + + # set prev_obj to propper indentation in block + state['prev_obj'] = obj + for line in _build_lines(obj['block']): yield line - yield margin + '}' + yield _put_line('}', obj) + + state['prev_obj'] = obj + state['depth'] = state['depth'] - 1 - lines = _build_lines(payload, depth=0) - return '\n'.join(lines) + lines = _build_lines(payload) + return ''.join(lines) diff --git a/crossplane/lexer.py b/crossplane/lexer.py index 9dae9c7..c47625e 100644 --- a/crossplane/lexer.py +++ b/crossplane/lexer.py @@ -43,10 +43,13 @@ def _lex_file_object(file_obj): while char.isspace(): char, line = next(it) - # if starting comment then disregard until EOL + # if starting comment if not token and char == '#': - while not char.endswith('\n'): # don't escape newlines - char, line = next(it) + while not char.endswith('\n'): + token = token + char + char, _ = next(it) + yield (token, line) + token = '' continue if not token: diff --git a/crossplane/parser.py b/crossplane/parser.py index d91d835..d0325cc 100644 --- a/crossplane/parser.py +++ b/crossplane/parser.py @@ -19,7 +19,7 @@ def _prepare_if_args(stmt): args[:] = args[start:end] -def parse(filename, onerror=None, catch_errors=True, ignore=(), single=False): +def parse(filename, onerror=None, catch_errors=True, ignore=(), single=False, comments=False): """ Parses an nginx config file and returns a nested dict payload @@ -28,6 +28,7 @@ def parse(filename, onerror=None, catch_errors=True, ignore=(), single=False): :param catch_errors: bool; if False, parse stops after first error :param ignore: list or tuple of directives to exclude from the payload :param single: bool; if True, including from other files doesn't happen + :param comments: bool; if True, including comments to json payload :returns: a payload that describes the parsed nginx config """ config_dir = os.path.dirname(filename) @@ -77,6 +78,19 @@ def _parse(parsing, tokens, ctx=(), consume=False): _parse(parsing, tokens, consume=True) continue + # if token is comment + if token.startswith('#'): + if comments: + stmt = { + "directive": "#", + "args": [], + "line": lineno, + "comment": token[1:] + } + + parsed.append(stmt) + continue + # the first token should always(?) be an nginx directive stmt = { 'directive': token, diff --git a/tests/configs/with-comments/nginx.conf b/tests/configs/with-comments/nginx.conf new file mode 100644 index 0000000..76d2e95 --- /dev/null +++ b/tests/configs/with-comments/nginx.conf @@ -0,0 +1,14 @@ +events { + worker_connections 1024; +} +#comment +http { + server { + listen 127.0.0.1:8080; #listen + server_name default_server; + location / { ## this is brace + # location / + return 200 "foo bar baz"; + } + } +} diff --git a/tests/test_build.py b/tests/test_build.py index 2cdf7d0..18f0110 100644 --- a/tests/test_build.py +++ b/tests/test_build.py @@ -25,16 +25,16 @@ def assert_equal_payloads(a, b, ignore_keys=()): assert a == b -def compare_parsed_and_built(conf_dirname, conf_basename, tmpdir): +def compare_parsed_and_built(conf_dirname, conf_basename, tmpdir, **kwargs): original_dirname = os.path.join(here, 'configs', conf_dirname) original_path = os.path.join(original_dirname, conf_basename) - original_payload = crossplane.parse(original_path) + original_payload = crossplane.parse(original_path, **kwargs) original_parsed = original_payload['config'][0]['parsed'] build1_config = crossplane.build(original_parsed) build1_file = tmpdir.join('build1.conf') build1_file.write(build1_config) - build1_payload = crossplane.parse(build1_file.strpath) + build1_payload = crossplane.parse(build1_file.strpath, **kwargs) build1_parsed = build1_payload['config'][0]['parsed'] assert_equal_payloads(original_parsed, build1_parsed, ignore_keys=['line']) @@ -42,7 +42,7 @@ def compare_parsed_and_built(conf_dirname, conf_basename, tmpdir): build2_config = crossplane.build(build1_parsed) build2_file = tmpdir.join('build2.conf') build2_file.write(build2_config) - build2_payload = crossplane.parse(build2_file.strpath) + build2_payload = crossplane.parse(build2_file.strpath, **kwargs) build2_parsed = build2_payload['config'][0]['parsed'] assert build1_config == build2_config @@ -110,6 +110,118 @@ def test_build_nested_and_multiple_args(): '}' ]) +def test_build_with_comments(): + payload = [ + { + "args" : [], + "block" : [ + { + "args" : [ + "1024" + ], + "line" : 2, + "directive" : "worker_connections" + } + ], + "line" : 1, + "directive" : "events" + }, + { + "directive" : "#", + "line" : 4, + "comment" : "comment", + "args" : [] + }, + { + "directive" : "http", + "block" : [ + { + "args" : [], + "line" : 6, + "block" : [ + { + "args" : [ + "127.0.0.1:8080" + ], + "line" : 7, + "directive" : "listen" + }, + { + "directive" : "#", + "line" : 7, + "comment" : "listen", + "args" : [] + }, + { + "args" : [ + "default_server" + ], + "directive" : "server_name", + "line" : 8 + }, + { + "args" : [ + "/" + ], + "directive" : "location", + "line" : 9, + "block" : [ + { + "args" : [], + "directive" : "#", + "line" : 9, + "comment" : "# this is brace" + }, + { + "directive" : "#", + "comment" : " location /", + "line" : 10, + "args" : [] + }, + { + "directive" : "#", + "comment" : " is here", + "line" : 11, + "args" : [] + }, + { + "args" : [ + "200", + "foo bar baz" + ], + "line" : 11, + "directive" : "return" + } + ] + } + ], + "directive" : "server" + } + ], + "line" : 5, + "args" : [] + } + ] + + built = crossplane.build(payload, indent=4, tabs=False) + + assert built == '\n'.join([ + 'events {', + ' worker_connections 1024;', + '}', + '#comment', + 'http {', + ' server {', + ' listen 127.0.0.1:8080; #listen', + ' server_name default_server;', + ' location / { ## this is brace', + ' # location /', + ' # is here', + " return 200 'foo bar baz';", + ' }', + ' }', + '}' + ]) def test_compare_parsed_and_built_simple(tmpdir): compare_parsed_and_built('simple', 'nginx.conf', tmpdir) @@ -117,3 +229,7 @@ def test_compare_parsed_and_built_simple(tmpdir): def test_compare_parsed_and_built_messy(tmpdir): compare_parsed_and_built('messy', 'nginx.conf', tmpdir) + + +def test_compare_parsed_and_built_messy_with_comments(tmpdir): + compare_parsed_and_built('with-comments', 'nginx.conf', tmpdir, comments=True) \ No newline at end of file diff --git a/tests/test_lex.py b/tests/test_lex.py index 8a5b61a..ee4afcc 100644 --- a/tests/test_lex.py +++ b/tests/test_lex.py @@ -19,34 +19,48 @@ def test_simple_config(): ('}', 12), ('}', 13) ] - -def test_messy_config(): - dirname = os.path.join(here, 'configs', 'messy') +def test_with_config_comments(): + dirname = os.path.join(here, 'configs', 'with-comments') config = os.path.join(dirname, 'nginx.conf') tokens = crossplane.lex(config) assert list(tokens) == [ - ('user', 1), ('nobody', 1), (';', 1), ('events', 3), ('{', 3), - ('worker_connections', 3), ('2048', 3), (';', 3), ('}', 3), - ('http', 5), ('{', 5), ('access_log', 7), ('off', 7), (';', 7), - ('default_type', 7), ('text/plain', 7), (';', 7), ('error_log', 7), - ('off', 7), (';', 7), ('server', 8), ('{', 8), ('listen', 9), - ('8083', 9), (';', 9), ('return', 10), ('200', 10), - ('Ser" \' \' ver\\\\ \\ $server_addr:\\$server_port\\n\\nTime: $time_local\\n\\n', 10), - (';', 10), ('}', 11), ('server', 12), ('{', 12), - ('listen', 12), ('8080', 12), (';', 12), ('root', 13), - ('/usr/share/nginx/html', 13), (';', 13), ('location', 14), ('~', 14), - ('/hello/world;', 14), ('{', 14), ('return', 14), ('301', 14), - ('/status.html', 14), (';', 14), ('}', 14), ('location', 15), - ('/foo', 15), ('{', 15), ('}', 15), ('location', 15), ('/bar', 15), - ('{', 15), ('}', 15), ('location', 16), ('/\\{\\;\\}\\ #\\ ab', 16), - ('{', 16), ('}', 16), ('if', 17), ('($request_method', 17), ('=', 17), - ('P\\{O\\)\\###\\;ST', 17), (')', 17), ('{', 17), - ('}', 17), ('location', 18), ('/status.html', 18), ('{', 18), - ('try_files', 19), ('/abc/${uri} /abc/${uri}.html', 19), ('=404', 19), - (';', 19), ('}', 20), ('location', 21), - ('/sta;\n tus', 21), ('{', 22), ('return', 22), - ('302', 22), ('/status.html', 22), (';', 22), ('}', 22), - ('location', 23), ('/upstream_conf', 23), ('{', 23), ('return', 23), - ('200', 23), ('/status.html', 23), (';', 23), ('}', 23), ('}', 23), - ('server', 24), ('{', 25), ('}', 25), ('}', 25) + (u'events', 1), (u'{', 1), (u'worker_connections', 2), (u'1024', 2), + (u';', 2), (u'}', 3),(u'#comment', 4), (u'http', 5), (u'{', 5), + (u'server', 6), (u'{', 6), (u'listen', 7), (u'127.0.0.1:8080', 7), + (u';', 7), (u'#listen', 7), (u'server_name', 8), + (u'default_server', 8),(u';', 8), (u'location', 9), (u'/', 9), + (u'{', 9), (u'## this is brace', 9), (u'# location /', 10), (u'return', 11), (u'200', 11), + (u'foo bar baz', 11), (u';', 11), (u'}', 12), (u'}', 13), (u'}', 14) ] + +def test_messy_config(): + dirname = os.path.join(here, 'configs', 'messy') + config = os.path.join(dirname, 'nginx.conf') + tokens = crossplane.lex(config) + assert list(tokens) == [(u'user', 1), (u'nobody', 1), (u';', 1), + (u'# hello\\n\\\\n\\\\\\n worlddd \\#\\\\#\\\\\\# dfsf\\n \\\\n \\\\\\n ', 2), + (u'events', 3), (u'{', 3), (u'worker_connections', 3), (u'2048', 3), + (u';', 3), (u'}', 3), (u'http', 5), (u'{', 5), (u'#forteen', 5), + (u'# this is a comment', 6),(u'access_log', 7), (u'off', 7), (u';', 7), + (u'default_type', 7), (u'text/plain', 7), (u';', 7), (u'error_log', 7), + (u'off', 7), (u';', 7), (u'server', 8), (u'{', 8), (u'listen', 9), + (u'8083', 9), (u';', 9), (u'return', 10), (u'200', 10), + (u'Ser" \' \' ver\\\\ \\ $server_addr:\\$server_port\\n\\nTime: $time_local\\n\\n', 10), + (u';', 10), (u'}', 11), (u'server', 12), (u'{', 12), (u'listen', 12), + (u'8080', 12), (u';', 12), (u'root', 13), (u'/usr/share/nginx/html', 13), + (u';', 13), (u'location', 14), (u'~', 14), (u'/hello/world;', 14), + (u'{', 14), (u'return', 14), (u'301', 14), (u'/status.html', 14), + (u';', 14), (u'}', 14), (u'location', 15), (u'/foo', 15), + (u'{', 15), (u'}', 15), (u'location', 15), (u'/bar', 15), + (u'{', 15), (u'}', 15), (u'location', 16), (u'/\\{\\;\\}\\ #\\ ab', 16), + (u'{', 16), (u'}', 16), (u'# hello', 16), (u'if', 17), + (u'($request_method', 17), (u'=', 17), (u'P\\{O\\)\\###\\;ST', 17), + (u')', 17), (u'{', 17), (u'}', 17), (u'location', 18), (u'/status.html', 18), + (u'{', 18), (u'try_files', 19), (u'/abc/${uri} /abc/${uri}.html', 19), + (u'=404', 19), (u';', 19), (u'}', 20), (u'location', 21), + (u'/sta;\n tus', 21), (u'{', 22), (u'return', 22), + (u'302', 22), (u'/status.html', 22), (u';', 22), (u'}', 22), + (u'location', 23), (u'/upstream_conf', 23), (u'{', 23), + (u'return', 23), (u'200', 23), (u'/status.html', 23), (u';', 23), + (u'}', 23), (u'}', 23), (u'server', 24), (u'{', 25), (u'}', 25), + (u'}', 25)] diff --git a/tests/test_parse.py b/tests/test_parse.py index c407ee9..0b25a33 100644 --- a/tests/test_parse.py +++ b/tests/test_parse.py @@ -367,3 +367,181 @@ def test_ignore_directives(): } ] } + +def test_config_with_comments(): + dirname = os.path.join(here, 'configs', 'with-comments') + config = os.path.join(dirname, 'nginx.conf') + payload = crossplane.parse(config, comments=True) + assert payload == { + "errors" : [], + "status" : "ok", + "config" : [ + { + "errors" : [], + "parsed" : [ + { + "block" : [ + { + "directive" : "worker_connections", + "args" : [ + "1024" + ], + "line" : 2 + } + ], + "line" : 1, + "args" : [], + "directive" : "events" + }, + { + "line" : 4, + "directive": "#", + "args": [], + "comment" : "comment" + }, + { + "block" : [ + { + "args" : [], + "directive" : "server", + "line" : 6, + "block" : [ + { + "args" : [ + "127.0.0.1:8080" + ], + "directive" : "listen", + "line" : 7 + }, + { + "args": [], + "directive": "#", + "comment" : "listen", + "line" : 7 + }, + { + "args" : [ + "default_server" + ], + "directive" : "server_name", + "line" : 8 + }, + { + "block" : [ + { + "args": [], + "directive": "#", + "line" : 9, + "comment" : "# this is brace" + }, + { + "args": [], + "directive": "#", + "line" : 10, + "comment" : " location /" + }, + { + "line" : 11, + "directive" : "return", + "args" : [ + "200", + "foo bar baz" + ] + } + ], + "line" : 9, + "directive" : "location", + "args" : [ + "/" + ] + } + ] + } + ], + "line" : 5, + "args" : [], + "directive" : "http" + } + ], + "status" : "ok", + "file" : os.path.join(dirname, 'nginx.conf') + } + ] + } + +def test_config_without_comments(): + dirname = os.path.join(here, 'configs', 'with-comments') + config = os.path.join(dirname, 'nginx.conf') + payload = crossplane.parse(config, comments=False) + assert payload == { + "errors" : [], + "status" : "ok", + "config" : [ + { + "errors" : [], + "parsed" : [ + { + "block" : [ + { + "directive" : "worker_connections", + "args" : [ + "1024" + ], + "line" : 2 + } + ], + "line" : 1, + "args" : [], + "directive" : "events" + }, + { + "block" : [ + { + "args" : [], + "directive" : "server", + "line" : 6, + "block" : [ + { + "args" : [ + "127.0.0.1:8080" + ], + "directive" : "listen", + "line" : 7 + }, + { + "args" : [ + "default_server" + ], + "directive" : "server_name", + "line" : 8 + }, + { + "block" : [ + { + "line" : 11, + "directive" : "return", + "args" : [ + "200", + "foo bar baz" + ] + } + ], + "line" : 9, + "directive" : "location", + "args" : [ + "/" + ] + } + ] + } + ], + "line" : 5, + "args" : [], + "directive" : "http" + } + ], + "status" : "ok", + "file" : os.path.join(dirname, 'nginx.conf') + } + ] + }