[gherkin] Update gherkin-python for the Rule keyword

* Make gherkin-python up to date (primarily the Rule keyword was
  missing).
* Add back gherkin-python to the gherkin Makefile.
* Enable the use of GHERKIN_PYTHON_VERSION to select between python2
  and python3 when running the acceptance tests (python3 is the
  default).

Makefile

@@ -1,4 +1,4 @@
-LANGUAGES ?= perl go javascript ruby java elixir
+LANGUAGES ?= perl go javascript ruby java elixir python
 include default.mk
 
 post-release: print-documentation-instructions

python/bin/gherkin

@@ -1,27 +1,8 @@
-#!/usr/bin/env python
-import os
-from optparse import OptionParser
-import sys
-if sys.version_info < (3, 0) and os.name != 'nt':
-    import codecs
-    UTF8Writer = codecs.getwriter('utf8')
-    sys.stdout = UTF8Writer(sys.stdout)
-
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
-import json
-from gherkin.stream.gherkin_events import GherkinEvents
-from gherkin.stream.source_events import SourceEvents
-
-parser = OptionParser()
-parser.add_option("--no-source",  action="store_false", dest="print_source",  default=True, help="don't print source events")
-parser.add_option("--no-ast",     action="store_false", dest="print_ast",     default=True, help="don't print ast events")
-parser.add_option("--no-pickles", action="store_false", dest="print_pickles", default=True, help="don't print pickle events")
-
-(options, args) = parser.parse_args()
-
-source_events = SourceEvents(args)
-gherkin_events = GherkinEvents(options)
-
-for source_event in source_events.enum():
-    for event in gherkin_events.enum(source_event):
-        print(json.dumps(event))
+#!/usr/bin/env sh
+# Use "make GHERKIN_PYTHON_VERSION=python2 ..." to use python2
+if [ -z "$GHERKIN_PYTHON_VERSION" ];
+then
+    python3 `dirname $0`/gherkin.py $@;
+else
+    $GHERKIN_PYTHON_VERSION `dirname $0`/gherkin.py $@;
+fi

python/bin/gherkin-generate-tokens

@@ -1,19 +1,8 @@
-#!/usr/bin/env python
-import codecs
-import os
-import sys
-if sys.version_info < (3, 0):
-    import codecs
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
-from gherkin.token_scanner import TokenScanner
-from gherkin.token_formatter_builder import TokenFormatterBuilder
-from gherkin.parser import Parser
-
-files = sys.argv[1:]
-if sys.version_info < (3, 0) and os.name != 'nt':  # for Python2 unless on Windows native
-    UTF8Writer = codecs.getwriter('utf8')
-    sys.stdout = UTF8Writer(sys.stdout)
-parser = Parser(TokenFormatterBuilder())
-for file in files:
-    scanner = TokenScanner(file)
-    print(parser.parse(scanner))
+#!/usr/bin/env sh
+# Use "make GHERKIN_PYTHON_VERSION=python2 ..." to use python2
+if [ -z "$GHERKIN_PYTHON_VERSION" ];
+then
+    python3 `dirname $0`/gherkin_generate_tokens.py $@;
+else
+    $GHERKIN_PYTHON_VERSION `dirname $0`/gherkin_generate_tokens.py $@;
+fi

python/bin/gherkin.py

@@ -0,0 +1,40 @@
+import os
+from optparse import OptionParser
+import sys
+if sys.version_info < (3, 0):
+    string_type = basestring
+    if os.name != 'nt':
+        import codecs
+        UTF8Writer = codecs.getwriter('utf8')
+        sys.stdout = UTF8Writer(sys.stdout)
+else:
+    string_type = str
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+import json
+from gherkin.stream.gherkin_events import GherkinEvents
+from gherkin.stream.source_events import SourceEvents
+
+parser = OptionParser()
+parser.add_option("--no-source",  action="store_false", dest="print_source",  default=True, help="don't print source events")
+parser.add_option("--no-ast",     action="store_false", dest="print_ast",     default=True, help="don't print ast events")
+parser.add_option("--no-pickles", action="store_false", dest="print_pickles", default=True, help="don't print pickle events")
+
+(options, args) = parser.parse_args()
+
+source_events = SourceEvents(args)
+gherkin_events = GherkinEvents(options)
+
+
+def reject_empty_values(data):
+    if isinstance(data, dict):
+        return {k: reject_empty_values(v) for k, v in data.items() if v}
+    if isinstance(data, list):
+        return [reject_empty_values(v) for v in data if v]
+    else:
+        return data
+
+
+for source_event in source_events.enum():
+    for event in gherkin_events.enum(source_event):
+        print(json.dumps(reject_empty_values(event)))

python/bin/gherkin_generate_tokens.py

@@ -0,0 +1,18 @@
+import codecs
+import os
+import sys
+if sys.version_info < (3, 0):
+    import codecs
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
+from gherkin.token_scanner import TokenScanner
+from gherkin.token_formatter_builder import TokenFormatterBuilder
+from gherkin.parser import Parser
+
+files = sys.argv[1:]
+if sys.version_info < (3, 0) and os.name != 'nt':  # for Python2 unless on Windows native
+    UTF8Writer = codecs.getwriter('utf8')
+    sys.stdout = UTF8Writer(sys.stdout)
+parser = Parser(TokenFormatterBuilder())
+for file in files:
+    scanner = TokenScanner(file)
+    print(parser.parse(scanner))