use request converter to generate python examples

Author: miguelgrinberg

utils/generate-examples.py

@@ -19,18 +19,17 @@
 
 import collections
 import json
-import os
-import tempfile
 from pathlib import Path
+import subprocess
 
-import black
 from click.testing import CliRunner
 from jinja2 import Environment, FileSystemLoader
 
 code_root = Path(__file__).absolute().parent.parent
 asciidocs_dir = code_root / "docs/examples"
 flight_recorder_dir = code_root.parent / "clients-flight-recorder"
 report_path = flight_recorder_dir / "recordings/docs/parsed-alternative-report.json"
+request_converter_path = flight_recorder_dir / "scripts/parsed-alternative-report-recorder/node_modules/.bin/es-request-converter"
 substitutions = {"type": "doc_type", "from": "from_"}
 
 jinja_env = Environment(
@@ -195,14 +194,6 @@
 ParsedSource = collections.namedtuple("ParsedSource", ["api", "params", "body"])
 
 
-def blacken(filename):
-    runner = CliRunner()
-    result = runner.invoke(
-        black.main, [str(filename), "--line-length=75", "--target-version=py37"]
-    )
-    assert result.exit_code == 0, result.output
-
-
 def main():
     for filepath in asciidocs_dir.iterdir():
         if filepath.name.endswith(".asciidoc"):
@@ -213,62 +204,40 @@ def main():
             f"clients-flight-recorder repository not checked out at {flight_recorder_dir}"
         )
 
+    if not request_converter_path.exists():
+        raise RuntimeError(
+            f"request-converter not installed in clients-flight-recorder"
+        )
+
     with report_path.open() as f:
         report = json.loads(f.read())
 
-    t = jinja_env.get_template("example")
-
+    count = 0
+    errors = 0
     for exm in report:
         if exm["lang"] != "console":
             continue
-        if exm["source_location"]["file"] not in files_to_generate:
-            continue
-
-        parsed_sources = []
-        for src in exm["parsed_source"]:
-            params = (src.get("params") or {}).copy()
-            params.update(src.get("query") or {})
-            params = {
-                k: (list(v.split(",")) if isinstance(v, str) and "," in v else v)
-                for k, v in params.items()
-            }
 
-            parsed_sources.append(
-                ParsedSource(
-                    api=src["api"],
-                    params={
-                        substitutions.get(k, k): repr(v) for k, v in params.items()
-                    },
-                    body=src.get("body", None) or None,
+        proc = subprocess.run(
+            [request_converter_path, '-f', 'python', '--print-response'],
+            input=exm['source'].encode(), capture_output=True)
+        if proc.returncode == 0:
+            data = proc.stdout.decode()
+
+            with (asciidocs_dir / f"{exm['digest']}.asciidoc").open(mode="w") as f:
+                f.truncate()
+                f.write(
+                    f"""// {exm['source_location']['file']}:{exm['source_location']['line']}
+
+    [source, python]
+    ----
+    {data}
+    ----"""
                 )
-            )
-
-        with tempfile.NamedTemporaryFile("w+", delete=False) as tmp_file:
-            tmp_file.write(t.render(parsed_sources=parsed_sources))
-
-        try:
-            blacken(tmp_file.name)
-        except AssertionError:
-            loc = exm["source_location"]
-            print(f"Failed to format {loc['file']}:{loc['line']}, skipping.")
-            continue
-
-        with open(tmp_file.name) as f:
-            data = f.read()
-            data = data.rstrip().replace(",)", ")")
-
-        os.unlink(tmp_file.name)
-
-        with (asciidocs_dir / f"{exm['digest']}.asciidoc").open(mode="w") as f:
-            f.truncate()
-            f.write(
-                f"""// {exm['source_location']['file']}:{exm['source_location']['line']}
-
-[source, python]
-----
-{data}
-----"""
-            )
+        else:
+            errors += 1
+        count += 1
+        print(f'\r{count}/{len(report)} generated ({errors} errors)', end='')
 
 
 if __name__ == "__main__":