Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add nbqa-yapf #547

Merged
merged 12 commits into from
Apr 18, 2021
Prev Previous commit
Next Next commit
check for fase positives
  • Loading branch information
MarcoGorelli committed Apr 17, 2021
commit 8379462dae105111e005ce3cd3b2865f5b58996c
54 changes: 29 additions & 25 deletions nbqa/__main__.py
Original file line number Diff line number Diff line change
@@ -18,7 +18,7 @@
from nbqa.find_root import find_project_root
from nbqa.notebook_info import NotebookInfo
from nbqa.optional import metadata
from nbqa.output_parser import map_python_line_to_nb_lines
from nbqa.output_parser import Output, map_python_line_to_nb_lines
from nbqa.text import BOLD, RESET

BASE_ERROR_MESSAGE = dedent(
@@ -180,7 +180,7 @@ def _replace_temp_python_file_references_in_out_err(
notebook: Path,
out: str,
err: str,
) -> Tuple[str, str]:
) -> Output:
"""
Replace references to temporary Python file with references to notebook.

@@ -199,10 +199,8 @@ def _replace_temp_python_file_references_in_out_err(

Returns
-------
out
Stdout with temporary directory replaced by current working directory.
err
Stderr with temporary directory replaced by current working directory.
Output
Stdout, stderr with temporary directory replaced by current working directory.
"""
# 1. Relative path is used because some tools like pylint always report only
# the relative path of the file(relative to project root),
@@ -231,7 +229,7 @@ def _replace_temp_python_file_references_in_out_err(
out = out.replace(temp_python_file.stem, notebook.stem)
err = err.replace(temp_python_file.stem, notebook.stem)

return out, err
return Output(out, err)


def _create_blank_init_files(
@@ -382,7 +380,7 @@ def _run_command(
tmpdirname: str,
cmd_args: Sequence[str],
args: Sequence[Path],
) -> Tuple[str, str, int, bool]:
) -> Tuple[Output, int, bool]:
"""
Run third-party tool against given file or directory.

@@ -399,10 +397,8 @@ def _run_command(

Returns
-------
out
Captured stdout from running third-party tool.
err
Captured stderr from running third-party tool.
output
Captured stdout, stderr from running third-party tool.
output_code
Return code from third-party tool.
mutated
@@ -440,7 +436,7 @@ def _run_command(
out = output.stdout
err = output.stderr

return out, err, output_code, mutated
return Output(out, err), output_code, mutated


def _get_command_not_found_msg(command: str) -> str:
@@ -569,7 +565,7 @@ def _run_on_one_root_dir(

_create_blank_init_files(notebook, tmpdirname, project_root)

out, err, output_code, mutated = _run_command(
output, output_code, mutated = _run_command(
cli_args.command,
tmpdirname,
configs.nbqa_addopts,
@@ -578,15 +574,16 @@ def _run_on_one_root_dir(
),
)

actually_mutated = False
for notebook, temp_python_file in nb_to_py_mapping.items():
out, err = _replace_temp_python_file_references_in_out_err(
tmpdirname, temp_python_file, notebook, out, err
output = _replace_temp_python_file_references_in_out_err(
tmpdirname, temp_python_file, notebook, output.out, output.err
)
try:
out, err = map_python_line_to_nb_lines(
output = map_python_line_to_nb_lines(
cli_args.command,
out,
err,
output.out,
output.err,
notebook,
nb_info_mapping[notebook].cell_mappings,
)
@@ -615,10 +612,13 @@ def _run_on_one_root_dir(
raise SystemExit(msg)

try:
REPLACE_FUNCTION[configs.nbqa_diff](
temp_python_file,
notebook,
nb_info_mapping[notebook],
actually_mutated = (
REPLACE_FUNCTION[configs.nbqa_diff](
temp_python_file,
notebook,
nb_info_mapping[notebook],
)
or actually_mutated
)
except Exception as exc:
raise RuntimeError(
@@ -627,8 +627,12 @@ def _run_on_one_root_dir(
)
) from exc

sys.stdout.write(out)
sys.stderr.write(err)
sys.stdout.write(output.out)
sys.stderr.write(output.err)

if mutated and not actually_mutated:
output_code = 0
mutated = False

if configs.nbqa_diff:
if mutated:
13 changes: 10 additions & 3 deletions nbqa/output_parser.py
Original file line number Diff line number Diff line change
@@ -3,14 +3,21 @@
import re
from functools import partial
from pathlib import Path
from typing import Callable, Mapping, Match, Sequence, Tuple, Union
from typing import Callable, Mapping, Match, NamedTuple, Sequence, Tuple, Union


def _line_to_cell(match: Match[str], cell_mapping: Mapping[int, str]) -> str:
"""Replace Python line with corresponding Jupyter notebook cell."""
return str(cell_mapping[int(match.group())])


class Output(NamedTuple):
"""Captured stdout and stderr."""

out: str
err: str


def _get_pattern(
notebook: Path, command: str, cell_mapping: Mapping[int, str]
) -> Sequence[Tuple[str, Union[str, Callable[[Match[str]], str]]]]:
@@ -58,7 +65,7 @@ def _get_pattern(

def map_python_line_to_nb_lines(
command: str, out: str, err: str, notebook: Path, cell_mapping: Mapping[int, str]
) -> Tuple[str, str]:
) -> Output:
"""
Make sure stdout and stderr make reference to Jupyter Notebook cells and lines.

@@ -89,4 +96,4 @@ def map_python_line_to_nb_lines(
out = re.sub(pattern_, substitution_, out, flags=re.MULTILINE)
err = re.sub(pattern_, substitution_, err, flags=re.MULTILINE)

return out, err
return Output(out, err)
34 changes: 30 additions & 4 deletions nbqa/replace_source.py
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@
The converted file will have had the third-party tool run against it by now.
"""

import copy
import json
import sys
from difflib import unified_diff
@@ -156,7 +157,7 @@ def _notebook_code_cells(
yield cell


def mutate(python_file: "Path", notebook: "Path", notebook_info: NotebookInfo) -> None:
def mutate(python_file: "Path", notebook: "Path", notebook_info: NotebookInfo) -> bool:
"""
Replace :code:`source` code cells of original notebook.

@@ -168,8 +169,14 @@ def mutate(python_file: "Path", notebook: "Path", notebook_info: NotebookInfo) -
Jupyter Notebook third-party tool is run against (unmodified).
notebook_info
Information about notebook cells used for processing

Returns
-------
bool
Whether mutation actually happened.
"""
notebook_json = json.loads(notebook.read_text(encoding="utf-8"))
original_notebook_json = copy.deepcopy(notebook_json)

pycells = _get_pycells(python_file)
for code_cell_number, cell in enumerate(
@@ -179,14 +186,18 @@ def mutate(python_file: "Path", notebook: "Path", notebook_info: NotebookInfo) -
continue
cell["source"] = _get_new_source(code_cell_number, notebook_info, next(pycells))

if original_notebook_json == notebook_json:
return False

temp_notebook = python_file.parent / notebook.name
temp_notebook.write_text(
f"{json.dumps(notebook_json, indent=1, ensure_ascii=False)}\n", encoding="utf-8"
)
move(str(temp_notebook), str(notebook))
return True


def _print_diff(code_cell_number: int, cell_diff: Iterator[str]) -> None:
def _print_diff(code_cell_number: int, cell_diff: Iterator[str]) -> bool:
"""
Print diff between cells, colouring as appropriate.

@@ -196,6 +207,11 @@ def _print_diff(code_cell_number: int, cell_diff: Iterator[str]) -> None:
Current cell number
cell_diff
Diff between original and new versions of cell.

Returns
-------
bool
Whether non-null diff was printed.
"""
line_changes = []
for line in cell_diff:
@@ -212,9 +228,11 @@ def _print_diff(code_cell_number: int, cell_diff: Iterator[str]) -> None:
header = f"Cell {code_cell_number}"
headers = [f"{BOLD}{header}{RESET}\n", f"{'-'*len(header)}\n"]
sys.stdout.writelines(headers + line_changes + ["\n"])
return True
return False


def diff(python_file: "Path", notebook: "Path", notebook_info: NotebookInfo) -> None:
def diff(python_file: "Path", notebook: "Path", notebook_info: NotebookInfo) -> bool:
"""
View diff between new source of code cells and original sources.

@@ -226,11 +244,18 @@ def diff(python_file: "Path", notebook: "Path", notebook_info: NotebookInfo) ->
Jupyter Notebook third-party tool is run against (unmodified).
notebook_info
Information about notebook cells used for processing

Returns
-------
bool
Whether non-null diff was produced.
"""
notebook_json = json.loads(notebook.read_text(encoding="utf-8"))

pycells = _get_pycells(python_file)

actually_mutated = False

for code_cell_number, cell in enumerate(
_notebook_code_cells(notebook_json), start=1
):
@@ -245,4 +270,5 @@ def diff(python_file: "Path", notebook: "Path", notebook_info: NotebookInfo) ->
fromfile=str(notebook),
tofile=str(notebook),
)
_print_diff(code_cell_number, cell_diff)
actually_mutated = _print_diff(code_cell_number, cell_diff) or actually_mutated
return actually_mutated
Loading