From 8df9b0bd73d3b52f9bf04e09538979259b2abc89 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Sun, 1 Nov 2020 15:53:36 +0000 Subject: [PATCH 1/9] Add `mypy` config/requirement/test. --- setup.cfg | 5 +++++ setup.py | 3 +++ 2 files changed, 8 insertions(+) diff --git a/setup.cfg b/setup.cfg index de66717fb..549e17227 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,6 +21,11 @@ whitelist = words.txt dictionaries=en_US,python,technical,django +[mypy] +files = pyinfra/ pyinfra_cli/ +ignore_missing_imports = true + + [coverage:report] show_missing = true skip_covered = true diff --git a/setup.py b/setup.py index 9a2934b23..77be84c50 100644 --- a/setup.py +++ b/setup.py @@ -41,6 +41,9 @@ 'mock==3.0.5', 'codecov==2.1.8', + # Type checking + 'mypy==0.790', + # Linting 'flake8==3.8.3', 'flake8-commas==2.0.0', From 48ce4b8b235feed841a22fcb35bd148dcad96576 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Sun, 1 Nov 2020 15:53:45 +0000 Subject: [PATCH 2/9] Add stub file for facts API. --- pyinfra/api/facts.pyi | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 pyinfra/api/facts.pyi diff --git a/pyinfra/api/facts.pyi b/pyinfra/api/facts.pyi new file mode 100644 index 000000000..31dc056e4 --- /dev/null +++ b/pyinfra/api/facts.pyi @@ -0,0 +1,32 @@ +from typing import Any, Callable, Generic, Optional, Union + +FACTS: Any +FACT_LOCK: Any +SUDO_REGEX: str +SU_REGEXES: Any + +def is_fact(name: str): ... +def get_fact_class(name: str): ... +def get_fact_names(): ... + +class FactMeta(type): + def __init__(cls, name: str, bases: Any, attrs: dict) -> None: ... + +class FactBase(Generic, metaclass=FactMeta): + abstract: bool = ... + shell_executable: Optional[Union[str, Callable]] = ... + requires_command: Optional[Union[str, Callable]] = ... + @staticmethod + def default() -> Any: ... + @staticmethod + def process(output: list[str]) -> Any: ... + def process_pipeline(self, args: Any, output: Any): ... + +class ShortFactBase(metaclass=FactMeta): + fact: FactBase = ... + +def get_short_facts(state: Any, short_fact: Any, **kwargs: Any): ... +def get_facts(state: Any, name: Any, args: Optional[Any] = ..., ensure_hosts: Optional[Any] = ..., apply_failed_hosts: bool = ...): ... +def get_host_fact(state: Any, host: Any, name: Any): ... +def create_host_fact(state: Any, host: Any, name: Any, data: Any, args: Optional[Any] = ...) -> None: ... +def delete_host_fact(state: Any, host: Any, name: Any, args: Optional[Any] = ...) -> None: ... From a26fa1e50c6539e07a28f742f639e1a9fd7ed7dc Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Sun, 1 Nov 2020 15:56:48 +0000 Subject: [PATCH 3/9] Only install mypy in python3. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 77be84c50..dd29be900 100644 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ 'codecov==2.1.8', # Type checking - 'mypy==0.790', + 'mypy==0.790 ; python_version >= "3"', # Linting 'flake8==3.8.3', From f4a862745b3a1b8127c8ef85c7514f63a5c1ced5 Mon Sep 17 00:00:00 2001 From: Ian Woloschin Date: Sun, 1 Nov 2020 10:58:03 -0500 Subject: [PATCH 4/9] Add typing information for operation decorator (#444) * Add typing information for operation decorator * Move package_data to MANIFEST.in --- MANIFEST.in | 3 ++- pyinfra/api/operation.pyi | 7 +++++++ pyinfra/py.typed | 0 3 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 pyinfra/api/operation.pyi create mode 100644 pyinfra/py.typed diff --git a/MANIFEST.in b/MANIFEST.in index 0e52f9e8c..140fd13c4 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1 +1,2 @@ -include README.md LICENSE.md CHANGELOG.md +include README.md LICENSE.md CHANGELOG.md pyinfra/py.typed +global-include *.pyi diff --git a/pyinfra/api/operation.pyi b/pyinfra/api/operation.pyi new file mode 100644 index 000000000..d6b9eec20 --- /dev/null +++ b/pyinfra/api/operation.pyi @@ -0,0 +1,7 @@ +from typing import Any, Callable, TypeVar + +_TFunc = TypeVar("_TFunc", bound=Callable[..., Any]) + + +def operation(func: _TFunc = None, pipeline_facts=None, name: str = None) -> _TFunc: + ... diff --git a/pyinfra/py.typed b/pyinfra/py.typed new file mode 100644 index 000000000..e69de29bb From 9d1369651c4a0ffbd4fcd5ae9e3708e856c9b089 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Sun, 1 Nov 2020 16:18:21 +0000 Subject: [PATCH 5/9] Fix mypy files config. --- setup.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 549e17227..42d4eb4b4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -22,7 +22,6 @@ dictionaries=en_US,python,technical,django [mypy] -files = pyinfra/ pyinfra_cli/ ignore_missing_imports = true From 59c8237177ae7d0afb0e5d243d57c7be3050d2fd Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Fri, 22 Jan 2021 14:36:42 +0000 Subject: [PATCH 6/9] Run mypy checks in test workflow (ubuntu18/python3.8). --- .github/workflows/test.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d20442c67..5eb5414c3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,3 +44,7 @@ jobs: - name: Local integration tests run: pytest -m local if: ${{ matrix.os != 'windows-2019' }} + + - name: Mypy typing checks + run: mypy pyinfra/ pyinfra_cli/ + if: ${{ matrix.os == 'ubuntu-18.04' && matrix.python-version == '3.8' }} From 9f4402c1d0c8a97373a8b8a7094e1c3a06bf4627 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Sat, 23 Jan 2021 16:24:08 +0000 Subject: [PATCH 7/9] Tidy up facts/operation API type files. --- pyinfra/api/facts.pyi | 77 ++++++++++++++++++++++++++++++++------- pyinfra/api/operation.pyi | 31 ++++++++++++++-- 2 files changed, 91 insertions(+), 17 deletions(-) diff --git a/pyinfra/api/facts.pyi b/pyinfra/api/facts.pyi index 31dc056e4..51eb5d8ff 100644 --- a/pyinfra/api/facts.pyi +++ b/pyinfra/api/facts.pyi @@ -1,32 +1,81 @@ -from typing import Any, Callable, Generic, Optional, Union +from typing import Any, Callable, Generic, Optional, Type, Union FACTS: Any FACT_LOCK: Any SUDO_REGEX: str SU_REGEXES: Any -def is_fact(name: str): ... -def get_fact_class(name: str): ... -def get_fact_names(): ... + +def is_fact(name: str): + ... + + +def get_fact_class(name: str): + ... + + +def get_fact_names(): + ... + class FactMeta(type): - def __init__(cls, name: str, bases: Any, attrs: dict) -> None: ... + def __init__(cls, name: str, bases: Any, attrs: dict) -> None: + ... + class FactBase(Generic, metaclass=FactMeta): abstract: bool = ... shell_executable: Optional[Union[str, Callable]] = ... requires_command: Optional[Union[str, Callable]] = ... + @staticmethod - def default() -> Any: ... + def default() -> Any: + ... + @staticmethod - def process(output: list[str]) -> Any: ... - def process_pipeline(self, args: Any, output: Any): ... + def process(output: list[str]) -> Any: + ... + + def process_pipeline(self, args: Any, output: Any): + ... + class ShortFactBase(metaclass=FactMeta): - fact: FactBase = ... + fact: Type[FactBase] = ... + + +def get_short_facts(state: Any, short_fact: Any, **kwargs: Any): + ... + + +def get_facts( + state: Any, + name: Any, + args: Optional[Any] = ..., + ensure_hosts: Optional[Any] = ..., + apply_failed_hosts: bool = ..., +): + ... + + +def get_host_fact(state: Any, host: Any, name: Any): + ... + + +def create_host_fact( + state: Any, + host: Any, + name: Any, + data: Any, + args: Optional[Any] = ..., +) -> None: + ... + -def get_short_facts(state: Any, short_fact: Any, **kwargs: Any): ... -def get_facts(state: Any, name: Any, args: Optional[Any] = ..., ensure_hosts: Optional[Any] = ..., apply_failed_hosts: bool = ...): ... -def get_host_fact(state: Any, host: Any, name: Any): ... -def create_host_fact(state: Any, host: Any, name: Any, data: Any, args: Optional[Any] = ...) -> None: ... -def delete_host_fact(state: Any, host: Any, name: Any, args: Optional[Any] = ...) -> None: ... +def delete_host_fact( + state: Any, + host: Any, + name: Any, + args: Optional[Any] = ..., +) -> None: + ... diff --git a/pyinfra/api/operation.pyi b/pyinfra/api/operation.pyi index d6b9eec20..f79f8e067 100644 --- a/pyinfra/api/operation.pyi +++ b/pyinfra/api/operation.pyi @@ -1,7 +1,32 @@ -from typing import Any, Callable, TypeVar +from typing import Any, Callable, Optional -_TFunc = TypeVar("_TFunc", bound=Callable[..., Any]) +OPERATIONS: Any -def operation(func: _TFunc = None, pipeline_facts=None, name: str = None) -> _TFunc: +def get_operation_names(): + ... + + +class OperationMeta: + commands: Any = ... + hash: Any = ... + changed: Any = ... + + def __init__(self, hash: Optional[Any] = ..., commands: Optional[Any] = ...) -> None: + ... + + +def add_op(state: Any, op_func: Any, *args: Any, **kwargs: Any): + ... + + +def show_set_name_warning(call_location: Any) -> None: + ... + + +def show_state_host_arguments_warning(call_location: Any) -> None: + ... + + +def operation(func: Optional[Callable[..., Any]] = ..., pipeline_facts: Optional[Any] = ...): ... From 1021520ab8e116c387106efb69fec8253181d5f8 Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Sat, 23 Jan 2021 16:24:31 +0000 Subject: [PATCH 8/9] Move pseudo module imports into pyinfra module init. --- pyinfra/__init__.py | 21 +++++++++++---------- pyinfra/pseudo_modules.py | 24 +++--------------------- 2 files changed, 14 insertions(+), 31 deletions(-) diff --git a/pyinfra/__init__.py b/pyinfra/__init__.py index b5b38a39b..d4e8dc591 100644 --- a/pyinfra/__init__.py +++ b/pyinfra/__init__.py @@ -11,18 +11,19 @@ # Global pyinfra logger logger = logging.getLogger('pyinfra') -# Setup package level version -from .version import __version__ # noqa +from . import pseudo_modules # noqa: E402 +from .version import __version__ # noqa: E402, F401 -# Trigger pseudo_* creation -from . import pseudo_modules # noqa - -# Trigger fact index creation -from . import facts # noqa - -# Trigger module imports -from . import operations # noqa # pragma: no cover +# Setup pseudo modules +state = pseudo_state = pseudo_modules.pseudo_state +host = pseudo_host = pseudo_modules.pseudo_host +inventory = pseudo_inventory = pseudo_modules.pseudo_inventory # Initialise base classes - this sets the pseudo modules to point at the underlying # class objects (Host, etc), which makes ipython/etc work as expected. pseudo_modules.init_base_classes() + +# TODO: remove this once we have explicit fact importing (break in v2) +# Trigger fact/operations index creation +from . import facts # noqa +from . import operations # noqa # pragma: no cover diff --git a/pyinfra/pseudo_modules.py b/pyinfra/pseudo_modules.py index 774f457d3..982db6e92 100644 --- a/pyinfra/pseudo_modules.py +++ b/pyinfra/pseudo_modules.py @@ -9,10 +9,6 @@ executing in CLI mode). ''' -import sys - -import pyinfra - class PseudoModule(object): _module = None @@ -57,23 +53,9 @@ def isset(self): return self._module is not None -# The current deploy state -pseudo_state = \ - sys.modules['pyinfra.pseudo_state'] = sys.modules['pyinfra.state'] = \ - pyinfra.pseudo_state = pyinfra.state = \ - PseudoModule() - -# The current deploy inventory -pseudo_inventory = \ - sys.modules['pyinfra.pseudo_inventory'] = sys.modules['pyinfra.inventory'] = \ - pyinfra.pseudo_inventory = pyinfra.inventory = \ - PseudoModule() - -# The current target host -pseudo_host = \ - sys.modules['pyinfra.pseudo_host'] = sys.modules['pyinfra.host'] = \ - pyinfra.pseudo_host = pyinfra.host = \ - PseudoModule() +pseudo_state = PseudoModule() +pseudo_inventory = PseudoModule() +pseudo_host = PseudoModule() def init_base_classes(): From ba984d2de6815a1d9dbaaab92c4b3e94c7aef00a Mon Sep 17 00:00:00 2001 From: Nick Barrett Date: Sat, 23 Jan 2021 16:25:14 +0000 Subject: [PATCH 9/9] Capture base `TemplateError` in `files.template` operation. --- pyinfra/operations/files.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyinfra/operations/files.py b/pyinfra/operations/files.py index fb4755406..74d9b011b 100644 --- a/pyinfra/operations/files.py +++ b/pyinfra/operations/files.py @@ -14,7 +14,7 @@ import six -from jinja2 import TemplateRuntimeError, TemplateSyntaxError, UndefinedError +from jinja2 import TemplateError, UndefinedError from pyinfra import logger from pyinfra.api import ( @@ -802,7 +802,7 @@ def template( # Render and make file-like it's output try: output = get_template(src).render(data) - except (TemplateRuntimeError, TemplateSyntaxError, UndefinedError) as e: + except (TemplateError, UndefinedError) as e: trace_frames = traceback.extract_tb(sys.exc_info()[2]) trace_frames = [ frame for frame in trace_frames