From 97f51a1d43d9056a0fa9c3efe05cf712c27a4509 Mon Sep 17 00:00:00 2001 From: Justin DuJardin Date: Sun, 22 Nov 2020 12:05:47 -0800 Subject: [PATCH] Feature/test doc snippets (#40) * test(ci): add readme snippet test runner - gather up and execute the python snippets in the Readme to make sure they work. * chore: update docs and mathy_pydoc version * chore: update docs --- .travis.yml | 1 + README.md | 52 ++++++++++++++++++++++---------------------- requirements-dev.txt | 2 +- tools/test_readme.py | 48 ++++++++++++++++++++++++++++++++++++++++ tools/test_readme.sh | 4 ++++ 5 files changed, 80 insertions(+), 27 deletions(-) create mode 100644 tools/test_readme.py create mode 100644 tools/test_readme.sh diff --git a/.travis.yml b/.travis.yml index b35ed8a..bc168a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,7 @@ matrix: - sh tools/setup.sh script: - sh tools/lint.sh + - sh tools/test_readme.sh - sh tools/test.sh after_success: - sh tools/codecov.sh diff --git a/README.md b/README.md index dab35b4..a70e422 100644 --- a/README.md +++ b/README.md @@ -65,7 +65,7 @@ pathy>=0.1.37,<0.2.0 # Pathy class -```python +```python (doc) Pathy(self, args, kwargs) ``` @@ -73,7 +73,7 @@ Subclass of `pathlib.Path` that works with bucket APIs. ## exists method -```python +```python (doc) Pathy.exists(self) -> bool ``` @@ -81,7 +81,7 @@ Returns True if the path points to an existing bucket, blob, or prefix. ## fluid classmethod -```python +```python (doc) Pathy.fluid( path_candidate: Union[str, Pathy, pathlib.Path], ) -> Union[Pathy, pathlib.Path] @@ -108,7 +108,7 @@ assert fluid_path.prefix == "foo.txt/" ## from_bucket classmethod -```python +```python (doc) Pathy.from_bucket(bucket_name: str) -> 'Pathy' ``` @@ -124,7 +124,7 @@ assert str(Pathy.from_bucket("two")) == "gs://two/" ## glob method -```python +```python (doc) Pathy.glob( self: 'Pathy', pattern: str, @@ -136,7 +136,7 @@ blobs. ## is_dir method -```python +```python (doc) Pathy.is_dir(self: 'Pathy') -> bool ``` @@ -148,7 +148,7 @@ Returns False if it points to a blob or the path doesn't exist. ## is_file method -```python +```python (doc) Pathy.is_file(self: 'Pathy') -> bool ``` @@ -160,7 +160,7 @@ exist. ## iterdir method -```python +```python (doc) Pathy.iterdir( self: 'Pathy', ) -> Generator[Pathy, NoneType, NoneType] @@ -170,7 +170,7 @@ Iterate over the blobs found in the given bucket or blob prefix path. ## mkdir method -```python +```python (doc) Pathy.mkdir( self, mode: int = 511, @@ -193,7 +193,7 @@ Raises FileExistsError if exist_ok is false and the bucket already exists. ## open method -```python +```python (doc) Pathy.open( self: 'Pathy', mode: str = 'r', @@ -210,7 +210,7 @@ providers. ## owner method -```python +```python (doc) Pathy.owner(self: 'Pathy') -> Optional[str] ``` @@ -220,7 +220,7 @@ not supported by the bucket API provider. ## rename method -```python +```python (doc) Pathy.rename(self: 'Pathy', target: Union[str, pathlib.PurePath]) -> None ``` @@ -234,7 +234,7 @@ to match the target prefix. ## replace method -```python +```python (doc) Pathy.replace(self: 'Pathy', target: Union[str, pathlib.PurePath]) -> None ``` @@ -244,7 +244,7 @@ If target points to an existing path, it will be replaced. ## resolve method -```python +```python (doc) Pathy.resolve(self, strict: bool = False) -> 'Pathy' ``` @@ -259,7 +259,7 @@ assert path.resolve() == Pathy("gs://my_bucket/blob") ## rglob method -```python +```python (doc) Pathy.rglob( self: 'Pathy', pattern: str, @@ -271,7 +271,7 @@ all matched blobs. Imagine adding "\*\*/" before a call to glob. ## rmdir method -```python +```python (doc) Pathy.rmdir(self: 'Pathy') -> None ``` @@ -279,7 +279,7 @@ Removes this bucket or blob prefix. It must be empty. ## samefile method -```python +```python (doc) Pathy.samefile( self: 'Pathy', other_path: Union[str, bytes, int, pathlib.Path], @@ -290,7 +290,7 @@ Determine if this path points to the same location as other_path. ## stat method -```python +```python (doc) Pathy.stat(self: 'Pathy') -> pathy.base.BlobStat ``` @@ -298,7 +298,7 @@ Returns information about this bucket path. ## to_local classmethod -```python +```python (doc) Pathy.to_local( blob_path: Union[Pathy, str], recurse: bool = True, @@ -312,7 +312,7 @@ as their updated timestamps change. ## touch method -```python +```python (doc) Pathy.touch(self: 'Pathy', mode: int = 438, exist_ok: bool = True) -> None ``` @@ -324,7 +324,7 @@ FileExistsError is raised. # BlobStat dataclass -```python +```python (doc) BlobStat( self, size: Optional[int], @@ -336,7 +336,7 @@ Stat for a bucket item # use_fs function -```python +```python (doc) use_fs( root: Optional[str, pathlib.Path, bool] = None, ) -> Optional[pathy.file.BucketClientFS] @@ -349,7 +349,7 @@ applications. # get_fs_client function -```python +```python (doc) get_fs_client() -> Optional[pathy.file.BucketClientFS] ``` @@ -357,7 +357,7 @@ Get the file-system client (or None) # use_fs_cache function -```python +```python (doc) use_fs_cache( root: Optional[str, pathlib.Path, bool] = None, ) -> Optional[pathlib.Path] @@ -370,7 +370,7 @@ times, or need to pass a local file path to a third-party library. # get_fs_cache function -```python +```python (doc) get_fs_cache() -> Optional[pathlib.Path] ``` @@ -378,7 +378,7 @@ Get the folder that holds file-system cached blobs and timestamps. # set_client_params function -```python +```python (doc) set_client_params(scheme: str, kwargs: Any) -> None ``` diff --git a/requirements-dev.txt b/requirements-dev.txt index 3b0ecbb..4e46dff 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -7,7 +7,7 @@ black pytest-coverage tox mock -mathy_pydoc +mathy_pydoc>=0.7.22,<0.8.0 typer-cli # test serializing spacy models using Pathy in place of Path diff --git a/tools/test_readme.py b/tools/test_readme.py new file mode 100644 index 0000000..7dd0e22 --- /dev/null +++ b/tools/test_readme.py @@ -0,0 +1,48 @@ +import re +from pathlib import Path +from typing import List + +import typer + + +def extract_code_snippets(lines: List[str]) -> List[str]: + inside_codeblock = False + blocks = [] + current_block: List[str] = [] + while len(lines) > 0: + line = lines.pop(0) + if not inside_codeblock: + inside_codeblock = re.match(r"```(p|P)ython$", line.strip()) + else: + end_block = re.match(r"```", line.strip()) + if end_block: + blocks.append("\n".join(current_block)) + current_block = [] + inside_codeblock = False + else: + current_block.append(line) + + return blocks + + +def exec_snippet(text: str): + try: + exec(text) + except BaseException as identifier: + typer.echo(f"TEST Failed! == ERROR: {identifier}") + typer.echo("SNIPPET") + typer.echo(text) + typer.Exit(1) + raise identifier + + +def main(readme_file: Path): + readme_lines = readme_file.read_text().split("\n") + snippets = extract_code_snippets(readme_lines) + for snip in snippets: + exec_snippet(snip) + typer.echo("All snippets in readme executed without error!") + + +if __name__ == "__main__": + typer.run(main) diff --git a/tools/test_readme.sh b/tools/test_readme.sh new file mode 100644 index 0000000..5a18be2 --- /dev/null +++ b/tools/test_readme.sh @@ -0,0 +1,4 @@ +#!/bin/bash +set -e +. .env/bin/activate +python tools/test_readme.py README.md