diff --git a/noxfile.py b/noxfile.py index 7de717838..16243f8c1 100644 --- a/noxfile.py +++ b/noxfile.py @@ -20,16 +20,17 @@ def generate_protos(session): session.run( "python", "-m", "grpc_tools.protoc", "-Isynthtool/protos", "--python_out=synthtool/protos", "synthtool/protos/metadata.proto", "synthtool/protos/preconfig.proto") -@nox.session(python=['3.6', '3.8']) +@nox.session(python=['3.6', '3.9']) def blacken(session): - session.install('black==19.10b0') + session.install('black==22.3.0', 'click>8.0') session.run('black', 'synthtool', 'tests') @nox.session(python=['3.6', '3.8', '3.9']) def lint(session): - session.install('mypy==0.790', 'flake8', 'black==19.10b0') + session.install('mypy==0.790', 'flake8', 'black==22.3.0') session.run('pip', 'install', '-e', '.') + session.run('pip', 'install', 'click>8.0') session.run('black', '--check', 'synthtool', 'tests') session.run('flake8', 'synthtool', 'tests') session.run('mypy', 'synthtool') diff --git a/synthtool/gcp/common.py b/synthtool/gcp/common.py index 0a79d544d..b1fbb6564 100644 --- a/synthtool/gcp/common.py +++ b/synthtool/gcp/common.py @@ -423,7 +423,7 @@ def detect_versions( def decamelize(value: str): - """ parser to convert fooBar.js to Foo Bar. """ + """Parser to convert fooBar.js to Foo Bar.""" if not value: return "" str_decamelize = re.sub("^.", value[0].upper(), value) # apple -> Apple. diff --git a/synthtool/gcp/samples.py b/synthtool/gcp/samples.py index 35b9f214f..21a18d98b 100644 --- a/synthtool/gcp/samples.py +++ b/synthtool/gcp/samples.py @@ -78,7 +78,7 @@ def all_samples(sample_globs: List[str]) -> List[Dict[str, str]]: def _decamelize(value: str): - """ parser to convert fooBar.js to Foo Bar. """ + """Parser to convert fooBar.js to Foo Bar.""" if not value: return "" str_decamelize = re.sub("^.", value[0].upper(), value) # apple -> Apple. diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py index 4092178ee..84d1de387 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_cli.py @@ -85,7 +85,8 @@ def str2bool(v): args = parser.parse_args() notebooks = execute_changed_notebooks_helper.get_changed_notebooks( - test_paths_file=args.test_paths_file, base_branch=args.base_branch, + test_paths_file=args.test_paths_file, + base_branch=args.base_branch, ) execute_changed_notebooks_helper.process_and_execute_notebooks( diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_helper.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_helper.py index 228801744..f454205f7 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_helper.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_changed_notebooks_helper.py @@ -64,7 +64,9 @@ class NotebookExecutionResult: def _process_notebook( - notebook_path: str, variable_project_id: str, variable_region: str, + notebook_path: str, + variable_project_id: str, + variable_region: str, ): # Read notebook with open(notebook_path) as f: @@ -77,7 +79,10 @@ def _process_notebook( ) # Use no-execute preprocessor - (nb, resources,) = remove_no_execute_cells_preprocessor.preprocess(nb) + ( + nb, + resources, + ) = remove_no_execute_cells_preprocessor.preprocess(nb) (nb, resources) = update_variables_preprocessor.preprocess(nb, resources) @@ -183,7 +188,8 @@ def process_and_execute_notebook( def get_changed_notebooks( - test_paths_file: str, base_branch: Optional[str] = None, + test_paths_file: str, + base_branch: Optional[str] = None, ) -> List[str]: """ Get the notebooks that exist under the folders defined in the test_paths_file. @@ -293,7 +299,9 @@ def process_and_execute_notebooks( print("\n=== RESULTS ===\n") results_sorted = sorted( - notebook_execution_results, key=lambda result: result.is_pass, reverse=True, + notebook_execution_results, + key=lambda result: result.is_pass, + reverse=True, ) # Print results diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_helper.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_helper.py index 32c7de372..d59b7b616 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_helper.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/execute_notebook_helper.py @@ -28,7 +28,9 @@ def execute_notebook( - notebook_source: str, output_file_or_uri: str, should_log_output: bool, + notebook_source: str, + output_file_or_uri: str, + should_log_output: bool, ): """Execute a single notebook using Papermill""" file_name = os.path.basename(os.path.normpath(notebook_source)) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/NotebookProcessors.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/NotebookProcessors.py index 1b7934ea6..d445750e0 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/NotebookProcessors.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/NotebookProcessors.py @@ -54,7 +54,8 @@ def preprocess(self, notebook, resources=None): for cell in notebook.cells: if cell.cell_type == "code": cell.source = self.update_variables( - content=cell.source, replacement_map=self._replacement_map, + content=cell.source, + replacement_map=self._replacement_map, ) executable_cells.append(cell) diff --git a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/utils.py b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/utils.py index c72aedd1d..b4978e8b4 100644 --- a/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/utils.py +++ b/synthtool/gcp/templates/python_notebooks_testing_pipeline/.cloud-build/utils/utils.py @@ -31,7 +31,10 @@ def download_file(bucket_name: str, blob_name: str, destination_file: str) -> st return destination_file -def upload_file(local_file_path: str, remote_file_path: str,) -> str: +def upload_file( + local_file_path: str, + remote_file_path: str, +) -> str: """Copies a local file to a GCS path""" subprocess.check_output( ["gsutil", "cp", local_file_path, remote_file_path], encoding="UTF-8" diff --git a/synthtool/languages/java.py b/synthtool/languages/java.py index d775467e5..2202c69e8 100644 --- a/synthtool/languages/java.py +++ b/synthtool/languages/java.py @@ -455,7 +455,8 @@ def _common_template_metadata() -> Dict[str, Any]: ) metadata["latest_bom_version"] = latest_maven_version( - group_id="com.google.cloud", artifact_id="libraries-bom", + group_id="com.google.cloud", + artifact_id="libraries-bom", ) metadata["samples"] = samples.all_samples(["samples/**/src/main/java/**/*.java"]) diff --git a/synthtool/languages/php.py b/synthtool/languages/php.py index 99310914c..2e3a5ad68 100644 --- a/synthtool/languages/php.py +++ b/synthtool/languages/php.py @@ -66,8 +66,7 @@ def _merge(src: str, dest: str, path: Path): def _find_copy_target(src: Path, version_string: str) -> typing.Optional[Path]: - """Returns a directory contains the version subdirectory. - """ + """Returns a directory contains the version subdirectory.""" logger.debug("_find_copy_target called with %s and %s", src, version_string) entries = os.scandir(src) if not entries: @@ -81,10 +80,11 @@ def _find_copy_target(src: Path, version_string: str) -> typing.Optional[Path]: def owlbot_copy_version( - src: Path, dest: Path, copy_excludes: typing.Optional[typing.List[str]] = None, + src: Path, + dest: Path, + copy_excludes: typing.Optional[typing.List[str]] = None, ) -> None: - """Copies files from a version subdirectory. - """ + """Copies files from a version subdirectory.""" logger.debug("owlbot_copy_version called from %s to %s", src, dest) if copy_excludes is None: @@ -146,8 +146,7 @@ def owlbot_main( copy_excludes: typing.Optional[typing.List[str]] = None, patch_func: typing.Callable[[], None] = owlbot_patch, ) -> None: - """Copies files from generated tree. - """ + """Copies files from generated tree.""" entries = os.scandir(src) if not entries: logger.info("there is no version subdirectory to copy") @@ -161,9 +160,7 @@ def owlbot_main( def owlbot_entrypoint(staging_dir: str = STAGING_DIR) -> None: - """Copies files from staging and template directories into current working dir. - - """ + """Copies files from staging and template directories into current working dir.""" logging.basicConfig(level=logging.INFO) logger.debug("owlbot_main called") diff --git a/synthtool/languages/python.py b/synthtool/languages/python.py index 78a6a0ff2..71c8890d6 100644 --- a/synthtool/languages/python.py +++ b/synthtool/languages/python.py @@ -56,7 +56,7 @@ def fix_pb2_headers(*, proto_root: str = "**/*_pb2.py") -> None: s.replace( proto_root, PB2_HEADER, - fr"\g<1>{LICENSE}\n\n\g<2>", # change order to avoid stacking replacements + rf"\g<1>{LICENSE}\n\n\g<2>", # change order to avoid stacking replacements flags=re.DOTALL | re.MULTILINE, ) @@ -65,7 +65,7 @@ def fix_pb2_grpc_headers(*, proto_root: str = "**/*_pb2_grpc.py") -> None: s.replace( proto_root, PB2_GRPC_HEADER, - fr"{LICENSE}\n\n\g<1>\n\n\g<2>", # add line breaks to avoid stacking replacements + rf"{LICENSE}\n\n\g<1>\n\n\g<2>", # add line breaks to avoid stacking replacements ) @@ -150,15 +150,15 @@ def py_samples(*, root: PathOrStr = None, skip_readmes: bool = False) -> None: def configure_previous_major_version_branches() -> None: """Configure releases from previous major version branches by editing - `.github/release-please.yml`. + `.github/release-please.yml`. - The current library version is obtained from `version.py` in `google/**/version.py`, - or the `setup.py`. + The current library version is obtained from `version.py` in `google/**/version.py`, + or the `setup.py`. - Releases are configured for all previous major versions. For example, - if the library version is currently 3.5.1, the release-please config - will include v2, v1, and v0. - """ + Releases are configured for all previous major versions. For example, + if the library version is currently 3.5.1, the release-please config + will include v2, v1, and v0. + """ if list(Path(".").glob("google/**/version.py")): version_file = list(Path(".").glob("google/**/version.py"))[0] else: diff --git a/synthtool/protos/metadata_pb2.py b/synthtool/protos/metadata_pb2.py index 767a1ce98..fcfeb548e 100644 --- a/synthtool/protos/metadata_pb2.py +++ b/synthtool/protos/metadata_pb2.py @@ -22,7 +22,9 @@ serialized_options=None, create_key=_descriptor._internal_create_key, serialized_pb=b'\n\x0emetadata.proto\x12\x14yoshi.synth.metadata\x1a\x1fgoogle/protobuf/timestamp.proto"\xf6\x01\n\x08Metadata\x12\x33\n\x0bupdate_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.TimestampB\x02\x18\x01\x12-\n\x07sources\x18\x02 \x03(\x0b\x32\x1c.yoshi.synth.metadata.Source\x12\x37\n\x0c\x64\x65stinations\x18\x03 \x03(\x0b\x32!.yoshi.synth.metadata.Destination\x12\x34\n\tnew_files\x18\x04 \x03(\x0b\x32\x1d.yoshi.synth.metadata.NewFileB\x02\x18\x01\x12\x17\n\x0fgenerated_files\x18\x05 \x03(\t"\xb8\x01\n\x06Source\x12.\n\x03git\x18\x01 \x01(\x0b\x32\x1f.yoshi.synth.metadata.GitSourceH\x00\x12:\n\tgenerator\x18\x02 \x01(\x0b\x32%.yoshi.synth.metadata.GeneratorSourceH\x00\x12\x38\n\x08template\x18\x03 \x01(\x0b\x32$.yoshi.synth.metadata.TemplateSourceH\x00\x42\x08\n\x06source"m\n\tGitSource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06remote\x18\x02 \x01(\t\x12\x0b\n\x03sha\x18\x03 \x01(\t\x12\x14\n\x0cinternal_ref\x18\x04 \x01(\t\x12\x12\n\nlocal_path\x18\x05 \x01(\t\x12\x0b\n\x03log\x18\x06 \x01(\t"F\n\x0fGeneratorSource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t\x12\x14\n\x0c\x64ocker_image\x18\x03 \x01(\t"?\n\x0eTemplateSource\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06origin\x18\x02 \x01(\t\x12\x0f\n\x07version\x18\x03 \x01(\t"\x94\x01\n\x0b\x44\x65stination\x12\x39\n\x06\x63lient\x18\x01 \x01(\x0b\x32\'.yoshi.synth.metadata.ClientDestinationH\x00\x12;\n\x07\x66ileset\x18\x02 \x01(\x0b\x32(.yoshi.synth.metadata.FileSetDestinationH\x00\x42\r\n\x0b\x44\x65stination"\x17\n\x07NewFile\x12\x0c\n\x04path\x18\x01 \x01(\t"\x7f\n\x11\x43lientDestination\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\x10\n\x08\x61pi_name\x18\x02 \x01(\t\x12\x13\n\x0b\x61pi_version\x18\x03 \x01(\t\x12\x10\n\x08language\x18\x04 \x01(\t\x12\x11\n\tgenerator\x18\x05 \x01(\t\x12\x0e\n\x06\x63onfig\x18\x06 \x01(\t"3\n\x12\x46ileSetDestination\x12\x0e\n\x06source\x18\x01 \x01(\t\x12\r\n\x05\x66iles\x18\x02 \x03(\tb\x06proto3', - dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,], + dependencies=[ + google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR, + ], ) diff --git a/synthtool/protos/preconfig_pb2.py b/synthtool/protos/preconfig_pb2.py index 4225ea050..48a4b10d3 100644 --- a/synthtool/protos/preconfig_pb2.py +++ b/synthtool/protos/preconfig_pb2.py @@ -110,7 +110,9 @@ ), ], extensions=[], - nested_types=[_PRECONFIG_PRECLONEDREPOSENTRY,], + nested_types=[ + _PRECONFIG_PRECLONEDREPOSENTRY, + ], enum_types=[], serialized_options=None, is_extendable=False, diff --git a/synthtool/sources/git.py b/synthtool/sources/git.py index c9406a894..14db9a06a 100644 --- a/synthtool/sources/git.py +++ b/synthtool/sources/git.py @@ -72,7 +72,10 @@ def _local_default_branch(path: pathlib.Path) -> Union[str, None]: def clone( - url: str, dest: pathlib.Path = None, committish: str = None, force: bool = False, + url: str, + dest: pathlib.Path = None, + committish: str = None, + force: bool = False, ) -> pathlib.Path: """Clones a remote git repo. diff --git a/synthtool/transforms.py b/synthtool/transforms.py index 5c3170a75..c3c9dd81a 100644 --- a/synthtool/transforms.py +++ b/synthtool/transforms.py @@ -137,7 +137,8 @@ def _copy_dir_to_existing_dir( _merge_file(source_path, dest_path, merge) except Exception: logger.exception( - "_merge_file failed for %s, fall back to copy", source_path, + "_merge_file failed for %s, fall back to copy", + source_path, ) shutil.copy2(str(source_path), str(dest_path)) else: @@ -147,7 +148,9 @@ def _copy_dir_to_existing_dir( return copied -def dont_overwrite(patterns: ListOfPathsOrStrs,) -> Callable[[str, str, Path], str]: +def dont_overwrite( + patterns: ListOfPathsOrStrs, +) -> Callable[[str, str, Path], str]: """Returns a merge function that doesn't overwrite the specified files. Pass the return value to move() or copy() to avoid overwriting existing diff --git a/tests/test_php.py b/tests/test_php.py index 280383753..d3b384374 100644 --- a/tests/test_php.py +++ b/tests/test_php.py @@ -51,8 +51,7 @@ def docker_image(): @pytest.fixture(scope="session") def hybrid_tmp_path(): - """A tmp dir implementation both for local run and on Kokoro. - """ + """A tmp dir implementation both for local run and on Kokoro.""" # Trampoline mount KOKORO_ROOT at the same path. # So we can mount files under there with docker in docker. hybrid_dir = os.environ.get("KOKORO_ROOT", None) @@ -65,8 +64,7 @@ def hybrid_tmp_path(): @pytest.fixture(scope="function", params=["php_asset"]) def copy_fixture(request, hybrid_tmp_path): - """A fixture for preparing test data. - """ + """A fixture for preparing test data.""" param = request.param test_dir = Path(f"{hybrid_tmp_path}/{param}") diff --git a/tests/test_python_library.py b/tests/test_python_library.py index e91a5ceaf..de17c8be0 100644 --- a/tests/test_python_library.py +++ b/tests/test_python_library.py @@ -137,7 +137,10 @@ ) def test_library_noxfile(template_kwargs, expected_text): t = templates.Templates(PYTHON_LIBRARY) - result = t.render("noxfile.py.j2", **template_kwargs,).read_text() + result = t.render( + "noxfile.py.j2", + **template_kwargs, + ).read_text() # Validate Python syntax. result_code = compile(result, "noxfile.py", "exec") assert result_code is not None