diff --git a/.config/requirements-dev.txt b/.config/requirements-dev.txt index 07e745c..0dba487 100644 --- a/.config/requirements-dev.txt +++ b/.config/requirements-dev.txt @@ -17,6 +17,7 @@ gitpython ifaddr jake liccheck==0.4.3 +navio-builder-win pebble==4.5.0 pip-check==2.6 psutil==5.6.7 diff --git a/Pipfile b/Pipfile index 3cf927c..5b515f5 100644 --- a/Pipfile +++ b/Pipfile @@ -4,6 +4,7 @@ url = "https://pypi.org/simple" verify_ssl = true [dev-packages] +navio-builder-win = "*" pytest = "==6.0.1" pytest-cov = ">=2.10.1" pytest-timeout = "*" diff --git a/navio/LICENSE.txt b/navio/LICENSE.txt deleted file mode 100644 index 5d33074..0000000 --- a/navio/LICENSE.txt +++ /dev/null @@ -1,19 +0,0 @@ -Copyright (C) 2012 Raghunandan Rao -Copyright (C) 2012 Calum J. Eadie - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in the -Software without restriction, including without limitation the rights to use, copy, -modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -and to permit persons to whom the Software is furnished to do so, subject to the -following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, -INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT -HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE -SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/navio/__init__.py b/navio/__init__.py deleted file mode 100644 index b36383a..0000000 --- a/navio/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from pkgutil import extend_path - -__path__ = extend_path(__path__, __name__) diff --git a/navio/__main__.py b/navio/__main__.py deleted file mode 100644 index 2cf9335..0000000 --- a/navio/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -import navio.builder - -if __name__ == "__main__": - navio.builder.main() diff --git a/navio/builder/__init__.py b/navio/builder/__init__.py deleted file mode 100644 index 94c8e30..0000000 --- a/navio/builder/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -""" -Lightweight Python Build Tool -""" - -from ._nb import task -from ._nb import add_env -from ._nb import main # , nsh -from ._nb import dump, pushd, zipdir - -# import sh -import pkgutil - -__path__ = pkgutil.extend_path(__path__, __name__) - -__all__ = [ - "task", - "main", - # 'nsh', 'sh', - "zipdir", - "add_env", - "dump", - "dumps", - "pushd", - "print_out", - "print_err", -] diff --git a/navio/builder/_nb.py b/navio/builder/_nb.py deleted file mode 100644 index e8e36be..0000000 --- a/navio/builder/_nb.py +++ /dev/null @@ -1,455 +0,0 @@ -""" -Lightweight Python Build Tool - -""" - -from os import path -from os import chdir, getcwd -from os.path import realpath -import inspect -import argparse -import logging -import os -import re -import imp -import sys -import time - -# import sh -import json -import zipfile -from datetime import datetime, date -from navio.meta_builder import __version__ - -_CREDIT_LINE = "Powered by nb %s " "- A Lightweight Python Build Tool." % __version__ -_LOGGING_FORMAT = "[ %(name)s - %(message)s ]" -_TASK_PATTERN = re.compile(r"^([^\[]+)(\[([^\]]*)\])?$") -# "^([^\[]+)(\[([^\],=]*(,[^\],=]+)*(,[^\],=]+=[^\],=]+)*)\])?$" - - -def build(args): - """ - Build the specified module with specified arguments. - - @type module: module - @type args: list of arguments - """ - # Build the command line. - parser = _create_parser() - - # No args passed. - # if not args: #todo: execute default task. - # parser.print_help() - # print("\n\n"+_CREDIT_LINE) - # exit - # Parse arguments. - args = parser.parse_args(args) - - if args.version: - print("nb %s" % __version__) - sys.exit(0) - - # load build file as a module - if not path.isfile(args.file): - print( - "Build file '%s' does not exist. " - "Please specify a build file\n" % args.file - ) - parser.print_help() - sys.exit(1) - - module = imp.load_source(path.splitext(path.basename(args.file))[0], args.file) - - # Run task and all its dependencies. - if args.list_tasks: - print_tasks(module, args.file) - elif not args.tasks: - if not _run_default_task(module): - parser.print_help() - print("\n") - print_tasks(module, args.file) - else: - _run_from_task_names(module, args.tasks) - - -def print_tasks(module, file): - # Get all tasks. - tasks = _get_tasks(module) - - # Build task_list to describe the tasks. - task_list = "Tasks in build file %s:" % file - name_width = _get_max_name_length(module) + 4 - task_help_format = "\n {0:<%s} {1: ^10} {2}" % name_width - default = _get_default_task(module) - for task in sorted(tasks, key=lambda task: task.name): - attributes = [] - if task.ignored: - attributes.append("Ignored") - if default and task.name == default.name: - attributes.append("Default") - - joined_attributes = ", ".join(attributes) - task_list += task_help_format.format( - task.name, - (f"[{joined_attributes}]") if attributes else "", - task.doc, - ) - print(task_list + "\n\n" + _CREDIT_LINE) - - -def _get_default_task(module): - matching_tasks = [ - task - for name, task in inspect.getmembers(module, Task.is_task) - if name == "__DEFAULT__" - ] - if matching_tasks: - return matching_tasks[0] - - -def _run_default_task(module): - default_task = _get_default_task(module) - if not default_task: - return False - _run(module, _get_logger(module), default_task, set()) - return True - - -def _run_from_task_names(module, task_names): - """ - @type module: module - @type task_name: string - @param task_name: Task name, exactly corresponds to function name. - """ - # Create logger. - logger = _get_logger(module) - all_tasks = _get_tasks(module) - completed_tasks = set() - for task_name in task_names: - task, args, kwargs = _get_task(module, task_name, all_tasks) - _run(module, logger, task, completed_tasks, True, args, kwargs) - - -def _get_task(module, name, tasks): - # Get all tasks. - match = _TASK_PATTERN.match(name) - if not match: - raise Exception("Invalid task argument %s" % name) - task_name, _, args_str = match.groups() - - args, kwargs = _parse_args(args_str) - if hasattr(module, task_name): - return getattr(module, task_name), args, kwargs - matching_tasks = [task for task in tasks if task.name.startswith(task_name)] - - if not matching_tasks: - raise Exception( - "Invalid task '%s'. Task should be one of %s" - % (name, ", ".join([task.name for task in tasks])) - ) - if len(matching_tasks) == 1: - return matching_tasks[0], args, kwargs - raise Exception( - "Conflicting matches %s for task %s" - % (", ".join([task.name for task in matching_tasks]), task_name) - ) - - -def _parse_args(args_str): - args = [] - kwargs = {} - if not args_str: - return args, kwargs - arg_parts = args_str.split(",") - - for i, part in enumerate(arg_parts): - if "=" in part: - key, value = [_str.strip() for _str in part.split("=")] - if key in kwargs: - raise Exception("duplicate keyword argument %s" % part) - kwargs[key] = value - else: - if len(kwargs) > 0: - raise Exception( - "Non keyword arg %s " - "cannot follows a keyword arg %s" % (part, arg_parts[i - 1]) - ) - args.append(part.strip()) - return args, kwargs - - -def _run( - module, - logger, - task, - completed_tasks, - from_command_line=False, - args=None, - kwargs=None, -): - """ - @type module: module - @type logging: Logger - @type task: Task - @type completed_tasts: set Task - @rtype: set Task - @return: Updated set of completed tasks after satisfying all dependencies. - """ - # Satsify dependencies recursively. Maintain set of completed tasks so each - # task is only performed once. - for dependency in task.dependencies: - completed_tasks = _run(module, logger, dependency, completed_tasks) - - # Perform current task, if need to. - if from_command_line or task not in completed_tasks: - - if task.ignored: - - logger.info('Ignoring task "%s"' % task.name) - - else: - - logger.info('Starting task "{}{}"'.format(task.name, str(args or []))) - - try: - # Run task. - startTime = int(round(time.time() * 1000)) - task(*(args or []), **(kwargs or {})) - stopTime = int(round(time.time() * 1000)) - except Exception: - stopTime = int(round(time.time() * 1000)) - logger.critical( - 'Error in task "%s". Time: %s sec' - % (task.name, (float(stopTime) - startTime) / 1000) - ) - logger.critical( - "Aborting build for %s" % os.path.abspath(module.__file__) - ) - raise - - logger.info( - 'Completed task "%s". Time: %s sec' - % (task.name, (float(stopTime) - startTime) / 1000) - ) - - completed_tasks.add(task) - - return completed_tasks - - -def _create_parser(): - """ - @rtype: argparse.ArgumentParser - """ - parser = argparse.ArgumentParser() - parser.add_argument( - "tasks", - help="perform specified task and all its dependencies", - metavar="task", - nargs="*", - ) - - parser.add_argument( - "-l", "--list-tasks", help="List the tasks", action="store_true" - ) - - parser.add_argument( - "-v", "--version", help="Display the version information", action="store_true" - ) - - parser.add_argument( - "-f", - "--file", - help=( - "Build file to read the tasks from. " - "'build.py' is default value assumed " - "if this argument is unspecified" - ), - metavar="file", - default="build.py", - ) - - return parser - - -# Abbreviate for convenience. -# task = _TaskDecorator - - -def task(*dependencies, **options): - for i, dependency in enumerate(dependencies): - if not Task.is_task(dependency): - if inspect.isfunction(dependency): - # Throw error specific to the most likely form of misuse. - if i == 0: - raise Exception("Replace use of @task with @task().") - else: - raise Exception( - "%s is not a task. " - "Each dependency should be a task." % dependency - ) - else: - raise Exception("%s is not a task." % dependency) - - def decorator(fn): - return Task(fn, dependencies, options) - - return decorator - - -class Task: - def __init__(self, func, dependencies, options): - """ - @type func: 0-ary function - @type dependencies: list of Task objects - """ - self.func = func - self.name = func.__name__ - self.doc = inspect.getdoc(func) or "" - self.dependencies = dependencies - self.ignored = bool(options.get("ignore", False)) - - def __call__(self, *args, **kwargs): - self.func.__call__(*args, **kwargs) - - @classmethod - def is_task(cls, obj): - """ - Returns true is an object is a build task. - """ - return isinstance(obj, cls) - - -def _get_tasks(module): - """ - Returns all functions marked as tasks. - - @type module: module - """ - # Get all functions that are marked as task and pull out the task object - # from each (name,value) pair. - return {member[1] for member in inspect.getmembers(module, Task.is_task)} - - -def _get_max_name_length(module): - """ - Returns the length of the longest task name. - - @type module: module - """ - return max([len(task.name) for task in _get_tasks(module)]) - - -def _get_logger(module): - """ - @type module: module - @rtype: logging.Logger - """ - - # Create Logger - logger = logging.getLogger(os.path.basename(module.__file__)) - logger.setLevel(logging.DEBUG) - - # Create console handler and set level to debug - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) - - # Create formatter - formatter = logging.Formatter(_LOGGING_FORMAT) - - # Add formatter to ch - ch.setFormatter(formatter) - - # Add ch to logger - logger.addHandler(ch) - - return logger - - -def main(): - build(sys.argv[1:]) - - -def json_serial(obj): - """JSON serializer for objects not serializable by default json code""" - - if isinstance(obj, (datetime, date)): - return obj.isoformat() - raise TypeError("Type %s not serializable" % type(obj)) - - -def dump(obj): - print("DUMP: {}".format(json.dumps(obj, indent=1, default=json_serial))) - - -def dumps(obj): - return json.dumps(obj, indent=1, default=json_serial) - - -def print_out(line): - sys.stdout.write(line) - sys.stdout.write("\n") - sys.stdout.flush() - - -def print_err(line): - sys.stderr.write(line) - sys.stderr.write("\n") - sys.stderr.flush() - - -def zipdir(zip_file, *paths): - import zipfile - - zipf = zipfile.ZipFile(zip_file, "w", zipfile.ZIP_DEFLATED) - for i, path in enumerate(paths): - if os.path.isfile(path): - print(f"Adding file: {path}") - zipf.write(path) - elif os.path.isdir(path): - for root, dirs, files in os.walk(path): - for file in files: - zipf.write(os.path.join(root, file)) - print("Adding file: {}".format(os.path.join(root, file))) - zipf.close() - - -class PushdContext: - cwd = None - original_dir = None - - def __init__(self, dirname): - self.cwd = realpath(dirname) - - def __enter__(self): - self.original_dir = getcwd() - chdir(self.cwd) - return self - - def __exit__(self, type, value, tb): - chdir(self.original_dir) - - -def pushd(dirname): - return PushdContext(dirname) - - -def add_env(*envs): - new_env = os.environ.copy() - for env in envs: - new_env.update(env) - return new_env - - -# @contextlib.contextmanager -# def pushd(path): -# starting_directory = os.getcwd() -# try: -# os.chdir(path) -# yield -# finally: -# os.chdir(starting_directory) - - -# Navio shell overriden call -# nsh = sh(_out=sys.stdout, _err_to_out=True) diff --git a/navio/builder/tests/__init__.py b/navio/builder/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/navio/builder/tests/build_scripts/__init__.py b/navio/builder/tests/build_scripts/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/navio/builder/tests/build_scripts/annotation_misuse_1.py b/navio/builder/tests/build_scripts/annotation_misuse_1.py deleted file mode 100644 index 3196b7d..0000000 --- a/navio/builder/tests/build_scripts/annotation_misuse_1.py +++ /dev/null @@ -1,8 +0,0 @@ -from navio.builder import task - -# Uses @_nb.task form instead of @_nb.task() form. - - -@task -def clean(): - pass diff --git a/navio/builder/tests/build_scripts/annotation_misuse_2.py b/navio/builder/tests/build_scripts/annotation_misuse_2.py deleted file mode 100644 index 2cc4192..0000000 --- a/navio/builder/tests/build_scripts/annotation_misuse_2.py +++ /dev/null @@ -1,21 +0,0 @@ -from navio.builder import task - - -@task() -def clean(): - pass - - -# Should be marked as task. - - -def html(): - pass - - -# References a non task. - - -@task(clean, html) -def android(): - pass diff --git a/navio/builder/tests/build_scripts/annotation_misuse_3.py b/navio/builder/tests/build_scripts/annotation_misuse_3.py deleted file mode 100644 index 85026b4..0000000 --- a/navio/builder/tests/build_scripts/annotation_misuse_3.py +++ /dev/null @@ -1,14 +0,0 @@ -from navio.builder import task - - -@task() -def clean(): - pass - - -# Referring to clean by name rather than reference. - - -@task(1234) -def html(): - pass diff --git a/navio/builder/tests/build_scripts/build_with_params.py b/navio/builder/tests/build_scripts/build_with_params.py deleted file mode 100644 index 5855663..0000000 --- a/navio/builder/tests/build_scripts/build_with_params.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/python - -from navio.builder import task - -tasks_run = [] - - -@task() -def clean(directory="/tmp"): - tasks_run.append("clean[%s]" % directory) - - -@task(clean) -def html(): - tasks_run.append("html") - - -@task() -def tests(*test_names): - tasks_run.append("tests[%s]" % ",".join(test_names)) - - -@task(clean) -def copy_file(from_, to, fail_on_error="True"): - tasks_run.append(f"copy_file[{from_},{to},{fail_on_error}]") - - -@task(clean) -def start_server(port="80", debug="True"): - tasks_run.append(f"start_server[{port},{debug}]") - - -@task(ignore=True) -def ignored(file, contents): - tasks_run.append(f"append_to_file[{file},{contents}]") - - -@task(clean, ignored) -def append_to_file(file, contents): - tasks_run.append(f"append_to_file[{file},{contents}]") - - -@task(ignored) -def echo(*args, **kwargs): - args_str = [] - if args: - args_str.append(",".join(args)) - if kwargs: - args_str.append( - ",".join("{}={}".format(kw, kwargs[kw]) for kw in sorted(kwargs)) - ) - - tasks_run.append("echo[%s]" % ",".join(args_str)) diff --git a/navio/builder/tests/build_scripts/default_task_and_import_dependencies.py b/navio/builder/tests/build_scripts/default_task_and_import_dependencies.py deleted file mode 100644 index 6de9c50..0000000 --- a/navio/builder/tests/build_scripts/default_task_and_import_dependencies.py +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/python - -from navio.builder import task - -from navio.builder.tests.build_scripts.simple import * -from navio.builder.tests.build_scripts import build_with_params - -tasks_run = [] - - -@task() -def local_task(): - tasks_run.append("local_task") - - -@task(clean, build_with_params.html, local_task) -def task_with_imported_dependencies(): - tasks_run.append("task_with_imported_dependencies") - - -__DEFAULT__ = task_with_imported_dependencies diff --git a/navio/builder/tests/build_scripts/dependencies.py b/navio/builder/tests/build_scripts/dependencies.py deleted file mode 100644 index cbaa9fc..0000000 --- a/navio/builder/tests/build_scripts/dependencies.py +++ /dev/null @@ -1,42 +0,0 @@ -from navio.builder import task - - -@task() -def clean(): - """Clean build directory.""" - - print("clean") - - -@task(clean) -def html(): - """Generate HTML.""" - - print("html") - - -@task(clean) -def images(): - """Prepare images.""" - - print("images") - - -@task(clean, html, images) -def android(): - """Package Android app.""" - - print("android") - - -@task(clean, html, images) -def ios(): - """Package iOS app.""" - - print("ios") - - -def some_utility_method(): - """Some utility method.""" - - print("some utility method") diff --git a/navio/builder/tests/build_scripts/options.py b/navio/builder/tests/build_scripts/options.py deleted file mode 100644 index 073d871..0000000 --- a/navio/builder/tests/build_scripts/options.py +++ /dev/null @@ -1,30 +0,0 @@ -from navio.builder import task - -tasks_run = [] - - -@task() -def clean(): - tasks_run.append("clean") - - -@task(clean) -def html(): - "Generate HTML." - tasks_run.append("html") - - -@task(clean, ignore=True) -def images(): - """Prepare images. - - Should be ignored.""" - - raise Exception("This task should have been ignored.") - - -@task(clean, html, images) -def android(): - "Package Android app." - - tasks_run.append("android") diff --git a/navio/builder/tests/build_scripts/runtime_error.py b/navio/builder/tests/build_scripts/runtime_error.py deleted file mode 100644 index 0b95fc1..0000000 --- a/navio/builder/tests/build_scripts/runtime_error.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -Build script with a runtime error. -""" -from navio.builder import task - - -@task() -def images(): - """Prepare images. Raises IOError.""" - global ran_images - ran_images = True - raise OSError - - -@task(images) -def android(): - """Package Android app.""" - global ran_android - print("android") - ran_android = True diff --git a/navio/builder/tests/build_scripts/simple.py b/navio/builder/tests/build_scripts/simple.py deleted file mode 100644 index 62bbf9f..0000000 --- a/navio/builder/tests/build_scripts/simple.py +++ /dev/null @@ -1,47 +0,0 @@ -#!/usr/bin/python - -from navio.builder import task - - -@task() -def clean(): - """Clean build directory.""" - - print("clean") - - -@task() -def html(): - """Generate HTML.""" - - print("html") - - -@task() -def images(): - """Prepare images.""" - - print("images") - - -@task() -def android(): - """Package Android app.""" - - print("android") - - -@task() -def ios(): - """Package iOS app.""" - - print("ios") - - -def some_utility_method(): - """Some utility method.""" - - print("some utility method") - - -__DEFAULT__ = ios diff --git a/navio/builder/tests/test_nb.py b/navio/builder/tests/test_nb.py deleted file mode 100644 index aaad230..0000000 --- a/navio/builder/tests/test_nb.py +++ /dev/null @@ -1,354 +0,0 @@ -import pytest -import re -from navio.builder import _nb, main -import sys -import os -from os import path -import imp -import contextlib - -if sys.version.startswith("3."): - from io import StringIO as SOut -else: - from StringIO import StringIO as SOut - - -def fpath(mod): - return path.splitext(mod.__file__)[0] + ".py" - - -def simulate_dynamic_module_load(mod): - file_path = fpath(mod) - dynamically_loaded_mod = imp.load_source( - path.splitext(path.basename(file_path))[0], file_path - ) - return dynamically_loaded_mod - - -def reset_build_file(mod): - mod.tasks_run = [] - - -def build(mod, params=None, init_mod=reset_build_file): - dynamically_loaded_mod = simulate_dynamic_module_load(mod) - dynamically_loaded_mod.tasks_run = [] - sys.argv = ["nb", "-f", fpath(mod)] + (params or []) - main() - return dynamically_loaded_mod - - -class TestParseArgs: - def test_parsing_commandline(self): - args = _nb._create_parser().parse_args(["-f", "foo.py", "task1", "task2"]) - assert "foo.py" == args.file - assert not args.list_tasks - assert ["task1", "task2"] == args.tasks - - def test_parsing_commandline_help(self): - assert _nb._create_parser().parse_args(["-l"]).list_tasks - assert _nb._create_parser().parse_args(["--list-tasks"]).list_tasks - - def test_parsing_commandline_build_file(self): - assert "some_file" == _nb._create_parser().parse_args(["-f", "some_file"]).file - assert "build.py" == _nb._create_parser().parse_args([]).file - assert ( - "/foo/bar" == _nb._create_parser().parse_args(["--file", "/foo/bar"]).file - ) - - with pytest.raises(SystemExit): - _nb._create_parser().parse_args(["--file"]) - with pytest.raises(SystemExit): - _nb._create_parser().parse_args(["-f"]) - - -class TestImport: - def test_import(self): - import navio.builder - from navio.builder import nsh - from navio.builder import sh - from navio.builder import dump - - -class TestBuildSimple: - def test_get_tasks(self): - from .build_scripts import simple - - ts = _nb._get_tasks(simple) - assert len(ts) == 5 - - -class TestBuildWithDependencies: - def test_get_tasks(self): - from .build_scripts import dependencies - - tasks = _nb._get_tasks(dependencies) - assert len(tasks) == 5 - assert 3 == len( - [task for task in tasks if task.name == "android"][0].dependencies - ) - assert 3 == len([task for task in tasks if task.name == "ios"][0].dependencies) - - def test_dependencies_for_imported(self): - from .build_scripts import default_task_and_import_dependencies - - tasks = _nb._get_tasks(default_task_and_import_dependencies) - assert 7 == len(tasks) - assert [task for task in tasks if task.name == "clean"] - assert [task for task in tasks if task.name == "local_task"] - assert [task for task in tasks if task.name == "android"] - assert 3 == len( - [task for task in tasks if task.name == "task_with_imported_dependencies"][ - 0 - ].dependencies - ) - - def test_dependencies_that_are_imported_e2e(self): - from .build_scripts import default_task_and_import_dependencies - - def mod_init(mod): - mod.tasks_run = [] - mod.build_with_params.tasks_run = [] - - module = build( - default_task_and_import_dependencies, - ["task_with_imported_dependencies"], - init_mod=mod_init, - ) - assert module.tasks_run == ["local_task", "task_with_imported_dependencies"] - assert module.build_with_params.tasks_run == ["clean[/tmp]", "html"] - - -class TestDecorationValidation: - def test_task_without_braces(self): - with pytest.raises(Exception) as exc: - from .build_scripts import annotation_misuse_1 - assert "Replace use of @task with @task()." in str(exc.value) - - def test_dependency_not_a_task(self): - with pytest.raises(Exception) as exc: - from .build_scripts import annotation_misuse_2 - assert re.findall("function html.* is not a task.", str(exc.value)) - - def test_dependency_not_a_function(self): - with pytest.raises(Exception) as exc: - from .build_scripts import annotation_misuse_3 - assert "1234 is not a task." in str(exc.value) - - -@contextlib.contextmanager -def mock_stdout(): - oldout, olderr = sys.stdout, sys.stderr - try: - out = [SOut(), SOut()] - sys.stdout, sys.stderr = out - yield out - finally: - sys.stdout, sys.stderr = oldout, olderr - out[0] = out[0].getvalue() - out[1] = out[1].getvalue() - - -class TestOptions: - @pytest.fixture - def module(self): - from .build_scripts import options as module - - self.docs = { - "clean": "", - "html": "Generate HTML.", - "images": """Prepare images.\n\nShould be ignored.""", - "android": "Package Android app.", - } - return module - - def test_ignore_tasks(self, module): - module = build(module, ["android"]) - assert ["clean", "html", "android"] == module.tasks_run - - def test_docs(self, module): - tasks = _nb._get_tasks(module) - assert 4 == len(tasks) - - for task_ in tasks: - assert task_.name in self.docs - assert self.docs[task_.name] == task_.doc - - @pytest.mark.parametrize("args", [["-l"], ["--list-tasks"], []]) - def test_list_docs(self, module, args): - with mock_stdout() as out: - build(module, args) - stdout = out[0] - tasks = _nb._get_tasks(module) - for task in tasks: - if task.ignored: - assert re.findall( - r"{}\s+{}\s+{}".format(task.name, r"\[Ignored\]", task.doc), stdout - ) - else: - assert re.findall(fr"{task.name}\s+{task.doc}", stdout) - - -class TestRuntimeError: - def test_stop_on_exception(self): - from .build_scripts import runtime_error as re - - with pytest.raises(IOError): - build(re, ["android"]) - mod = simulate_dynamic_module_load(re) - assert mod.ran_images - assert not hasattr(mod, "ran_android") - - def test_exception_on_invalid_task_name(self): - from .build_scripts import build_with_params - - with pytest.raises(Exception) as exc: - build(build_with_params, ["doesnt_exist"]) - - assert ( - "task should be one of append_to_file, clean" - ", copy_file, echo, html, start_server, tests" in str(exc.value) - ) - - -class TestPartialTaskNames: - def setup_method(self, method): - from .build_scripts import build_with_params - - self._mod = build_with_params - - def test_with_partial_name(self): - mod = build(self._mod, ["cl"]) - assert ["clean[/tmp]"] == mod.tasks_run - - def test_with_partial_name_and_dependencies(self): - mod = build(self._mod, ["htm"]) - assert ["clean[/tmp]", "html"] == mod.tasks_run - - def test_exception_on_conflicting_partial_names(self): - with pytest.raises(Exception) as exc: - build(self._mod, ["c"]) - assert "Conflicting matches clean, copy_file for task c" in str( - exc.value - ) or "Conflicting matches copy_file, clean for task c" in str(exc.value) - - -class TestDefaultTask: - def test_simple_default_task(self): - from .build_scripts import simple - - # returns false if no default task - assert _nb._run_default_task(simple) - - def test_mod_with_defaults_which_imports_other_files_with_defaults(self): - from .build_scripts import default_task_and_import_dependencies - - mod = build(default_task_and_import_dependencies) - assert "task_with_imported_dependencies" in mod.tasks_run - - -class TestMultipleTasks: - def setup_method(self, method): - from .build_scripts import build_with_params - - self._mod = build_with_params - - def test_dependency_is_run_only_once_unless_explicitly_invoked_again(self): - mod = build(self._mod, ["clean", "html", "tests", "clean"]) - assert ["clean[/tmp]", "html", "tests[]", "clean[/tmp]"] == mod.tasks_run - - def test_multiple_partial_names(self): - assert ["clean[/tmp]", "html"] == build(self._mod, ["cl", "htm"]).tasks_run - - -class TesttaskArguments: - def setup_method(self, method): - from .build_scripts import build_with_params - - self._mod = build_with_params - self._mod.tasks_run = [] - - def test_passing_optional_params_with_dependencies(self): - mod = build( - self._mod, - [ - "clean[~/project/foo]", - "append_to_file[/foo/bar,ABCDEF]", - "copy_file[/foo/bar,/foo/blah,False]", - "start_server[8080]", - ], - ) - assert [ - "clean[~/project/foo]", - "append_to_file[/foo/bar,ABCDEF]", - "copy_file[/foo/bar,/foo/blah,False]", - "start_server[8080,True]", - ] == mod.tasks_run - - def test_invoking_varargs_task(self): - mod = build(self._mod, ["tests[test1,test2,test3]"]) - assert ["tests[test1,test2,test3]"] == mod.tasks_run - - def test_partial_name_with_args(self): - - mod = build(self._mod, ["co[foo,bar]", "star"]) - assert [ - "clean[/tmp]", - "copy_file[foo,bar,True]", - "start_server[80,True]", - ] == mod.tasks_run - - def test_passing_keyword_args(self): - mod = build( - self._mod, - ["co[to=bar,from_=foo]", "star[80,debug=False]", "echo[foo=bar,blah=123]"], - ) - - assert [ - "clean[/tmp]", - "copy_file[foo,bar,True]", - "start_server[80,False]", - "echo[blah=123,foo=bar]", - ] == mod.tasks_run - - def test_passing_varargs_and_keyword_args(self): - assert ["echo[1,2,3,some_str,111=333,bar=123.3,foo=xyz]"] == build( - self._mod, ["echo[1,2,3,some_str,111=333,foo=xyz,bar=123.3]"] - ).tasks_run - - def test_validate_keyword_arguments_always_after_args(self): - with pytest.raises(Exception) as exc: - build(self._mod, ["echo[bar=123.3,foo]"]) - assert "Non keyword arg foo cannot follows" " a keyword arg bar=123.3" in str( - exc.value - ) - - with pytest.raises(Exception) as exc: - build(self._mod, ["copy[from_=/foo,/foo1]"]) - - assert ( - "Non keyword arg /foo1 cannot follows" - " a keyword arg from_=/foo" in str(exc.value) - ) - - def test_invalid_number_of_args(self): - with pytest.raises(TypeError) as exc: - build(self._mod, ["append[1,2,3]"]) - print(str(exc.value)) - assert re.findall("takes .*2 .*arguments", str(exc.value)) - - def test_invalid_names_for_kwargs(self): - with pytest.raises(TypeError) as exc: - build(self._mod, ["copy[1=2,to=bar]"]) - assert "got an unexpected keyword argument '1'" in str(exc.value) - - with pytest.raises(TypeError) as exc: - build(self._mod, ["copy[bar123=2]"]) - assert "got an unexpected keyword argument 'bar123'" in str(exc.value) - - -class TestPushd: - def test_pushd(self): - from navio.builder import pushd - - with pushd("."): - pass diff --git a/navio/meta_builder.py b/navio/meta_builder.py deleted file mode 100644 index e78806d..0000000 --- a/navio/meta_builder.py +++ /dev/null @@ -1,7 +0,0 @@ -__version__ = "0.1.51" -__license__ = "MIT License" -__website__ = "https://oss.navio.tech/navio-builder/" -__download_url__ = ( - "https://github.com/naviotech/navio-builder/archive/" - "{}.tar.gz".format(__version__) -) diff --git a/poetry.lock b/poetry.lock index 0e5ec8a..05a7da5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -751,6 +751,17 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "navio-builder-win" +version = "0.1.52" +description = "navio-builder, but doesn't import sh and works on windows." +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +sh = "*" + [[package]] name = "nbformat" version = "5.1.2" @@ -1357,6 +1368,14 @@ category = "dev" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +[[package]] +name = "sh" +version = "1.14.1" +description = "Python subprocess replacement" +category = "main" +optional = false +python-versions = "*" + [[package]] name = "six" version = "1.15.0" @@ -1655,7 +1674,7 @@ testing = ["pytest (>=3.5,!=3.7.3)", "pytest-checkdocs (>=1.2.3)", "pytest-flake [metadata] lock-version = "1.1" python-versions = ">=3.6, <4.0" -content-hash = "4fb4ce3a4641a092ac0fe5d6182bea8c1dcd2dac7d84bb374789db3e89ca8fae" +content-hash = "607d136492c2901da8d34856344035ec951fd5d561f4ed95377e13dc19b2b91d" [metadata.files] 2to3 = [ @@ -2197,6 +2216,9 @@ mypy-extensions = [ {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, ] +navio-builder-win = [ + {file = "navio_builder_win-0.1.52-py2.py3-none-any.whl", hash = "sha256:2918520bb738aeae4ab55c988417a60d82981a82df3308721d5068e450c5e366"}, +] nbformat = [ {file = "nbformat-5.1.2-py3-none-any.whl", hash = "sha256:3949fdc8f5fa0b1afca16fb307546e78494fa7a7bceff880df8168eafda0e7ac"}, {file = "nbformat-5.1.2.tar.gz", hash = "sha256:1d223e64a18bfa7cdf2db2e9ba8a818312fc2a0701d2e910b58df66809385a56"}, @@ -2515,6 +2537,10 @@ semantic-version = [ {file = "semantic_version-2.8.5-py2.py3-none-any.whl", hash = "sha256:45e4b32ee9d6d70ba5f440ec8cc5221074c7f4b0e8918bdab748cc37912440a9"}, {file = "semantic_version-2.8.5.tar.gz", hash = "sha256:d2cb2de0558762934679b9a104e82eca7af448c9f4974d1f3eeccff651df8a54"}, ] +sh = [ + {file = "sh-1.14.1-py2.py3-none-any.whl", hash = "sha256:75e86a836f47de095d4531718fe8489e6f7446c75ddfa5596f632727b919ffae"}, + {file = "sh-1.14.1.tar.gz", hash = "sha256:39aa9af22f6558a0c5d132881cf43e34828ca03e4ae11114852ca6a55c7c1d8e"}, +] six = [ {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, diff --git a/pyproject.toml b/pyproject.toml index 65ddd40..4de8f98 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ slpp-23 = "*" # projects split out from so_pip random-names = "*" whats-that-code = ">=0.1.10" +navio-builder-win = "^0.1.52" [tool.poetry.dev-dependencies] pytest = "==6.0.1" diff --git a/scripts/mass_harvest.py b/scripts/mass_harvest.py new file mode 100644 index 0000000..0266206 --- /dev/null +++ b/scripts/mass_harvest.py @@ -0,0 +1,19 @@ +""" +Gather lots of answers, lots of questions, put all into one folder. + +Same as "all in one" workflow except more than one question. +""" +import so_pip.settings as settings +from so_pip.models.count_loc_in_post import post_has_code, count_loc +from so_pip.utils.files_utils import find_file + +settings.OUTPUT_FOLDER = find_file("../../output/unittest/", __file__) + +if __name__ == '__main__': + from so_pip.api_clients.stackapi_facade import get_json_by_advanced_search + for page in range(1, 5000): + results = get_json_by_advanced_search("",tagged=["python","boto"], page=page) + for result in results["items"]: + body = result["body"] + if post_has_code(result) and count_loc(result)>15: + print(result) diff --git a/scripts/abc.py b/scripts/ruby_things.py similarity index 100% rename from scripts/abc.py rename to scripts/ruby_things.py diff --git a/so_pip/api_clients/search_engine_facade.py b/so_pip/api_clients/search_engine_facade.py index 19b94bc..0c43629 100644 --- a/so_pip/api_clients/search_engine_facade.py +++ b/so_pip/api_clients/search_engine_facade.py @@ -3,17 +3,19 @@ from search_engine_parser.core.engines.stackoverflow import Search -def search_string(query:str)->Any: + +def search_string(query: str) -> Any: """ This is likely to fail unless you are on a brand new IP, and even then, it might fail. SO doesn't like screen scrapers. """ search_args = (query, 1) - engine= Search() + engine = Search() results = engine.search(*search_args) return results -if __name__ == '__main__': - results = search_string(query='[python] is:answer def class aws') + +if __name__ == "__main__": + results = search_string(query="[python] is:answer def class aws") for result in results: - pprint.pprint(result) + pprint.pprint(result) diff --git a/so_pip/api_clients/stackapi_facade.py b/so_pip/api_clients/stackapi_facade.py index eab4781..bd538cc 100644 --- a/so_pip/api_clients/stackapi_facade.py +++ b/so_pip/api_clients/stackapi_facade.py @@ -37,21 +37,28 @@ def get_json_by_search(query: str, tagged: Tuple[str, ...]) -> Dict[str, Any]: ), ) -def get_json_by_advanced_search(query: str, tagged: Tuple[str, ...]) -> Dict[str, Any]: + +def get_json_by_advanced_search( + query: str, tagged: Tuple[str, ...], page: int, minimal: bool = True +) -> Dict[str, Any]: """Low level access, returns unprocessed json example: /2.2/search/advanced?order=desc&sort=activity&answers=1&body=def&tagged=python&site=stackoverflow """ + pager = StackAPI("stackoverflow", key=os.environ["key"], max_pages=5000) + # filter = "&filter=!)5IW-1CBJh7IUcXv2R9eY(KE__tA" if minimal else "" return cast( Dict[str, Any], - SITE.fetch( - "search/advanced?order={desc}&sort={sort}&answers={answers}&body={body}&tagged={tagged}&site={site}", - sort="votes", - order="desc", - answers=1, - body=query, + pager.fetch( + "search/advanced?order=desc&sort={sort}&answers={answers}&body={body}&tagged={tagged}", + page=page, + sort=["votes"], + # order=["desc"], + answers=[1], + body=[query], tagged=[";".join(tagged)], - site="stackoverflow" + filter="!BHMIb2uw8ZCNzk.BY)VCLpavh_59fq" if minimal else None + # site="stackoverflow" ), ) diff --git a/so_pip/commands/search.py b/so_pip/commands/search.py index 37d9bda..a4c59e4 100644 --- a/so_pip/commands/search.py +++ b/so_pip/commands/search.py @@ -36,6 +36,7 @@ def import_so_search( all_results = [] found = 0 + inform(f"Found {len(possibles['items'])} possible answers") for possible in possibles["items"]: result = import_so_question( package_prefix, diff --git a/so_pip/parse_python/post_to_code.py b/so_pip/parse_python/post_to_code.py index edccc4a..5c8b8a4 100644 --- a/so_pip/parse_python/post_to_code.py +++ b/so_pip/parse_python/post_to_code.py @@ -94,7 +94,12 @@ def handle_post( else: post = question - if count_loc(post) < minimum_loc: + loc = count_loc(post) + if loc < minimum_loc: + inform( + f"Answer lacks minimum lines of code, " + f"{loc} vs {minimum_loc} ...skipping" + ) continue if ( diff --git a/test/test_slow/test_live_search.py b/test/test_slow/test_live_search.py index 30c76d7..45befa2 100644 --- a/test/test_slow/test_live_search.py +++ b/test/test_slow/test_live_search.py @@ -9,13 +9,15 @@ # Only "intitle" so can't search if body has def/class! (i.e. basic units of re-usability) def test_import_so_search(): + import so_pip.settings as settings + settings.QUIET = False result = import_so_search( package_prefix="test", query="aws", tags=["python"], output_folder=settings.OUTPUT_FOLDER, stop_after=2, - minimum_loc=15 + minimum_loc=5 ) print(result) assert result