From bce8c90f44cbbfb6a09191bb2bbdc557be2ed309 Mon Sep 17 00:00:00 2001 From: Stefan Tatschner Date: Thu, 27 Oct 2022 10:38:19 +0200 Subject: [PATCH] feat: Add progressbar support `gallia scan uds identifiers` is the first scanner with a progressbar. --- poetry.lock | 57 ++++++++++++++++----- pyproject.toml | 2 + src/gallia/cli.py | 1 + src/gallia/command.py | 12 ++++- src/gallia/commands/scan/uds/identifiers.py | 17 +++++- src/gallia/log.py | 8 ++- 6 files changed, 81 insertions(+), 16 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3740ef0af..096a34a9c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -102,7 +102,7 @@ python-versions = ">=3.5" dev = ["cloudpickle", "coverage[toml] (>=5.0.2)", "furo", "hypothesis", "mypy (>=0.900,!=0.940)", "pre-commit", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "sphinx", "sphinx-notfound-page", "zope.interface"] docs = ["furo", "sphinx", "sphinx-notfound-page", "zope.interface"] tests = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins", "zope.interface"] -tests_no_zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] +tests-no-zope = ["cloudpickle", "coverage[toml] (>=5.0.2)", "hypothesis", "mypy (>=0.900,!=0.940)", "pympler", "pytest (>=4.3.0)", "pytest-mypy-plugins"] [[package]] name = "Babel" @@ -191,7 +191,7 @@ optional = false python-versions = ">=3.6.0" [package.extras] -unicode_backport = ["unicodedata2"] +unicode-backport = ["unicodedata2"] [[package]] name = "click" @@ -208,7 +208,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "dev" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" @@ -333,9 +333,9 @@ python-versions = ">=3.6.1,<4.0" [package.extras] colors = ["colorama (>=0.4.3,<0.5.0)"] -pipfile_deprecated_finder = ["pipreqs", "requirementslib"] +pipfile-deprecated-finder = ["pipreqs", "requirementslib"] plugins = ["setuptools"] -requirements_deprecated_finder = ["pip-api", "pipreqs"] +requirements-deprecated-finder = ["pip-api", "pipreqs"] [[package]] name = "jedi" @@ -413,7 +413,7 @@ mdurl = ">=0.1,<1.0" [package.extras] benchmarking = ["psutil", "pytest", "pytest-benchmark (>=3.2,<4.0)"] -code_style = ["pre-commit (==2.6)"] +code-style = ["pre-commit (==2.6)"] compare = ["commonmark (>=0.9.1,<0.10.0)", "markdown (>=3.3.6,<3.4.0)", "mistletoe (>=0.8.1,<0.9.0)", "mistune (>=2.0.2,<2.1.0)", "panflute (>=2.1.3,<2.2.0)"] linkify = ["linkify-it-py (>=1.0,<2.0)"] plugins = ["mdit-py-plugins"] @@ -449,7 +449,7 @@ python-versions = ">=3.7" markdown-it-py = ">=1.0.0,<3.0.0" [package.extras] -code_style = ["pre-commit"] +code-style = ["pre-commit"] rtd = ["attrs", "myst-parser (>=0.16.1,<0.17.0)", "sphinx-book-theme (>=0.1.0,<0.2.0)"] testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] @@ -529,7 +529,7 @@ sphinx = ">=4,<6" typing-extensions = "*" [package.extras] -code_style = ["pre-commit (>=2.12,<3.0)"] +code-style = ["pre-commit (>=2.12,<3.0)"] linkify = ["linkify-it-py (>=1.0,<2.0)"] rtd = ["ipython", "sphinx-book-theme", "sphinx-design", "sphinxcontrib.mermaid (>=0.7.1,<0.8.0)", "sphinxext-opengraph (>=0.6.3,<0.7.0)", "sphinxext-rediraffe (>=0.2.7,<0.3.0)"] testing = ["beautifulsoup4", "coverage[toml]", "pytest (>=6,<7)", "pytest-cov", "pytest-param-files (>=0.3.4,<0.4.0)", "pytest-regressions", "sphinx (<5.2)", "sphinx-pytest"] @@ -784,7 +784,7 @@ wrapt = ">=1.10,<2.0" [package.extras] canalystii = ["canalystii (>=0.1.0)"] cantact = ["cantact (>=0.0.7)"] -gs_usb = ["gs-usb (>=0.2.1)"] +gs-usb = ["gs-usb (>=0.2.1)"] neovi = ["filelock", "python-ics (>=2.12)"] nixnet = ["nixnet (>=0.3.1)"] pcan = ["uptime (>=3.0.1,<3.1.0)"] @@ -875,7 +875,7 @@ tomli = {version = ">=2.0", markers = "python_version < \"3.11\""} [package.extras] doc = ["sphinx (>=4.5.0)", "tabulate (>=0.8.9)"] -gen_docs = ["pytoolconfig[doc]", "sphinx (>=4.5.0)", "sphinx-autodoc-typehints (>=1.18.1)", "sphinx-rtd-theme (>=1.0.0)"] +gen-docs = ["pytoolconfig[doc]", "sphinx (>=4.5.0)", "sphinx-autodoc-typehints (>=1.18.1)", "sphinx-rtd-theme (>=1.0.0)"] global = ["appdirs (>=1.4.4)"] validation = ["pydantic (>=1.7.4)"] @@ -919,7 +919,7 @@ urllib3 = ">=1.21.1,<1.27" [package.extras] socks = ["PySocks (>=1.5.6,!=1.5.7)"] -use_chardet_on_py3 = ["chardet (>=3.0.2,<6)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] [[package]] name = "reuse" @@ -1126,6 +1126,23 @@ category = "dev" optional = false python-versions = ">=3.6,<4.0" +[[package]] +name = "tqdm" +version = "4.64.1" +description = "Fast, Extensible Progress Meter" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + [[package]] name = "types-aiofiles" version = "22.1.0" @@ -1142,6 +1159,14 @@ category = "dev" optional = false python-versions = "*" +[[package]] +name = "types-tqdm" +version = "4.64.7.1" +description = "Typing stubs for tqdm" +category = "dev" +optional = false +python-versions = "*" + [[package]] name = "typing-extensions" version = "4.4.0" @@ -1224,7 +1249,7 @@ cffi = ["cffi (>=1.11)"] [metadata] lock-version = "1.1" python-versions = ">=3.10,<3.12" -content-hash = "e5ba60636c4c251b397bc64e8d5c48fe64965da05d48a659150a9991a869f7b7" +content-hash = "055d98402f44b4de0779aac6c463cf83b6893cc5394f2f39d1047e2859e15c20" [metadata.files] aiofiles = [ @@ -2197,6 +2222,10 @@ tomlkit = [ {file = "tomlkit-0.11.5-py3-none-any.whl", hash = "sha256:f2ef9da9cef846ee027947dc99a45d6b68a63b0ebc21944649505bf2e8bc5fe7"}, {file = "tomlkit-0.11.5.tar.gz", hash = "sha256:571854ebbb5eac89abcb4a2e47d7ea27b89bf29e09c35395da6f03dd4ae23d1c"}, ] +tqdm = [ + {file = "tqdm-4.64.1-py2.py3-none-any.whl", hash = "sha256:6fee160d6ffcd1b1c68c65f14c829c22832bc401726335ce92c52d395944a6a1"}, + {file = "tqdm-4.64.1.tar.gz", hash = "sha256:5f4f682a004951c1b450bc753c710e9280c5746ce6ffedee253ddbcbf54cf1e4"}, +] types-aiofiles = [ {file = "types-aiofiles-22.1.0.tar.gz", hash = "sha256:8ae2e1d2f99b62754cffe6a8761fd6eeb8a0675569dc73818248e5bc30cb9074"}, {file = "types_aiofiles-22.1.0-py3-none-any.whl", hash = "sha256:460c3b1c7fa24ec5f2c7d46cf3a849f07c7007a57fc19884be17e5159351fc39"}, @@ -2205,6 +2234,10 @@ types-tabulate = [ {file = "types-tabulate-0.9.0.0.tar.gz", hash = "sha256:4a79474714cea156bcd2185bb9bddd8fb9d3d5227c8d0a7f21bf21caf5f60e67"}, {file = "types_tabulate-0.9.0.0-py3-none-any.whl", hash = "sha256:a1cc2aa52d2a9abfe0acbb361ccd49e3400794086a41626ce8784ed2e1ec0b0d"}, ] +types-tqdm = [ + {file = "types-tqdm-4.64.7.1.tar.gz", hash = "sha256:436b7996d83bac8e539a82ded72e02f0837693c2288d954d30ea610bcf4d1e13"}, + {file = "types_tqdm-4.64.7.1-py3-none-any.whl", hash = "sha256:0bd113b8084a8f3ea7fa6b49ec8fde5842892f1119844aef549772e5e302db67"}, +] typing-extensions = [ {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, diff --git a/pyproject.toml b/pyproject.toml index dafeec0d2..c12985452 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ tomli = { version = "^2.0.1", python = "<3.11" } msgspec = ">=0.8,<0.10" pydantic = "^1.10" pygit2 = "^1.10" +tqdm = "^4.64.1" [tool.poetry.group.dev.dependencies] black = "^22.10" @@ -59,6 +60,7 @@ sphinx-rtd-theme = "^1.0" reuse = "^1.0.0" construct-typing = "^0.5.2" pytest-cov = "^4.0" +types-tqdm = "^4.64.7.1" [tool.poetry.scripts] "gallia" = "gallia.cli:main" diff --git a/src/gallia/cli.py b/src/gallia/cli.py index b400885c4..9179b6883 100644 --- a/src/gallia/cli.py +++ b/src/gallia/cli.py @@ -330,6 +330,7 @@ def cmd_template(args: argparse.Namespace) -> None: # dumpcap = # artifacts_dir = # artifacts_base = +# progress = [gallia.protocols.uds] # dumpcap = diff --git a/src/gallia/command.py b/src/gallia/command.py index 905f4407d..633170007 100644 --- a/src/gallia/command.py +++ b/src/gallia/command.py @@ -227,6 +227,15 @@ def configure_class_parser(self) -> None: default=self.config.get_value("gallia.lock_file", None), help="path to file used for a posix lock", ) + group.add_argument( + "--progress", + action=argparse.BooleanOptionalAction, + default=self.config.get_value( + "gallia.scanner.progress", + default=sys.stderr.isatty(), + ), + help="Enable/Disable progress bar", + ) if self.HAS_ARTIFACTS_DIR: mutex_group = group.add_mutually_exclusive_group() @@ -344,9 +353,10 @@ def entry_point(self, args: Namespace) -> int: self.get_log_level(args), self.get_file_log_level(args), self.artifacts_dir.joinpath(FileNames.LOGFILE.value), + progress=args.progress, ) else: - setup_logging(self.get_log_level(args)) + setup_logging(self.get_log_level(args), progress=args.progress) self.run_hook(HookVariant.PRE, args) diff --git a/src/gallia/commands/scan/uds/identifiers.py b/src/gallia/commands/scan/uds/identifiers.py index d6cd53d11..1645563ff 100644 --- a/src/gallia/commands/scan/uds/identifiers.py +++ b/src/gallia/commands/scan/uds/identifiers.py @@ -8,6 +8,8 @@ from argparse import Namespace from itertools import product +from tqdm import tqdm + from gallia.command import UDSScanner from gallia.services.uds.core.client import UDSRequestConfig from gallia.services.uds.core.constants import RCSubFuncs, UDSErrorCodes, UDSIsoServices @@ -152,8 +154,19 @@ async def main(self, args: Namespace) -> None: ) args.end = 0xFF - for (DID, sub_function) in product( - range(args.start, args.end + 1), sub_functions + end_index = args.end + 1 + n_iterations = (end_index - args.start) * len(sub_functions) + for (DID, sub_function) in tqdm( + product( + range(args.start, end_index), + sub_functions, + ), + disable=not args.progress, + total=n_iterations, + leave=False, + desc=f"session {session:#x}", + unit="", + miniters=100, ): if session in args.skip and DID in args.skip[session]: self.logger.info(f"{g_repr(DID)}: skipped") diff --git a/src/gallia/log.py b/src/gallia/log.py index 481d3d838..c871a3361 100644 --- a/src/gallia/log.py +++ b/src/gallia/log.py @@ -26,6 +26,7 @@ import msgspec import zstandard +from tqdm.contrib import DummyTqdmFile if TYPE_CHECKING: from logging import _ExcInfoType @@ -250,6 +251,7 @@ def setup_logging( file_level: Loglevel = Loglevel.DEBUG, path: Path | None = None, color_mode: ColorMode = ColorMode.AUTO, + progress: bool = False, ) -> None: """Enable and configure gallia's logging system. If this fuction is not called as early as possible, @@ -265,6 +267,9 @@ def setup_logging( :param file_level: The loglevel to enable for the file handler. :param path: The path to the logfile containing json records. :param color_mode: The color mode to use for the console. + :param progress: If a progress bar via ``tqdm`` is used, set + this to ``True`` to setup the console handler + correctly. """ set_color_mode(color_mode) @@ -283,7 +288,8 @@ def setup_logging( logging.getLogger("asyncio").setLevel(logging.CRITICAL) logging.getLogger("aiosqlite").setLevel(logging.CRITICAL) - stderr_handler = logging.StreamHandler(sys.stderr) + stream = DummyTqdmFile(sys.stderr) if progress else sys.stderr + stderr_handler = logging.StreamHandler(stream) # type: ignore stderr_handler.setLevel(level) stderr_handler.setFormatter(_ConsoleFormatter())