From f304397ef0dae70082d90e6d30d47b1d8e998e63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Tue, 9 Jul 2024 14:46:17 +0200 Subject: [PATCH] chore: Improve code style checks --- .github/workflows/code-style.yml | 75 +- .github/workflows/test.yml | 2 +- .pylintrc.ini | 637 +++++++++++ .../dsw/command_queue/command_queue.py | 57 +- .../dsw/command_queue/query.py | 16 +- packages/dsw-command-queue/pyproject.toml | 4 +- packages/dsw-config/dsw/config/keys.py | 29 +- packages/dsw-config/dsw/config/model.py | 12 +- packages/dsw-config/dsw/config/parser.py | 20 +- packages/dsw-config/pyproject.toml | 4 +- .../dsw-data-seeder/dsw/data_seeder/cli.py | 15 +- .../dsw-data-seeder/dsw/data_seeder/config.py | 2 +- .../dsw/data_seeder/context.py | 21 +- .../dsw/data_seeder/handlers.py | 3 +- .../dsw-data-seeder/dsw/data_seeder/seeder.py | 73 +- packages/dsw-data-seeder/pyproject.toml | 4 +- .../dsw-database/dsw/database/database.py | 60 +- packages/dsw-database/dsw/database/model.py | 64 +- packages/dsw-database/pyproject.toml | 4 +- .../dsw/document_worker/cli.py | 15 +- .../dsw/document_worker/config.py | 12 +- .../dsw/document_worker/context.py | 21 +- .../dsw/document_worker/conversions.py | 20 +- .../dsw/document_worker/documents.py | 4 +- .../dsw/document_worker/exceptions.py | 3 +- .../dsw/document_worker/handlers.py | 5 +- .../dsw/document_worker/limits.py | 4 +- .../dsw/document_worker/model/context.py | 987 ++++++++++-------- .../dsw/document_worker/model/http.py | 9 +- .../dsw/document_worker/templates/filters.py | 20 +- .../dsw/document_worker/templates/formats.py | 2 +- .../document_worker/templates/steps/base.py | 11 +- .../templates/steps/conversion.py | 2 +- .../document_worker/templates/steps/excel.py | 28 +- .../templates/steps/template.py | 73 +- .../document_worker/templates/steps/word.py | 56 +- .../document_worker/templates/templates.py | 62 +- .../dsw/document_worker/worker.py | 70 +- packages/dsw-document-worker/pyproject.toml | 4 +- packages/dsw-mailer/dsw/mailer/cli.py | 19 +- packages/dsw-mailer/dsw/mailer/context.py | 17 +- packages/dsw-mailer/dsw/mailer/mailer.py | 45 +- packages/dsw-mailer/dsw/mailer/model.py | 45 +- .../dsw/mailer/sender/amazon_ses.py | 6 +- packages/dsw-mailer/dsw/mailer/sender/base.py | 6 +- packages/dsw-mailer/dsw/mailer/sender/smtp.py | 6 +- packages/dsw-mailer/dsw/mailer/templates.py | 37 +- packages/dsw-mailer/pyproject.toml | 4 +- packages/dsw-models/dsw/models/km/events.py | 97 +- packages/dsw-models/dsw/models/km/package.py | 8 +- packages/dsw-models/pyproject.toml | 4 +- packages/dsw-storage/dsw/storage/s3storage.py | 11 +- packages/dsw-storage/pyproject.toml | 4 +- packages/dsw-tdk/pyproject.toml | 4 +- scripts/clean.sh | 10 + 55 files changed, 1790 insertions(+), 1043 deletions(-) create mode 100644 .pylintrc.ini create mode 100755 scripts/clean.sh diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml index 951d86f0..64c5753c 100644 --- a/.github/workflows/code-style.yml +++ b/.github/workflows/code-style.yml @@ -16,7 +16,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.12 cache: pip cache-dependency-path: | **/pyproject.toml @@ -26,10 +26,10 @@ jobs: run: | bash scripts/build-info.sh - - name: Install Flake8 (5.0.4) + - name: Install Flake8 (7.1.0) run: | python -m pip install --upgrade pip - pip install flake8==5.0.4 + pip install flake8==7.1.0 - name: Install dependencies run: | @@ -51,6 +51,7 @@ jobs: echo "- $package" echo "-------------------------------------------------" pip install packages/$package + rm -rf packages/$package/build echo "=================================================" done @@ -71,7 +72,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.12 cache: pip cache-dependency-path: | **/pyproject.toml @@ -81,10 +82,10 @@ jobs: run: | bash scripts/build-info.sh - - name: Install MyPy (1.4.1) + - name: Install MyPy (1.11.2) run: | python -m pip install --upgrade pip - pip install mypy==1.4.1 + pip install mypy==1.11.2 - name: Install dependencies run: | @@ -106,14 +107,17 @@ jobs: echo "- $package" echo "-------------------------------------------------" pip install packages/$package + rm -rf packages/$package/build echo "=================================================" done - name: Check typing with MyPy run: | - mypy --install-types --ignore-missing-imports --check-untyped-defs --non-interactive packages/*/dsw - + mypy --install-types --ignore-missing-imports \ + --check-untyped-defs --non-interactive \ + packages/*/dsw + # Consistency of version tagging version: name: Version consts.py runs-on: ubuntu-latest @@ -145,3 +149,58 @@ jobs: bash scripts/check-version.sh \ packages/dsw-tdk/dsw/tdk/consts.py \ packages/dsw-tdk/pyproject.toml + + # Pylint + pylint: + name: Pylint + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.12 + cache: pip + cache-dependency-path: | + **/pyproject.toml + **/requirements*.txt + + - name: Create build info + run: | + bash scripts/build-info.sh + + - name: Install PyLint (3.2.5) + run: | + python -m pip install --upgrade pip + pip install pylint==3.2.5 + + - name: Install dependencies + run: | + ROOT=$(pwd) + for package in $(ls packages); do + echo "-------------------------------------------------" + echo "- $package" + echo "-------------------------------------------------" + cd "$ROOT/packages/$package" + pip install -r requirements.txt + make local-deps + echo "=================================================" + done + + - name: Install packages + run: | + for package in $(ls packages); do + echo "-------------------------------------------------" + echo "- $package" + echo "-------------------------------------------------" + pip install packages/$package + rm -rf packages/$package/build + echo "=================================================" + done + + - name: Lint with PyLint + run: | + pylint --rcfile=.pylintrc.ini packages/*/dsw diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 52007c66..4463f6a4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,8 +15,8 @@ jobs: - 'ubuntu-latest' - 'windows-latest' python-version: - - '3.10' - '3.11' + - '3.12' runs-on: ${{ matrix.os }} diff --git a/.pylintrc.ini b/.pylintrc.ini new file mode 100644 index 00000000..5c95b490 --- /dev/null +++ b/.pylintrc.ini @@ -0,0 +1,637 @@ +[MAIN] + +# Analyse import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analysed. +analyse-fallback-blocks=no + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=10 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked +# (useful for modules/projects where namespaces are manipulated during runtime +# and thus existing member attributes cannot be deduced by static analysis). It +# supports qualified module names, as well as Unix pattern matching. +ignored-modules= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.11 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +# When enabled, pylint would attempt to guess common misconfiguration and emit +# user-friendly hints instead of false-positive error messages. +suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + db, + s3, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=10 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=100 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-symbolic-message-instead, + missing-function-docstring, + missing-module-docstring, + missing-class-docstring, + broad-exception-caught + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable=c-extension-no-member + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX, + TODO + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are text, parseable, colorized, json +# and msvs (visual studio). You can also give a reporter class, e.g. +# mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=no + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work.. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The minimum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io diff --git a/packages/dsw-command-queue/dsw/command_queue/command_queue.py b/packages/dsw-command-queue/dsw/command_queue/command_queue.py index 56911f46..945b05ac 100644 --- a/packages/dsw-command-queue/dsw/command_queue/command_queue.py +++ b/packages/dsw-command-queue/dsw/command_queue/command_queue.py @@ -1,13 +1,14 @@ import abc import datetime -import func_timeout import logging import os import platform -import psycopg -import psycopg.generators import select import signal + +import func_timeout +import psycopg +import psycopg.generators import tenacity from dsw.database import Database @@ -22,7 +23,6 @@ RETRY_QUEUE_MULTIPLIER = 0.5 RETRY_QUEUE_TRIES = 5 -INTERRUPTED = False IS_LINUX = platform == 'Linux' if IS_LINUX: @@ -30,20 +30,10 @@ signal.set_wakeup_fd(_QUEUE_PIPE_W) -def signal_handler(recv_signal, frame): - global INTERRUPTED - LOG.warning(f'Received interrupt signal: {recv_signal}') - INTERRUPTED = True - - -signal.signal(signal.SIGINT, signal_handler) -signal.signal(signal.SIGABRT, signal_handler) - - class CommandWorker: @abc.abstractmethod - def work(self, payload: PersistentCommand): + def work(self, command: PersistentCommand): pass def process_timeout(self, e: BaseException): @@ -66,6 +56,10 @@ def __init__(self, worker: CommandWorker, db: Database, ) self.wait_timeout = wait_timeout self.work_timeout = work_timeout + self._interrupted = False + + signal.signal(signal.SIGINT, self._signal_handler) + signal.signal(signal.SIGABRT, self._signal_handler) @tenacity.retry( reraise=True, @@ -92,19 +86,19 @@ def run(self): LOG.debug('Waiting for notifications') w = select.select(fds, [], [], self.wait_timeout) - if INTERRUPTED: + if self._interrupted: LOG.debug('Interrupt signal received, ending...') break if w == ([], [], []): - LOG.debug(f'Nothing received in this cycle ' - f'(timeouted after {self.wait_timeout} seconds)') + LOG.debug('Nothing received in this cycle (timeout %s seconds)', + self.wait_timeout) else: notifications = 0 for n in psycopg.generators.notifies(queue_conn.connection.pgconn): notifications += 1 LOG.debug(str(n)) - LOG.info(f'Notifications received ({notifications} in total)') + LOG.info('Notifications received (%s in total)', notifications) LOG.debug('Exiting command queue') @tenacity.retry( @@ -123,11 +117,12 @@ def _fetch_and_process_queued(self): count = 0 while self.fetch_and_process(): count += 1 - LOG.info(f'There are no more commands to process ({count} processed)') + LOG.info('There are no more commands to process (%s processed)', + count) def accept_notification(self, payload: psycopg.Notify) -> bool: - LOG.debug(f'Accepting notification from channel "{payload.channel}" ' - f'(PID = {payload.pid}) {payload.payload}') + LOG.debug('Accepting notification from channel "%s" (PID = %s) %s', + payload.channel, payload.pid, payload.payload) LOG.debug('Trying to fetch a new job') return self.fetch_and_process() @@ -139,14 +134,14 @@ def fetch_and_process(self) -> bool: ) result = cursor.fetchall() if len(result) != 1: - LOG.debug(f'Fetched {len(result)} persistent commands') + LOG.debug('Fetched %s persistent commands', len(result)) return False command = PersistentCommand.from_dict_row(result[0]) - LOG.info(f'Retrieved persistent command {command.uuid} for processing') - LOG.debug(f'Previous state: {command.state}') - LOG.debug(f'Attempts: {command.attempts} / {command.max_attempts}') - LOG.debug(f'Last error: {command.last_error_message}') + LOG.info('Retrieved persistent command %s for processing', command.uuid) + LOG.debug('Previous state: %s', command.state) + LOG.debug('Attempts: %s / %s', command.attempts, command.max_attempts) + LOG.debug('Last error: %s', command.last_error_message) attempt_number = command.attempts + 1 try: @@ -165,7 +160,8 @@ def work(): LOG.info('Processing (without any timeout set)') work() else: - LOG.info(f'Processing (with timeout set to {self.work_timeout} seconds)') + LOG.info('Processing (with timeout set to %s seconds)', + self.work_timeout) func_timeout.func_timeout( timeout=self.work_timeout, func=work, @@ -207,3 +203,8 @@ def work(): cursor.close() LOG.info('Notification processing finished') return True + + def _signal_handler(self, recv_signal, frame): + LOG.warning('Received interrupt signal: %s (frame: %s)', + recv_signal, frame) + self._interrupted = True diff --git a/packages/dsw-command-queue/dsw/command_queue/query.py b/packages/dsw-command-queue/dsw/command_queue/query.py index 50e2cda6..fd773003 100644 --- a/packages/dsw-command-queue/dsw/command_queue/query.py +++ b/packages/dsw-command-queue/dsw/command_queue/query.py @@ -1,4 +1,6 @@ -class CommandState: +import enum + +class CommandState(enum.Enum): NEW = 'NewPersistentCommandState' DONE = 'DonePersistentCommandState' ERROR = 'ErrorPersistentCommandState' @@ -20,9 +22,11 @@ def query_get_command(self, exp=2, interval='1 min') -> str: FROM persistent_command WHERE component = '{self.component}' AND attempts < max_attempts - AND state != '{CommandState.DONE}' - AND state != '{CommandState.IGNORE}' - AND (updated_at AT TIME ZONE 'UTC') < (%(now)s - ({exp} ^ attempts - 1) * INTERVAL '{interval}') + AND state != '{CommandState.DONE.value}' + AND state != '{CommandState.IGNORE.value}' + AND (updated_at AT TIME ZONE 'UTC') + < + (%(now)s - ({exp} ^ attempts - 1) * INTERVAL '{interval}') ORDER BY attempts ASC, updated_at DESC LIMIT 1 FOR UPDATE SKIP LOCKED; """ @@ -33,7 +37,7 @@ def query_command_error() -> str: UPDATE persistent_command SET attempts = %(attempts)s, last_error_message = %(error_message)s, - state = '{CommandState.ERROR}', + state = '{CommandState.ERROR.value}', updated_at = %(updated_at)s WHERE uuid = %(uuid)s; """ @@ -43,7 +47,7 @@ def query_command_done() -> str: return f""" UPDATE persistent_command SET attempts = %(attempts)s, - state = '{CommandState.DONE}', + state = '{CommandState.DONE.value}', updated_at = %(updated_at)s WHERE uuid = %(uuid)s; """ diff --git a/packages/dsw-command-queue/pyproject.toml b/packages/dsw-command-queue/pyproject.toml index d663cf04..9b06dcc3 100644 --- a/packages/dsw-command-queue/pyproject.toml +++ b/packages/dsw-command-queue/pyproject.toml @@ -16,13 +16,13 @@ classifiers = [ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Database', 'Topic :: Text Processing', 'Topic :: Utilities', ] -requires-python = '>=3.10, <4' +requires-python = '>=3.11, <4' dependencies = [ 'func-timeout', # DSW diff --git a/packages/dsw-config/dsw/config/keys.py b/packages/dsw-config/dsw/config/keys.py index 3a9cccf6..aa70f72e 100644 --- a/packages/dsw-config/dsw/config/keys.py +++ b/packages/dsw-config/dsw/config/keys.py @@ -1,60 +1,59 @@ import collections +import typing -from typing import Any, Optional, Generic, TypeVar, Callable +T = typing.TypeVar('T') -T = TypeVar('T') - -def cast_bool(value: Any) -> bool: +def cast_bool(value: typing.Any) -> bool: return bool(value) -def cast_optional_bool(value: Any) -> Optional[bool]: +def cast_optional_bool(value: typing.Any) -> bool | None: if value is None: return None return bool(value) -def cast_int(value: Any) -> int: +def cast_int(value: typing.Any) -> int: return int(value) -def cast_optional_int(value: Any) -> Optional[int]: +def cast_optional_int(value: typing.Any) -> int | None: if value is None: return None return int(value) -def cast_float(value: Any) -> float: +def cast_float(value: typing.Any) -> float: return float(value) -def cast_optional_float(value: Any) -> Optional[float]: +def cast_optional_float(value: typing.Any) -> float | None: if value is None: return None return float(value) -def cast_str(value: Any) -> str: +def cast_str(value: typing.Any) -> str: return str(value) -def cast_optional_str(value: Any) -> Optional[str]: +def cast_optional_str(value: typing.Any) -> str | None: if value is None: return None return str(value) -def cast_optional_dict(value: Any) -> Optional[dict]: +def cast_optional_dict(value: typing.Any) -> dict | None: if not isinstance(value, dict): return None return value -class ConfigKey(Generic[T]): +class ConfigKey(typing.Generic[T]): - def __init__(self, yaml_path: list[str], cast: Callable[[Any], T], + def __init__(self, yaml_path: list[str], cast: typing.Callable[[typing.Any], T], var_names=None, default=None, required=False): self.yaml_path = yaml_path self.var_names = var_names or [] # type: list[str] @@ -63,7 +62,7 @@ def __init__(self, yaml_path: list[str], cast: Callable[[Any], T], self.cast = cast def __str__(self): - return 'ConfigKey: ' + '.'.join(self.yaml_path) + return f'ConfigKey: {".".join(self.yaml_path)}' class ConfigKeysMeta(type): diff --git a/packages/dsw-config/dsw/config/model.py b/packages/dsw-config/dsw/config/model.py index 5ff3226c..2c7dfe55 100644 --- a/packages/dsw-config/dsw/config/model.py +++ b/packages/dsw-config/dsw/config/model.py @@ -1,5 +1,3 @@ -from typing import Optional - from .logging import prepare_logging, LOG_FILTER @@ -29,8 +27,8 @@ def __init__(self, environment: str, client_url: str, secret: str): class SentryConfig(ConfigModel): - def __init__(self, enabled: bool, workers_dsn: Optional[str], - traces_sample_rate: Optional[float], max_breadcrumbs: Optional[int]): + def __init__(self, enabled: bool, workers_dsn: str | None, + traces_sample_rate: float | None, max_breadcrumbs: int | None): self.enabled = enabled self.workers_dsn = workers_dsn self.traces_sample_rate = traces_sample_rate @@ -60,7 +58,7 @@ def __init__(self, url: str, username: str, password: str, class LoggingConfig(ConfigModel): def __init__(self, level: str, global_level: str, message_format: str, - dict_config: Optional[dict] = None): + dict_config: dict | None = None): self.level = level self.global_level = global_level self.message_format = message_format @@ -76,8 +74,8 @@ def set_logging_extra(key: str, value: str): class AWSConfig(ConfigModel): - def __init__(self, access_key_id: Optional[str], secret_access_key: Optional[str], - region: Optional[str]): + def __init__(self, access_key_id: str | None, secret_access_key: str | None, + region: str | None): self.access_key_id = access_key_id self.secret_access_key = secret_access_key self.region = region diff --git a/packages/dsw-config/dsw/config/parser.py b/packages/dsw-config/dsw/config/parser.py index b0a042a0..86c9315a 100644 --- a/packages/dsw-config/dsw/config/parser.py +++ b/packages/dsw-config/dsw/config/parser.py @@ -1,7 +1,7 @@ import os -import yaml +import typing -from typing import List, Any, IO +import yaml from .keys import ConfigKey, ConfigKeys from .model import GeneralConfig, SentryConfig, S3Config, \ @@ -10,14 +10,14 @@ class MissingConfigurationError(Exception): - def __init__(self, missing: List[str]): + def __init__(self, missing: list[str]): self.missing = missing class DSWConfigParser: def __init__(self, keys=ConfigKeys): - self.cfg = dict() + self.cfg = {} self.keys = keys @staticmethod @@ -28,7 +28,7 @@ def can_read(content: str): except Exception: return False - def read_file(self, fp: IO): + def read_file(self, fp: typing.IO): self.cfg = yaml.load(fp, Loader=yaml.FullLoader) or self.cfg def read_string(self, content: str): @@ -50,12 +50,12 @@ def has_value_for_key(self, key: ConfigKey): if self.has_value_for_path(key.yaml_path): return True for var_name in key.var_names: - if var_name in os.environ.keys() or \ - self._prefix_var(var_name) in os.environ.keys(): + if var_name in os.environ or self._prefix_var(var_name) in os.environ: return True + return False def get_or_default(self, key: ConfigKey): - x = self.cfg # type: Any + x: typing.Any = self.cfg for p in key.yaml_path: if not hasattr(x, 'keys') or p not in x.keys(): return key.default @@ -64,9 +64,9 @@ def get_or_default(self, key: ConfigKey): def get(self, key: ConfigKey): for var_name in key.var_names: - if var_name in os.environ.keys(): + if var_name in os.environ: return key.cast(os.environ[var_name]) - if self._prefix_var(var_name) in os.environ.keys(): + if self._prefix_var(var_name) in os.environ: return key.cast(os.environ[self._prefix_var(var_name)]) return key.cast(self.get_or_default(key)) diff --git a/packages/dsw-config/pyproject.toml b/packages/dsw-config/pyproject.toml index 3c318c5f..8326b0ae 100644 --- a/packages/dsw-config/pyproject.toml +++ b/packages/dsw-config/pyproject.toml @@ -16,12 +16,12 @@ classifiers = [ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Text Processing', 'Topic :: Utilities', ] -requires-python = '>=3.10, <4' +requires-python = '>=3.11, <4' dependencies = [ 'PyYAML', 'sentry-sdk', diff --git a/packages/dsw-data-seeder/dsw/data_seeder/cli.py b/packages/dsw-data-seeder/dsw/data_seeder/cli.py index 72a03836..2a3ff1d3 100644 --- a/packages/dsw-data-seeder/dsw/data_seeder/cli.py +++ b/packages/dsw-data-seeder/dsw/data_seeder/cli.py @@ -1,7 +1,8 @@ -import click # type: ignore import pathlib +import sys +import typing -from typing import IO, Optional +import click from dsw.config.parser import MissingConfigurationError @@ -15,7 +16,7 @@ def load_config_str(config_str: str) -> SeederConfig: parser = SeederConfigParser() if not parser.can_read(config_str): click.echo('Error: Cannot parse config file', err=True) - exit(1) + sys.exit(1) try: parser.read_string(config_str) @@ -24,14 +25,14 @@ def load_config_str(config_str: str) -> SeederConfig: click.echo('Error: Missing configuration', err=True) for missing_item in e.missing: click.echo(f' - {missing_item}', err=True) - exit(1) + sys.exit(1) config = parser.config config.log.apply() return config -def validate_config(ctx, param, value: Optional[IO]): +def validate_config(ctx, param, value: typing.IO | None): content = '' if value is not None: content = value.read() @@ -75,7 +76,7 @@ def seed(ctx: click.Context, recipe: str, tenant_uuid: str): @cli.command(name='list', help='List recipes for data seeding.') @click.pass_context -def list(ctx: click.Context): +def recipes_list(ctx: click.Context): workdir = ctx.obj['workdir'] recipes = SeedRecipe.load_from_dir(workdir) for recipe in recipes.values(): @@ -84,4 +85,4 @@ def list(ctx: click.Context): def main(): - cli(obj=dict()) + cli(obj={}) diff --git a/packages/dsw-data-seeder/dsw/data_seeder/config.py b/packages/dsw-data-seeder/dsw/data_seeder/config.py index 8ab2402d..ce220a8d 100644 --- a/packages/dsw-data-seeder/dsw/data_seeder/config.py +++ b/packages/dsw-data-seeder/dsw/data_seeder/config.py @@ -60,7 +60,7 @@ def __init__(self): @property def extra_dbs(self) -> dict[str, DatabaseConfig]: result = {} - for db_id, val in self.cfg.get('extraDatabases', {}).items(): + for db_id in self.cfg.get('extraDatabases', {}).keys(): result[db_id] = DatabaseConfig( connection_string=self.get( key=ConfigKey( diff --git a/packages/dsw-data-seeder/dsw/data_seeder/context.py b/packages/dsw-data-seeder/dsw/data_seeder/context.py index b8307fd1..499948f2 100644 --- a/packages/dsw-data-seeder/dsw/data_seeder/context.py +++ b/packages/dsw-data-seeder/dsw/data_seeder/context.py @@ -1,11 +1,9 @@ import pathlib -from typing import Optional, TYPE_CHECKING +from dsw.database import Database +from dsw.storage import S3Storage -if TYPE_CHECKING: - from .config import SeederConfig - from dsw.database import Database - from dsw.storage import S3Storage +from .config import SeederConfig class ContextNotInitializedError(RuntimeError): @@ -16,11 +14,12 @@ def __init__(self): class AppContext: - def __init__(self, db, s3, cfg, workdir): - self.db = db # type: Database - self.s3 = s3 # type: S3Storage - self.cfg = cfg # type: SeederConfig - self.workdir = workdir # type: pathlib.Path + def __init__(self, db: Database, s3: S3Storage, + cfg: SeederConfig, workdir: pathlib.Path): + self.db = db + self.s3 = s3 + self.cfg = cfg + self.workdir = workdir class JobContext: @@ -49,7 +48,7 @@ def reset_ids(self): class Context: - _instance = None # type: Optional[_Context] + _instance: _Context | None = None @classmethod def get(cls) -> _Context: diff --git a/packages/dsw-data-seeder/dsw/data_seeder/handlers.py b/packages/dsw-data-seeder/dsw/data_seeder/handlers.py index 1a7e0005..2bf84dba 100644 --- a/packages/dsw-data-seeder/dsw/data_seeder/handlers.py +++ b/packages/dsw-data-seeder/dsw/data_seeder/handlers.py @@ -1,5 +1,6 @@ import os import pathlib +import sys from .cli import load_config_str from .consts import VAR_APP_CONFIG_PATH, VAR_WORKDIR_PATH, VAR_SEEDER_RECIPE @@ -13,7 +14,7 @@ def lambda_handler(event, context): if recipe_name is None: print(f'Error: Missing recipe name (environment variable {VAR_SEEDER_RECIPE})') - exit(1) + sys.exit(1) config = load_config_str(config_path.read_text()) seeder = DataSeeder(config, workdir_path) diff --git a/packages/dsw-data-seeder/dsw/data_seeder/seeder.py b/packages/dsw-data-seeder/dsw/data_seeder/seeder.py index 06422d64..b4dacacd 100644 --- a/packages/dsw-data-seeder/dsw/data_seeder/seeder.py +++ b/packages/dsw-data-seeder/dsw/data_seeder/seeder.py @@ -1,13 +1,13 @@ import collections -import dateutil.parser import json import logging import mimetypes import pathlib import time +import typing import uuid -from typing import Optional +import dateutil.parser from dsw.command_queue import CommandWorker, CommandQueue from dsw.config.sentry import SentryReporter @@ -49,8 +49,8 @@ class SeedRecipe: def __init__(self, name: str, description: str, root: pathlib.Path, db_scripts: dict[str, DBScript], db_placeholder: str, - s3_dir: Optional[pathlib.Path], s3_fname_replace: dict[str, str], - uuids_count: int, uuids_placeholder: Optional[str], + s3_dir: pathlib.Path | None, s3_fname_replace: dict[str, str], + uuids_count: int, uuids_placeholder: str | None, init_wait: float): self.name = name self.description = description @@ -59,12 +59,12 @@ def __init__(self, name: str, description: str, root: pathlib.Path, self.db_placeholder = db_placeholder self.s3_dir = s3_dir self.s3_fname_replace = s3_fname_replace - self._db_scripts_data = collections.OrderedDict() # type: dict[str, str] - self.s3_objects = collections.OrderedDict() # type: dict[pathlib.Path, str] + self._db_scripts_data: dict[str, str] = collections.OrderedDict() + self.s3_objects: dict[pathlib.Path, str] = collections.OrderedDict() self.prepared = False self.uuids_count = uuids_count self.uuids_placeholder = uuids_placeholder - self.uuids_replacement = dict() # type: dict[str, str] + self.uuids_replacement: dict[str, str] = {} self.init_wait = init_wait def _load_db_scripts(self): @@ -147,10 +147,10 @@ def load_from_json(recipe_file: pathlib.Path) -> 'SeedRecipe': data = json.loads(recipe_file.read_text( encoding=DEFAULT_ENCODING, )) - db = data.get('db', {}) # type: dict - s3 = data.get('s3', {}) # type: dict - scripts = db.get('scripts', []) # type: list[dict] - db_scripts = collections.OrderedDict() # type: dict[str, DBScript] + db: dict[str, typing.Any] = data.get('db', {}) + s3: dict[str, typing.Any] = data.get('s3', {}) + scripts: list[dict] = db.get('scripts', []) + db_scripts: dict[str, DBScript] = collections.OrderedDict() for index, script in enumerate(scripts): target = script.get('target', '') filename = str(script.get('filename', '')) @@ -158,7 +158,7 @@ def load_from_json(recipe_file: pathlib.Path) -> 'SeedRecipe': continue filepath = pathlib.Path(filename) if '*' in filename: - for item in sorted([s for s in recipe_file.parent.glob(filename)]): + for item in sorted(list(recipe_file.parent.glob(filename))): s = DBScript(item, target, index) db_scripts[s.id] = s elif filepath.is_absolute(): @@ -275,15 +275,16 @@ def run_once(self, recipe_name: str): queue = self._run_preparation(recipe_name) queue.run_once() - def work(self, cmd: PersistentCommand): - Context.get().update_trace_id(cmd.uuid) - SentryReporter.set_context('cmd_uuid', cmd.uuid) + def work(self, command: PersistentCommand): + Context.get().update_trace_id(command.uuid) + SentryReporter.set_context('cmd_uuid', command.uuid) self.recipe.run_prepare() - tenant_uuid = cmd.body['tenantUuid'] - LOG.info(f'Seeding recipe "{self.recipe.name}" ' - f'to tenant with UUID "{tenant_uuid}"') - if cmd.attempts == 0 and self.recipe.init_wait > 0.01: - LOG.info(f'Waiting for {self.recipe.init_wait} seconds (first attempt)') + tenant_uuid = command.body['tenantUuid'] + LOG.info('Seeding recipe "%s" to tenant "%s"', + self.recipe.name, tenant_uuid) + if command.attempts == 0 and self.recipe.init_wait > 0.01: + LOG.info('Waiting for %s seconds (first attempt)', + self.recipe.init_wait) time.sleep(self.recipe.init_wait) self.execute(tenant_uuid) Context.get().update_trace_id('-') @@ -300,8 +301,8 @@ def process_exception(self, e: BaseException): @staticmethod def _update_component_info(): built_at = dateutil.parser.parse(BUILD_INFO.built_at) - LOG.info(f'Updating component info ({BUILD_INFO.version}, ' - f'{built_at.isoformat(timespec="seconds")})') + LOG.info('Updating component info (%s, %s)', + BUILD_INFO.version, built_at.isoformat(timespec="seconds")) Context.get().app.db.update_component_info( name=COMPONENT_NAME, version=BUILD_INFO.version, @@ -310,7 +311,7 @@ def _update_component_info(): def seed(self, recipe_name: str, tenant_uuid: str): self._prepare_recipe(recipe_name) - LOG.info(f'Executing recipe "{recipe_name}"') + LOG.info('Executing recipe "%s"', recipe_name) self.execute(tenant_uuid=tenant_uuid) def execute(self, tenant_uuid: str): @@ -322,9 +323,9 @@ def execute(self, tenant_uuid: str): try: LOG.info('Running SQL scripts') for script_id, sql_script in self.recipe.iterate_db_scripts(tenant_uuid): - LOG.debug(f' -> Executing script: {script_id}') + LOG.debug(' -> Executing script: %s', script_id) script = self.recipe.db_scripts[script_id] - if script.target in self.dbs.keys(): + if script.target in self.dbs: used_targets.add(script.target) with self.dbs[script.target].conn_query.new_cursor(use_dict=True) as c: c.execute(query=sql_script) @@ -335,9 +336,9 @@ def execute(self, tenant_uuid: str): phase = 'S3' LOG.info('Transferring S3 objects') for local_file, object_name in self.recipe.iterate_s3_objects(): - LOG.debug(f' -> Reading: {local_file.name}') + LOG.debug(' -> Reading: %s', local_file.name) data = local_file.read_bytes() - LOG.debug(f' -> Sending: {object_name}') + LOG.debug(' -> Sending: %s', object_name) app_ctx.s3.store_object( tenant_uuid=tenant_uuid, object_name=object_name, @@ -347,21 +348,25 @@ def execute(self, tenant_uuid: str): LOG.debug(' OK (stored)') except Exception as e: SentryReporter.capture_exception(e) - LOG.warning(f'Exception appeared [{type(e).__name__}]: {e}') + LOG.warning('Exception appeared [%s]: %s', type(e).__name__, str(e)) LOG.info('Failed with unexpected error', exc_info=e) LOG.info('Rolling back DB changes') - LOG.debug(f'Used extra DBs: {used_targets}') + LOG.debug('Used extra DBs: %s', str(used_targets)) conn = app_ctx.db.conn_query.connection - LOG.debug(f'DEFAULT will roll back: {conn.pgconn.status} / {conn.pgconn.transaction_status}') + LOG.debug('DEFAULT will roll back: %s / %s', + conn.pgconn.status, conn.pgconn.transaction_status) conn.rollback() - LOG.debug(f'DEFAULT rolled back: {conn.pgconn.status} / {conn.pgconn.transaction_status}') + LOG.debug('DEFAULT rolled back: %s / %s', + conn.pgconn.status, conn.pgconn.transaction_status) for target in used_targets: conn = self.dbs[target].conn_query.connection - LOG.debug(f'{target} will roll back: {conn.pgconn.status} / {conn.pgconn.transaction_status}') + LOG.debug('%s will roll back: %s / %s', + target, conn.pgconn.status, conn.pgconn.transaction_status) conn.rollback() - LOG.debug(f'{target} rolled back: {conn.pgconn.status} / {conn.pgconn.transaction_status}') - raise RuntimeError(f'{phase}: {e}') + LOG.debug('%s rolled back: %s / %s', + target, conn.pgconn.status, conn.pgconn.transaction_status) + raise RuntimeError(f'{phase}: {e}') from e else: LOG.info('Committing DB changes') app_ctx.db.conn_query.connection.commit() diff --git a/packages/dsw-data-seeder/pyproject.toml b/packages/dsw-data-seeder/pyproject.toml index dd98f054..a1213e5c 100644 --- a/packages/dsw-data-seeder/pyproject.toml +++ b/packages/dsw-data-seeder/pyproject.toml @@ -16,13 +16,13 @@ classifiers = [ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Database', 'Topic :: Text Processing', 'Topic :: Utilities', ] -requires-python = '>=3.10, <4' +requires-python = '>=3.11, <4' dependencies = [ 'click', 'python-dateutil', diff --git a/packages/dsw-database/dsw/database/database.py b/packages/dsw-database/dsw/database/database.py index c834a6e4..44c20587 100644 --- a/packages/dsw-database/dsw/database/database.py +++ b/packages/dsw-database/dsw/database/database.py @@ -1,13 +1,13 @@ import datetime import logging +import typing + import psycopg import psycopg.conninfo import psycopg.rows import psycopg.types.json import tenacity -from typing import List, Iterable, Optional - from dsw.config.model import DatabaseConfig from .model import DBDocumentTemplate, DBDocumentTemplateFile, \ @@ -121,7 +121,7 @@ def _check_table_exists(self, table_name: str) -> bool: before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def fetch_document(self, document_uuid: str, tenant_uuid: str) -> Optional[DBDocument]: + def fetch_document(self, document_uuid: str, tenant_uuid: str) -> DBDocument | None: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_DOCUMENT, @@ -139,7 +139,7 @@ def fetch_document(self, document_uuid: str, tenant_uuid: str) -> Optional[DBDoc before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def fetch_tenant_config(self, tenant_uuid: str) -> Optional[DBTenantConfig]: + def fetch_tenant_config(self, tenant_uuid: str) -> DBTenantConfig | None: return self.get_tenant_config(tenant_uuid) @tenacity.retry( @@ -149,7 +149,7 @@ def fetch_tenant_config(self, tenant_uuid: str) -> Optional[DBTenantConfig]: before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def fetch_tenant_limits(self, tenant_uuid: str) -> Optional[DBTenantLimits]: + def fetch_tenant_limits(self, tenant_uuid: str) -> DBTenantLimits | None: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_TENANT_LIMIT, @@ -169,7 +169,7 @@ def fetch_tenant_limits(self, tenant_uuid: str) -> Optional[DBTenantLimits]: ) def fetch_template( self, template_id: str, tenant_uuid: str - ) -> Optional[DBDocumentTemplate]: + ) -> DBDocumentTemplate | None: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_TEMPLATE, @@ -189,7 +189,7 @@ def fetch_template( ) def fetch_template_files( self, template_id: str, tenant_uuid: str - ) -> List[DBDocumentTemplateFile]: + ) -> list[DBDocumentTemplateFile]: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_TEMPLATE_FILES, @@ -206,7 +206,7 @@ def fetch_template_files( ) def fetch_template_assets( self, template_id: str, tenant_uuid: str - ) -> List[DBDocumentTemplateAsset]: + ) -> list[DBDocumentTemplateAsset]: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_TEMPLATE_ASSETS, @@ -221,7 +221,7 @@ def fetch_template_assets( before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def fetch_qtn_documents(self, questionnaire_uuid: str, tenant_uuid: str) -> List[DBDocument]: + def fetch_qtn_documents(self, questionnaire_uuid: str, tenant_uuid: str) -> list[DBDocument]: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_QTN_DOCUMENTS, @@ -236,7 +236,7 @@ def fetch_qtn_documents(self, questionnaire_uuid: str, tenant_uuid: str) -> List before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def fetch_document_submissions(self, document_uuid: str, tenant_uuid: str) -> List[DBSubmission]: + def fetch_document_submissions(self, document_uuid: str, tenant_uuid: str) -> list[DBSubmission]: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_DOCUMENT_SUBMISSIONS, @@ -251,7 +251,7 @@ def fetch_document_submissions(self, document_uuid: str, tenant_uuid: str) -> Li before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def fetch_questionnaire_submissions(self, questionnaire_uuid: str, tenant_uuid: str) -> List[DBSubmission]: + def fetch_questionnaire_submissions(self, questionnaire_uuid: str, tenant_uuid: str) -> list[DBSubmission]: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_QTN_SUBMISSIONS, @@ -358,7 +358,7 @@ def get_currently_used_size(self, tenant_uuid: str): before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def get_tenant_config(self, tenant_uuid: str) -> Optional[DBTenantConfig]: + def get_tenant_config(self, tenant_uuid: str) -> DBTenantConfig | None: if not self._check_table_exists(table_name='tenant_config'): return None with self.conn_query.new_cursor(use_dict=True) as cursor: @@ -370,8 +370,8 @@ def get_tenant_config(self, tenant_uuid: str) -> Optional[DBTenantConfig]: result = cursor.fetchone() return DBTenantConfig.from_dict_row(data=result) except Exception as e: - LOG.warning(f'Could not retrieve tenant_config for tenant' - f' "{tenant_uuid}": {str(e)}') + LOG.warning('Could not retrieve tenant_config for tenant "%s": %s', + tenant_uuid, str(e)) return None @tenacity.retry( @@ -381,7 +381,7 @@ def get_tenant_config(self, tenant_uuid: str) -> Optional[DBTenantConfig]: before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def get_mail_config(self, tenant_uuid: str) -> Optional[DBInstanceConfigMail]: + def get_mail_config(self, tenant_uuid: str) -> DBInstanceConfigMail | None: with self.conn_query.new_cursor(use_dict=True) as cursor: if not self._check_table_exists(table_name='instance_config_mail'): return None @@ -395,8 +395,8 @@ def get_mail_config(self, tenant_uuid: str) -> Optional[DBInstanceConfigMail]: return None return DBInstanceConfigMail.from_dict_row(data=result) except Exception as e: - LOG.warning(f'Could not retrieve instance_config_mail for tenant' - f' "{tenant_uuid}": {str(e)}') + LOG.warning('Could not retrieve instance_config_mail for tenant "%s": %s', + tenant_uuid, str(e)) return None @tenacity.retry( @@ -409,7 +409,7 @@ def get_mail_config(self, tenant_uuid: str) -> Optional[DBInstanceConfigMail]: def update_component_info(self, name: str, version: str, built_at: datetime.datetime): with self.conn_query.new_cursor(use_dict=True) as cursor: if not self._check_table_exists(table_name='component'): - return None + return ts_now = datetime.datetime.now(tz=datetime.UTC) try: cursor.execute( @@ -424,7 +424,7 @@ def update_component_info(self, name: str, version: str, built_at: datetime.date ) self.conn_query.connection.commit() except Exception as e: - LOG.warning(f'Could not update component info: {str(e)}') + LOG.warning('Could not update component info: %s', str(e)) @tenacity.retry( reraise=True, @@ -433,7 +433,7 @@ def update_component_info(self, name: str, version: str, built_at: datetime.date before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def get_component_info(self, name: str) -> Optional[DBComponent]: + def get_component_info(self, name: str) -> DBComponent | None: if not self._check_table_exists(table_name='component'): return None with self.conn_query.new_cursor(use_dict=True) as cursor: @@ -447,7 +447,7 @@ def get_component_info(self, name: str) -> Optional[DBComponent]: return None return DBComponent.from_dict_row(data=result) except Exception as e: - LOG.warning(f'Could not get component info: {str(e)}') + LOG.warning('Could not get component info: %s', str(e)) return None @tenacity.retry( @@ -457,7 +457,7 @@ def get_component_info(self, name: str) -> Optional[DBComponent]: before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def execute_queries(self, queries: Iterable[str]): + def execute_queries(self, queries: typing.Iterable[str]): with self.conn_query.new_cursor(use_dict=True) as cursor: for query in queries: cursor.execute(query=query) @@ -484,7 +484,7 @@ def __init__(self, name: str, dsn: str, timeout=30000, autocommit=False): connect_timeout=timeout, ) self.autocommit = autocommit - self._connection = None # type: Optional[psycopg.Connection] + self._connection: psycopg.Connection | None = None @tenacity.retry( reraise=True, @@ -494,11 +494,15 @@ def __init__(self, name: str, dsn: str, timeout=30000, autocommit=False): after=tenacity.after_log(LOG, logging.DEBUG), ) def _connect_db(self): - LOG.info(f'Creating connection to PostgreSQL database "{self.name}"') + LOG.info('Creating connection to PostgreSQL database "%s"', self.name) try: - connection = psycopg.connect(conninfo=self.dsn, autocommit=self.autocommit) # type: psycopg.Connection + connection: psycopg.Connection = psycopg.connect( + conninfo=self.dsn, + autocommit=self.autocommit, + ) except Exception as e: - LOG.error(f'Failed to connect to PostgreSQL database "{self.name}": {str(e)}') + LOG.error('Failed to connect to PostgreSQL database "%s": %s', + self.name, str(e)) raise e # test connection cursor = connection.cursor() @@ -506,7 +510,7 @@ def _connect_db(self): result = cursor.fetchone() if result is None: raise RuntimeError('Failed to verify DB connection') - LOG.debug(f'DB connection verified (result={result[0]})') + LOG.debug('DB connection verified (result=%s)', result[0]) cursor.close() connection.commit() self._connection = connection @@ -532,6 +536,6 @@ def reset(self): def close(self): if self._connection: - LOG.info(f'Closing connection to PostgreSQL database "{self.name}"') + LOG.info('Closing connection to PostgreSQL database "%s"', self.name) self._connection.close() self._connection = None diff --git a/packages/dsw-database/dsw/database/model.py b/packages/dsw-database/dsw/database/model.py index 2461ee80..74358de8 100644 --- a/packages/dsw-database/dsw/database/model.py +++ b/packages/dsw-database/dsw/database/model.py @@ -2,8 +2,6 @@ import datetime import json -from typing import Optional - NULL_UUID = '00000000-0000-0000-0000-000000000000' @@ -55,8 +53,8 @@ class DBDocument: content_type: str worker_log: str created_by: str - retrieved_at: Optional[datetime.datetime] - finished_at: Optional[datetime.datetime] + retrieved_at: datetime.datetime | None + finished_at: datetime.datetime | None created_at: datetime.datetime tenant_uuid: str file_size: int @@ -191,11 +189,11 @@ class PersistentCommand: component: str function: str body: dict - last_error_message: Optional[str] + last_error_message: str | None attempts: int max_attempts: int tenant_uuid: str - created_by: Optional[str] + created_by: str | None created_at: datetime.datetime updated_at: datetime.datetime @@ -220,28 +218,28 @@ def from_dict_row(data: dict): @dataclasses.dataclass class DBTenantConfig: uuid: str - organization: Optional[dict] - authentication: Optional[dict] - privacy_and_support: Optional[dict] - dashboard: Optional[dict] - look_and_feel: Optional[dict] - registry: Optional[dict] - knowledge_model: Optional[dict] - questionnaire: Optional[dict] - submission: Optional[dict] - owl: Optional[dict] - mail_config_uuid: Optional[str] + organization: dict | None + authentication: dict | None + privacy_and_support: dict | None + dashboard: dict | None + look_and_feel: dict | None + registry: dict | None + knowledge_model: dict | None + questionnaire: dict | None + submission: dict | None + owl: dict | None + mail_config_uuid: str | None created_at: datetime.datetime updated_at: datetime.datetime @property - def app_title(self) -> Optional[str]: + def app_title(self) -> str | None: if self.look_and_feel is None: return None return self.look_and_feel.get('appTitle', None) @property - def support_email(self) -> Optional[str]: + def support_email(self) -> str | None: if self.privacy_and_support is None: return None return self.privacy_and_support.get('supportEmail', None) @@ -269,7 +267,7 @@ def from_dict_row(data: dict): @dataclasses.dataclass class DBTenantLimits: tenant_uuid: str - storage: Optional[int] + storage: int | None @staticmethod def from_dict_row(data: dict): @@ -390,19 +388,19 @@ class DBInstanceConfigMail: uuid: str enabled: bool provider: str - sender_name: Optional[str] - sender_email: Optional[str] - smtp_host: Optional[str] - smtp_port: Optional[int] - smtp_security: Optional[str] - smtp_username: Optional[str] - smtp_password: Optional[str] - aws_access_key_id: Optional[str] - aws_secret_access_key: Optional[str] - aws_region: Optional[str] - rate_limit_window: Optional[int] - rate_limit_count: Optional[int] - timeout: Optional[int] + sender_name: str | None + sender_email: str | None + smtp_host: str | None + smtp_port: int | None + smtp_security: str | None + smtp_username: str | None + smtp_password: str | None + aws_access_key_id: str | None + aws_secret_access_key: str | None + aws_region: str | None + rate_limit_window: int | None + rate_limit_count: int | None + timeout: int | None @staticmethod def from_dict_row(data: dict): diff --git a/packages/dsw-database/pyproject.toml b/packages/dsw-database/pyproject.toml index 0f208852..f7ec4038 100644 --- a/packages/dsw-database/pyproject.toml +++ b/packages/dsw-database/pyproject.toml @@ -16,12 +16,12 @@ classifiers = [ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Database', 'Topic :: Utilities', ] -requires-python = '>=3.10, <4' +requires-python = '>=3.11, <4' dependencies = [ 'psycopg[binary]', 'tenacity', diff --git a/packages/dsw-document-worker/dsw/document_worker/cli.py b/packages/dsw-document-worker/dsw/document_worker/cli.py index 700cddd5..ad6ebdbd 100644 --- a/packages/dsw-document-worker/dsw/document_worker/cli.py +++ b/packages/dsw-document-worker/dsw/document_worker/cli.py @@ -1,7 +1,8 @@ -import click import pathlib +import sys +import typing -from typing import IO, Optional +import click from dsw.config.parser import MissingConfigurationError from dsw.config.sentry import SentryReporter @@ -15,7 +16,7 @@ def load_config_str(config_str: str) -> DocumentWorkerConfig: parser = DocumentWorkerConfigParser() if not parser.can_read(config_str): click.echo('Error: Cannot parse config file', err=True) - exit(1) + sys.exit(1) try: parser.read_string(config_str) @@ -24,14 +25,14 @@ def load_config_str(config_str: str) -> DocumentWorkerConfig: click.echo('Error: Missing configuration', err=True) for missing_item in e.missing: click.echo(f' - {missing_item}', err=True) - exit(1) + sys.exit(1) config = parser.config config.log.apply() return config -def validate_config(ctx, param, value: Optional[IO]): +def validate_config(ctx, param, value: typing.IO | None): content = '' if value is not None: content = value.read() @@ -52,11 +53,11 @@ def main(config: DocumentWorkerConfig, workdir: str): workdir_path.mkdir(parents=True, exist_ok=True) if not workdir_path.is_dir(): click.echo(f'Workdir {workdir_path.as_posix()} is not usable') - exit(2) + sys.exit(2) try: worker = DocumentWorker(config, workdir_path) worker.run() except Exception as e: SentryReporter.capture_exception(e) click.echo(f'Ended with error: {e}') - exit(2) + sys.exit(2) diff --git a/packages/dsw-document-worker/dsw/document_worker/config.py b/packages/dsw-document-worker/dsw/document_worker/config.py index d124c794..af0b372e 100644 --- a/packages/dsw-document-worker/dsw/document_worker/config.py +++ b/packages/dsw-document-worker/dsw/document_worker/config.py @@ -1,5 +1,4 @@ import shlex -from typing import List, Optional from dsw.config import DSWConfigParser from dsw.config.keys import ConfigKey, ConfigKeys, ConfigKeysContainer, \ @@ -69,8 +68,7 @@ def __init__(self, naming_strategy: str): class ExperimentalConfig(ConfigModel): - def __init__(self, job_timeout: Optional[int], - max_doc_size: Optional[float]): + def __init__(self, job_timeout: int | None, max_doc_size: float | None): self.job_timeout = job_timeout self.max_doc_size = max_doc_size @@ -83,7 +81,7 @@ def __init__(self, executable: str, args: str, timeout: float): self.timeout = timeout @property - def command(self) -> List[str]: + def command(self) -> list[str]: return [self.executable] + shlex.split(self.args) @@ -105,7 +103,7 @@ def load(data: dict): class TemplateConfig: - def __init__(self, ids: List[str], requests: TemplateRequestsConfig, + def __init__(self, ids: list[str], requests: TemplateRequestsConfig, secrets: dict[str, str], send_sentry: bool): self.ids = ids self.requests = requests @@ -126,10 +124,10 @@ def load(data: dict): class TemplatesConfig: - def __init__(self, templates: List[TemplateConfig]): + def __init__(self, templates: list[TemplateConfig]): self.templates = templates - def get_config(self, template_id: str) -> Optional[TemplateConfig]: + def get_config(self, template_id: str) -> TemplateConfig | None: for template in self.templates: if any((template_id.startswith(prefix) for prefix in template.ids)): diff --git a/packages/dsw-document-worker/dsw/document_worker/context.py b/packages/dsw-document-worker/dsw/document_worker/context.py index 2862dfcc..130de901 100644 --- a/packages/dsw-document-worker/dsw/document_worker/context.py +++ b/packages/dsw-document-worker/dsw/document_worker/context.py @@ -1,11 +1,9 @@ import pathlib -from typing import Optional, TYPE_CHECKING +from dsw.database import Database +from dsw.storage import S3Storage -if TYPE_CHECKING: - from .config import DocumentWorkerConfig - from dsw.database import Database - from dsw.storage import S3Storage +from .config import DocumentWorkerConfig class ContextNotInitializedError(RuntimeError): @@ -16,11 +14,12 @@ def __init__(self): class AppContext: - def __init__(self, db, s3, cfg, workdir): - self.db = db # type: Database - self.s3 = s3 # type: S3Storage - self.cfg = cfg # type: DocumentWorkerConfig - self.workdir = workdir # type: pathlib.Path + def __init__(self, db: Database, s3: S3Storage, cfg: DocumentWorkerConfig, + workdir: pathlib.Path): + self.db = db + self.s3 = s3 + self.cfg = cfg + self.workdir = workdir class JobContext: @@ -49,7 +48,7 @@ def reset_ids(self): class Context: - _instance = None # type: Optional[_Context] + _instance: _Context | None = None @classmethod def get(cls) -> _Context: diff --git a/packages/dsw-document-worker/dsw/document_worker/conversions.py b/packages/dsw-document-worker/dsw/document_worker/conversions.py index efe79a01..0b5d0686 100644 --- a/packages/dsw-document-worker/dsw/document_worker/conversions.py +++ b/packages/dsw-document-worker/dsw/document_worker/conversions.py @@ -1,10 +1,11 @@ import logging import os import pathlib -import rdflib import shlex import subprocess +import rdflib + from .config import DocumentWorkerConfig from .consts import EXIT_SUCCESS, DEFAULT_ENCODING from .documents import FileFormat, FileFormats @@ -16,14 +17,12 @@ def run_conversion(*, args: list, workdir: str, input_data: bytes, name: str, source_format: FileFormat, target_format: FileFormat, timeout=None) -> bytes: command = ' '.join(args) - LOG.info(f'Calling "{command}" to convert from {source_format} to {target_format}') - p = subprocess.Popen(args, - cwd=workdir, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - stdout, stderr = p.communicate(input=input_data, timeout=timeout) - exit_code = p.returncode + LOG.info('Calling "%s" to convert from %s to %s', + command, source_format, target_format) + with subprocess.Popen(args, cwd=workdir, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) as proc: + stdout, stderr = proc.communicate(input=input_data, timeout=timeout) + exit_code = proc.returncode if exit_code != EXIT_SUCCESS: raise FormatConversionException( name, source_format, target_format, @@ -49,7 +48,8 @@ class Pandoc: FILTERS_PATH = pathlib.Path(os.getenv('PANDOC_FILTERS', '/pandoc/filters')) TEMPLATES_PATH = pathlib.Path(os.getenv('PANDOC_TEMPLATES', '/pandoc/templates')) - def __init__(self, config: DocumentWorkerConfig, filter_names: list[str], template_name: str | None): + def __init__(self, config: DocumentWorkerConfig, filter_names: list[str], + template_name: str | None): self.config = config self.filter_names = filter_names self.template_name = template_name diff --git a/packages/dsw-document-worker/dsw/document_worker/documents.py b/packages/dsw-document-worker/dsw/document_worker/documents.py index 17e6ed47..d967c4a5 100644 --- a/packages/dsw-document-worker/dsw/document_worker/documents.py +++ b/packages/dsw-document-worker/dsw/document_worker/documents.py @@ -1,8 +1,6 @@ import pathvalidate import slugify -from typing import Optional - from dsw.database.database import DBDocument from .consts import DEFAULT_ENCODING, DocumentNamingStrategy @@ -107,7 +105,7 @@ def get(name: str): class DocumentFile: def __init__(self, file_format: FileFormat, content: bytes, - encoding: Optional[str] = None): + encoding: str | None = None): self.file_format = file_format self._content = content self.byte_size = len(content) diff --git a/packages/dsw-document-worker/dsw/document_worker/exceptions.py b/packages/dsw-document-worker/dsw/document_worker/exceptions.py index 2baca785..cc2ff857 100644 --- a/packages/dsw-document-worker/dsw/document_worker/exceptions.py +++ b/packages/dsw-document-worker/dsw/document_worker/exceptions.py @@ -11,8 +11,7 @@ def __str__(self): def log_message(self): if self.exc is None: return self.msg - else: - return f'{self.msg}: [{type(self.exc).__name__}] {str(self.exc)}' + return f'{self.msg}: [{type(self.exc).__name__}] {str(self.exc)}' def db_message(self): if self.exc is None: diff --git a/packages/dsw-document-worker/dsw/document_worker/handlers.py b/packages/dsw-document-worker/dsw/document_worker/handlers.py index be7f439b..22c8e199 100644 --- a/packages/dsw-document-worker/dsw/document_worker/handlers.py +++ b/packages/dsw-document-worker/dsw/document_worker/handlers.py @@ -2,14 +2,15 @@ import pathlib from .cli import load_config_str -from .consts import VAR_APP_CONFIG_PATH, VAR_WORKDIR_PATH +from .consts import VAR_APP_CONFIG_PATH, VAR_WORKDIR_PATH, DEFAULT_ENCODING from .worker import DocumentWorker def lambda_handler(event, context): + # pylint: disable=unused-argument config_path = pathlib.Path(os.getenv(VAR_APP_CONFIG_PATH, '/var/task/application.yml')) workdir_path = pathlib.Path(os.getenv(VAR_WORKDIR_PATH, '/var/task/templates')) - config = load_config_str(config_path.read_text()) + config = load_config_str(config_path.read_text(encoding=DEFAULT_ENCODING)) doc_worker = DocumentWorker(config, workdir_path) doc_worker.run_once() diff --git a/packages/dsw-document-worker/dsw/document_worker/limits.py b/packages/dsw-document-worker/dsw/document_worker/limits.py index 797a3ee5..3a024fdf 100644 --- a/packages/dsw-document-worker/dsw/document_worker/limits.py +++ b/packages/dsw-document-worker/dsw/document_worker/limits.py @@ -2,8 +2,6 @@ from .exceptions import JobException from .utils import byte_size_format -from typing import Optional - class LimitsEnforcer: @@ -20,7 +18,7 @@ def check_doc_size(job_id: str, doc_size: int): @staticmethod def check_size_usage(job_id: str, doc_size: int, - used_size: int, limit_size: Optional[int]): + used_size: int, limit_size: int | None): if limit_size is None or doc_size + used_size < limit_size: return remains = limit_size - used_size diff --git a/packages/dsw-document-worker/dsw/document_worker/model/context.py b/packages/dsw-document-worker/dsw/document_worker/model/context.py index 43c46159..e4c724ed 100644 --- a/packages/dsw-document-worker/dsw/document_worker/model/context.py +++ b/packages/dsw-document-worker/dsw/document_worker/model/context.py @@ -1,12 +1,12 @@ # TODO: move to dsw-models import datetime -import dateutil.parser as dp +import typing -from typing import Optional, Iterable, Union, ItemsView +import dateutil.parser as dp from ..consts import NULL_UUID -AnnotationsT = dict[str, Union[str, list[str]]] +AnnotationsT = dict[str, str | list[str]] TODO_LABEL_UUID = "615b9028-5e3f-414f-b245-12d2ae2eeb20" @@ -15,12 +15,12 @@ def _datetime(timestamp: str) -> datetime.datetime: def _load_annotations(annotations: list[dict[str, str]]) -> AnnotationsT: - result = {} # type: AnnotationsT - semi_result = {} # type: dict[str, list[str]] + result: AnnotationsT = {} + semi_result: dict[str, list[str]] = {} for item in annotations: key = item.get('key', '') value = item.get('value', '') - if key in semi_result.keys(): + if key in semi_result: semi_result[key].append(value) else: semi_result[key] = [value] @@ -32,14 +32,94 @@ def _load_annotations(annotations: list[dict[str, str]]) -> AnnotationsT: return result +class SimpleAuthor: + + def __init__(self, uuid: str, first_name: str, last_name: str, + image_url: str | None, gravatar_hash: str | None): + self.uuid = uuid + self.first_name = first_name + self.last_name = last_name + self.image_url = image_url + self.gravatar_hash = gravatar_hash + + @staticmethod + def load(data: dict | None, **options): + if data is None: + return None + return SimpleAuthor( + uuid=data['uuid'], + first_name=data['firstName'], + last_name=data['lastName'], + image_url=data['imageUrl'], + gravatar_hash=data['gravatarHash'], + ) + + +class User: + + def __init__(self, uuid: str, first_name: str, last_name: str, email: str, + role: str, created_at: datetime.datetime, updated_at: datetime.datetime, + affiliation: str | None, permissions: list[str], sources: list[str], + image_url: str | None): + self.uuid = uuid + self.first_name = first_name + self.last_name = last_name + self.email = email + self.role = role + self.image_url = image_url + self.affiliation = affiliation + self.permissions = permissions + self.sources = sources + self.created_at = created_at + self.updated_at = updated_at + + @staticmethod + def load(data: dict, **options): + if data is None: + return None + return User( + uuid=data['uuid'], + first_name=data['firstName'], + last_name=data['lastName'], + email=data['email'], + role=data['role'], + image_url=data['imageUrl'], + affiliation=data['affiliation'], + permissions=data['permissions'], + sources=data['sources'], + created_at=_datetime(data['createdAt']), + updated_at=_datetime(data['updatedAt']), + ) + + +class Organization: + + def __init__(self, org_id: str, name: str, description: str | None, + affiliations: list[str]): + self.id = org_id + self.name = name + self.description = description + self.affiliations = affiliations + + @staticmethod + def load(data: dict, **options): + return Organization( + org_id=data['organizationId'], + name=data['name'], + description=data['description'], + affiliations=data['affiliations'], + ) + + class Tag: - def __init__(self, uuid, name, description, color, annotations): - self.uuid = uuid # type: str - self.name = name # type: str - self.description = description # type: Optional[str] - self.color = color # type: str - self.annotations = annotations # type: AnnotationsT + def __init__(self, uuid: str, name: str, description: str | None, + color: str, annotations: AnnotationsT): + self.uuid = uuid + self.name = name + self.description = description + self.color = color + self.annotations = annotations @property def a(self): @@ -63,17 +143,18 @@ def load(data: dict, **options): class ResourceCollection: - def __init__(self, uuid, title, page_uuids, annotations): - self.uuid = uuid # type: str - self.title = title # type: str - self.page_uuids = page_uuids # type: list[str] - self.pages = list() # type: list[ResourcePage] - self.annotations = annotations # type: AnnotationsT + def __init__(self, uuid: str, title: str, page_uuids: list[str], + annotations: AnnotationsT): + self.uuid = uuid + self.title = title + self.page_uuids = page_uuids + self.pages = [] + self.annotations = annotations - def _resolve_links(self, ctx): + def resolve_links(self, ctx): self.pages = [ctx.e.resource_pages[key] for key in self.page_uuids - if key in ctx.e.resource_pages.keys()] + if key in ctx.e.resource_pages] for page in self.pages: page.collection = self @@ -93,12 +174,14 @@ def load(data: dict, **options): class ResourcePage: - def __init__(self, uuid, title, content, annotations): - self.uuid = uuid # type: str - self.title = title # type: str - self.content = content # type: str - self.collection = None # type: Optional[ResourceCollection] - self.annotations = annotations # type: AnnotationsT + def __init__(self, uuid: str, title: str, content: str, + annotations: AnnotationsT): + self.uuid = uuid + self.title = title + self.content = content + self.annotations = annotations + + self.collection: ResourceCollection | None = None @property def a(self): @@ -116,22 +199,23 @@ def load(data: dict, **options): class Integration: - def __init__(self, uuid, name, logo, integration_id, item_url, props, - integration_type, annotations): - self.uuid = uuid # type: str - self.name = name # type: str - self.id = integration_id # type: str - self.item_url = item_url # type: Optional[str] - self.logo = logo # type: Optional[str] - self.props = props # type: list[str] - self.type = integration_type # type: str - self.annotations = annotations # type: AnnotationsT + def __init__(self, uuid: str, name: str, integration_id: str, + item_url: str | None, logo: str | None, props: list[str], + integration_type: str, annotations: AnnotationsT): + self.uuid = uuid + self.name = name + self.id = integration_id + self.item_url = item_url + self.logo = logo + self.props = props + self.type = integration_type + self.annotations = annotations @property def a(self): return self.annotations - def item(self, item_id: str) -> Optional[str]: + def item(self, item_id: str) -> str | None: if self.item_url is None: return None return self.item_url.replace('${id}', item_id) @@ -144,9 +228,12 @@ def __eq__(self, other): class ApiIntegration(Integration): - def __init__(self, uuid, name, logo, integration_id, item_url, props, rq_body, - rq_headers, rq_method, rq_url, rs_list_field, rs_item_id, - rs_item_template, annotations): + def __init__(self, uuid: str, name: str, integration_id: str, + item_url: str | None, logo: str | None, + props: list[str], rq_body: str, rq_method: str, + rq_headers: dict[str, str], rq_url: str, rs_list_field: str | None, + rs_item_id: str | None, rs_item_template: str, + annotations: AnnotationsT): super().__init__( uuid=uuid, name=name, @@ -157,13 +244,13 @@ def __init__(self, uuid, name, logo, integration_id, item_url, props, rq_body, annotations=annotations, integration_type='ApiIntegration', ) - self.rq_body = rq_body # type: str - self.rq_method = rq_method # type: str - self.rq_url = rq_url # type: str - self.rq_headers = rq_headers # type: dict[str, str] - self.rs_list_field = rs_list_field # type: Optional[str] - self.rs_item_id = rs_item_id # type: Optional[str] - self.rs_item_template = rs_item_template # type: str + self.rq_body = rq_body + self.rq_method = rq_method + self.rq_url = rq_url + self.rq_headers = rq_headers + self.rs_list_field = rs_list_field + self.rs_item_id = rs_item_id + self.rs_item_template = rs_item_template @staticmethod def default(): @@ -206,8 +293,9 @@ def load(data: dict, **options): class WidgetIntegration(Integration): - def __init__(self, uuid, name, logo, integration_id, item_url, props, - widget_url, annotations): + def __init__(self, uuid: str, name: str, integration_id: str, + item_url: str | None, logo: str | None, props: list[str], + widget_url: str, annotations: AnnotationsT): super().__init__( uuid=uuid, name=name, @@ -218,7 +306,7 @@ def __init__(self, uuid, name, logo, integration_id, item_url, props, annotations=annotations, integration_type='WidgetIntegration', ) - self.widget_url = widget_url # type: str + self.widget_url = widget_url @staticmethod def load(data: dict, **options): @@ -236,12 +324,13 @@ def load(data: dict, **options): class Phase: - def __init__(self, uuid, title, description, annotations, order=0): - self.uuid = uuid # type: str - self.title = title # type: str - self.description = description # type: Optional[str] - self.order = order # type: int - self.annotations = annotations # type: AnnotationsT + def __init__(self, uuid: str, title: str, description: str | None, + annotations: AnnotationsT, order: int = 0): + self.uuid = uuid + self.title = title + self.description = description + self.order = order + self.annotations = annotations @property def a(self): @@ -273,12 +362,13 @@ def load(data: dict, **options): class Metric: - def __init__(self, uuid, title, description, abbreviation, annotations): - self.uuid = uuid # type: str - self.title = title # type: str - self.description = description # type: Optional[str] - self.abbreviation = abbreviation # type: str - self.annotations = annotations # type: AnnotationsT + def __init__(self, uuid: str, title: str, description: str | None, + abbreviation: str, annotations: AnnotationsT): + self.uuid = uuid + self.title = title + self.description = description + self.abbreviation = abbreviation + self.annotations = annotations @property def a(self): @@ -302,14 +392,15 @@ def load(data: dict, **options): class MetricMeasure: - def __init__(self, measure, weight, metric_uuid): - self.measure = measure # type: float - self.weight = weight # type: float - self.metric_uuid = metric_uuid # type: str - self.metric = None # type: Optional[Metric] + def __init__(self, measure: float, weight: float, metric_uuid: str): + self.measure = measure + self.weight = weight + self.metric_uuid = metric_uuid + + self.metric: Metric | None = None - def _resolve_links(self, ctx): - if self.metric_uuid in ctx.e.metrics.keys(): + def resolve_links(self, ctx): + if self.metric_uuid in ctx.e.metrics: self.metric = ctx.e.metrics[self.metric_uuid] @staticmethod @@ -323,10 +414,10 @@ def load(data: dict, **options): class Reference: - def __init__(self, uuid, ref_type, annotations): - self.uuid = uuid # type: str - self.type = ref_type # type: str - self.annotations = annotations # type: AnnotationsT + def __init__(self, uuid: str, ref_type: str, annotations: AnnotationsT): + self.uuid = uuid + self.type = ref_type + self.annotations = annotations @property def a(self): @@ -337,16 +428,17 @@ def __eq__(self, other): return False return other.uuid == self.uuid - def _resolve_links(self, ctx): + def resolve_links(self, ctx): pass class CrossReference(Reference): - def __init__(self, uuid, target_uuid, description, annotations): + def __init__(self, uuid: str, target_uuid: str, description: str, + annotations: AnnotationsT): super().__init__(uuid, 'CrossReference', annotations) - self.target_uuid = target_uuid # type: str - self.description = description # type: str + self.target_uuid = target_uuid + self.description = description @staticmethod def load(data: dict, **options): @@ -360,10 +452,11 @@ def load(data: dict, **options): class URLReference(Reference): - def __init__(self, uuid, label, url, annotations): + def __init__(self, uuid: str, label: str, url: str, + annotations: AnnotationsT): super().__init__(uuid, 'URLReference', annotations) - self.label = label # type: str - self.url = url # type: str + self.label = label + self.url = url @staticmethod def load(data: dict, **options): @@ -377,13 +470,15 @@ def load(data: dict, **options): class ResourcePageReference(Reference): - def __init__(self, uuid, resource_page_uuid, annotations): + def __init__(self, uuid: str, resource_page_uuid: str | None, + annotations: AnnotationsT): super().__init__(uuid, 'ResourcePageReference', annotations) - self.resource_page_uuid = resource_page_uuid # type: Optional[str] - self.resource_page = None # type: Optional[ResourcePage] + self.resource_page_uuid = resource_page_uuid - def _resolve_links(self, ctx): - if self.resource_page_uuid in ctx.e.resource_pages.keys(): + self.resource_page: ResourcePage | None = None + + def resolve_links(self, ctx): + if self.resource_page_uuid in ctx.e.resource_pages: self.resource_page = ctx.e.resource_pages[self.resource_page_uuid] @staticmethod @@ -397,11 +492,12 @@ def load(data: dict, **options): class Expert: - def __init__(self, uuid, name, email, annotations): - self.uuid = uuid # type: str - self.name = name # type: str - self.email = email # type: str - self.annotations = annotations # type: AnnotationsT + def __init__(self, uuid: str, name: str, email: str, + annotations: AnnotationsT): + self.uuid = uuid + self.name = name + self.email = email + self.annotations = annotations @property def a(self): @@ -424,38 +520,42 @@ def load(data: dict, **options): class Reply: - def __init__(self, path, created_at, created_by, reply_type): - self.path = path # type: str - self.fragments = path.split('.') # type: list[str] - self.created_at = created_at # type: datetime.datetime - self.created_by = created_by # type: Optional[SimpleAuthor] - self.type = reply_type # type: str - self.question = None # type: Optional[Question] + def __init__(self, path: str, created_at: datetime.datetime, + created_by: SimpleAuthor | None, reply_type: str): + self.path = path + self.created_at = created_at + self.created_by = created_by + self.type = reply_type + + self.question: Question | None = None + self.fragments: list[str] = path.split('.') - def _resolve_links_parent(self, ctx): + def resolve_links_parent(self, ctx): question_uuid = self.fragments[-1] - if question_uuid in ctx.e.questions.keys(): + if question_uuid in ctx.e.questions: self.question = ctx.e.questions.get(question_uuid, None) if self.question is not None: self.question.replies[self.path] = self - def _resolve_links(self, ctx): + def resolve_links(self, ctx): pass class AnswerReply(Reply): - def __init__(self, path, created_at, created_by, answer_uuid): + def __init__(self, path: str, created_at: datetime.datetime, + created_by: SimpleAuthor | None, answer_uuid: str): super().__init__(path, created_at, created_by, 'AnswerReply') - self.answer_uuid = answer_uuid # type: str - self.answer = None # type: Optional[Answer] + self.answer_uuid = answer_uuid + + self.answer: Answer | None = None @property - def value(self) -> Optional[str]: + def value(self) -> str | None: return self.answer_uuid - def _resolve_links(self, ctx): - super()._resolve_links_parent(ctx) + def resolve_links(self, ctx): + super().resolve_links_parent(ctx) self.answer = ctx.e.answers.get(self.answer_uuid, None) @staticmethod @@ -470,26 +570,27 @@ def load(path: str, data: dict, **options): class StringReply(Reply): - def __init__(self, path, created_at, created_by, value): + def __init__(self, path: str, created_at: datetime.datetime, + created_by: SimpleAuthor | None, value: str): super().__init__(path, created_at, created_by, 'StringReply') - self.value = value # type: str + self.value = value @property - def as_number(self) -> Optional[float]: + def as_number(self) -> float | None: try: return float(self.value) except Exception: return None @property - def as_datetime(self) -> Optional[datetime.datetime]: + def as_datetime(self) -> datetime.datetime | None: try: return dp.parse(self.value) except Exception: return None - def _resolve_links(self, ctx): - super()._resolve_links_parent(ctx) + def resolve_links(self, ctx): + super().resolve_links_parent(ctx) @staticmethod def load(path: str, data: dict, **options): @@ -503,9 +604,10 @@ def load(path: str, data: dict, **options): class ItemListReply(Reply): - def __init__(self, path, created_at, created_by, items): + def __init__(self, path: str, created_at: datetime.datetime, + created_by: SimpleAuthor | None, items: list[str]): super().__init__(path, created_at, created_by, 'ItemListReply') - self.items = items # type: list[str] + self.items = items @property def value(self) -> list[str]: @@ -517,8 +619,8 @@ def __iter__(self): def __len__(self): return len(self.items) - def _resolve_links(self, ctx): - super()._resolve_links_parent(ctx) + def resolve_links(self, ctx): + super().resolve_links_parent(ctx) @staticmethod def load(path: str, data: dict, **options): @@ -532,10 +634,12 @@ def load(path: str, data: dict, **options): class MultiChoiceReply(Reply): - def __init__(self, path, created_at, created_by, choice_uuids): + def __init__(self, path: str, created_at: datetime.datetime, + created_by: SimpleAuthor | None, choice_uuids: list[str]): super().__init__(path, created_at, created_by, 'MultiChoiceReply') - self.choice_uuids = choice_uuids # type: list[str] - self.choices = list() # type: list[Choice] + self.choice_uuids = choice_uuids + + self.choices: list[Choice] = [] @property def value(self) -> list[str]: @@ -547,11 +651,11 @@ def __iter__(self): def __len__(self): return len(self.choices) - def _resolve_links(self, ctx): - super()._resolve_links_parent(ctx) + def resolve_links(self, ctx): + super().resolve_links_parent(ctx) self.choices = [ctx.e.choices[key] for key in self.choice_uuids - if key in ctx.e.choices.keys()] + if key in ctx.e.choices] @staticmethod def load(path: str, data: dict, **options): @@ -565,13 +669,14 @@ def load(path: str, data: dict, **options): class IntegrationReply(Reply): - def __init__(self, path, created_at, created_by, item_id, value): + def __init__(self, path: str, created_at: datetime.datetime, + created_by: SimpleAuthor | None, item_id: str | None, value: str): super().__init__(path, created_at, created_by, 'IntegrationReply') - self.item_id = item_id # type: Optional[str] - self.value = value # type: str + self.item_id = item_id + self.value = value @property - def id(self) -> Optional[str]: + def id(self) -> str | None: return self.item_id @property @@ -583,7 +688,7 @@ def is_integration(self) -> bool: return not self.is_plain @property - def url(self) -> Optional[str]: + def url(self) -> str | None: if not self.is_integration or self.item_id is None: return None if isinstance(self.question, IntegrationQuestion) \ @@ -591,8 +696,8 @@ def url(self) -> Optional[str]: return self.question.integration.item(self.item_id) return None - def _resolve_links(self, ctx): - super()._resolve_links_parent(ctx) + def resolve_links(self, ctx): + super().resolve_links_parent(ctx) @staticmethod def load(path: str, data: dict, **options): @@ -607,17 +712,18 @@ def load(path: str, data: dict, **options): class ItemSelectReply(Reply): - def __init__(self, path, created_at, created_by, item_uuid): + def __init__(self, path: str, created_at: datetime.datetime, + created_by: SimpleAuthor | None, item_uuid: str | None): super().__init__(path, created_at, created_by, 'ItemSelectReply') - self.item_uuid = item_uuid # type: str - self.item_title = 'Item' # type: str + self.item_uuid = item_uuid + self.item_title: str = 'Item' @property def value(self) -> str: return self.item_uuid - def _resolve_links(self, ctx): - super()._resolve_links_parent(ctx) + def resolve_links(self, ctx): + super().resolve_links_parent(ctx) @staticmethod def load(path: str, data: dict, **options): @@ -631,16 +737,18 @@ def load(path: str, data: dict, **options): class Answer: - def __init__(self, uuid, label, advice, metric_measures, followup_uuids, - annotations): - self.uuid = uuid # type: str - self.label = label # type: str - self.advice = advice # type: Optional[str] - self.metric_measures = metric_measures # type: list[MetricMeasure] - self.followup_uuids = followup_uuids # type: list[str] - self.followups = list() # type: list[Question] - self.parent = None # type: Optional[OptionsQuestion] - self.annotations = annotations # type: AnnotationsT + def __init__(self, uuid: str, label: str, advice: str | None, + metric_measures: list[MetricMeasure], followup_uuids: list[str], + annotations: AnnotationsT): + self.uuid = uuid + self.label = label + self.advice = advice + self.metric_measures = metric_measures + self.followup_uuids = followup_uuids + self.annotations = annotations + + self.followups: list[Question] = [] + self.parent: OptionsQuestion | None = None @property def a(self): @@ -651,15 +759,15 @@ def __eq__(self, other): return False return other.uuid == self.uuid - def _resolve_links(self, ctx): + def resolve_links(self, ctx): self.followups = [ctx.e.questions[key] for key in self.followup_uuids - if key in ctx.e.questions.keys()] + if key in ctx.e.questions] for followup in self.followups: followup.parent = self - followup._resolve_links(ctx) + followup.resolve_links(ctx) for mm in self.metric_measures: - mm._resolve_links(ctx) + mm.resolve_links(ctx) @staticmethod def load(data: dict, **options): @@ -677,11 +785,12 @@ def load(data: dict, **options): class Choice: - def __init__(self, uuid, label, annotations): - self.uuid = uuid # type: str - self.label = label # type: str - self.parent = None # type: Optional[MultiChoiceQuestion] - self.annotations = annotations # type: AnnotationsT + def __init__(self, uuid: str, label: str, annotations: AnnotationsT): + self.uuid = uuid + self.label = label + self.annotations = annotations + + self.parent: MultiChoiceQuestion | None = None @property def a(self): @@ -703,24 +812,27 @@ def load(data: dict, **options): class Question: - def __init__(self, uuid, q_type, title, text, tag_uuids, reference_uuids, - expert_uuids, required_phase_uuid, annotations): - self.uuid = uuid # type: str - self.type = q_type # type: str - self.title = title # type: str - self.text = text # type: Optional[str] - self.tag_uuids = tag_uuids # type: list[str] - self.tags = list() # type: list[Tag] - self.reference_uuids = reference_uuids # type: list[str] - self.references = list() # type: list[Reference] - self.expert_uuids = expert_uuids # type: list[str] - self.experts = list() # type: list[Expert] - self.required_phase_uuid = required_phase_uuid # type: Optional[str] - self.required_phase = PHASE_NEVER # type: Phase - self.replies = dict() # type: dict[str, Reply] # added from replies - self.is_required = None # type: Optional[bool] - self.parent = None # type: Optional[Union[Chapter, ListQuestion, Answer]] - self.annotations = annotations # type: AnnotationsT + def __init__(self, uuid: str, q_type: str, title: str, text: str | None, + tag_uuids: list[str], reference_uuids: list[str], + expert_uuids: list[str], required_phase_uuid: str | None, + annotations: AnnotationsT): + self.uuid = uuid + self.type = q_type + self.title = title + self.text = text + self.tag_uuids = tag_uuids + self.reference_uuids = reference_uuids + self.expert_uuids = expert_uuids + self.required_phase_uuid = required_phase_uuid + self.annotations = annotations + + self.is_required: bool | None = None + self.parent: Chapter | ListQuestion | Answer | None = None + self.replies: dict[str, Reply] = {} + self.tags: list[Tag] = [] + self.references: list[Reference] = [] + self.experts: list[Expert] = [] + self.required_phase: Phase = PHASE_NEVER @property def a(self): @@ -731,25 +843,25 @@ def __eq__(self, other): return False return other.uuid == self.uuid - def _resolve_links_parent(self, ctx): + def resolve_links_parent(self, ctx): self.tags = [ctx.e.tags[key] for key in self.tag_uuids - if key in ctx.e.tags.keys()] + if key in ctx.e.tags] self.experts = [ctx.e.experts[key] for key in self.expert_uuids - if key in ctx.e.experts.keys()] + if key in ctx.e.experts] self.references = [ctx.e.references[key] for key in self.reference_uuids - if key in ctx.e.references.keys()] + if key in ctx.e.references] for ref in self.references: - ref._resolve_links(ctx) + ref.resolve_links(ctx) if self.required_phase_uuid is None or ctx.current_phase is None: self.is_required = False else: self.required_phase = ctx.e.phases.get(self.required_phase_uuid, PHASE_NEVER) self.is_required = ctx.current_phase.order >= self.required_phase.order - def _resolve_links(self, ctx): + def resolve_links(self, ctx): pass @property @@ -767,12 +879,14 @@ def cross_references(self) -> list[CrossReference]: class ValueQuestion(Question): - def __init__(self, uuid, title, text, tag_uuids, reference_uuids, - expert_uuids, required_phase_uuid, value_type, annotations): + def __init__(self, uuid: str, title: str, text: str | None, + tag_uuids: list[str], reference_uuids: list[str], + expert_uuids: list[str], required_phase_uuid: str | None, + value_type: str, annotations: AnnotationsT): super().__init__(uuid, 'ValueQuestion', title, text, tag_uuids, reference_uuids, expert_uuids, required_phase_uuid, annotations) - self.value_type = value_type # type: str + self.value_type = value_type @property def a(self): @@ -814,8 +928,8 @@ def is_datetime(self): def is_date(self): return self.value_type == 'DateQuestionValueType' - def _resolve_links(self, ctx): - super()._resolve_links_parent(ctx) + def resolve_links(self, ctx): + super().resolve_links_parent(ctx) @staticmethod def load(data: dict, **options): @@ -834,23 +948,25 @@ def load(data: dict, **options): class OptionsQuestion(Question): - def __init__(self, uuid, title, text, tag_uuids, reference_uuids, - expert_uuids, required_phase_uuid, answer_uuids, - annotations): + def __init__(self, uuid: str, title: str, text: str | None, + tag_uuids: list[str], reference_uuids: list[str], + expert_uuids: list[str], required_phase_uuid: str | None, + answer_uuids: list[str], annotations: AnnotationsT): super().__init__(uuid, 'OptionsQuestion', title, text, tag_uuids, reference_uuids, expert_uuids, required_phase_uuid, annotations) - self.answer_uuids = answer_uuids # type: list[str] - self.answers = list() # type: list[Answer] + self.answer_uuids = answer_uuids + + self.answers: list[Answer] = [] - def _resolve_links(self, ctx): - super()._resolve_links_parent(ctx) + def resolve_links(self, ctx): + super().resolve_links_parent(ctx) self.answers = [ctx.e.answers[key] for key in self.answer_uuids - if key in ctx.e.answers.keys()] + if key in ctx.e.answers] for answer in self.answers: answer.parent = self - answer._resolve_links(ctx) + answer.resolve_links(ctx) @staticmethod def load(data: dict, **options): @@ -869,20 +985,22 @@ def load(data: dict, **options): class MultiChoiceQuestion(Question): - def __init__(self, uuid, title, text, tag_uuids, reference_uuids, - expert_uuids, required_phase_uuid, choice_uuids, - annotations): + def __init__(self, uuid: str, title: str, text: str | None, + tag_uuids: list[str], reference_uuids: list[str], + expert_uuids: list[str], required_phase_uuid: str | None, + choice_uuids: list[str], annotations: AnnotationsT): super().__init__(uuid, 'MultiChoiceQuestion', title, text, tag_uuids, reference_uuids, expert_uuids, required_phase_uuid, annotations) - self.choice_uuids = choice_uuids # type: list[str] - self.choices = list() # type: list[Choice] + self.choice_uuids = choice_uuids - def _resolve_links(self, ctx): - super()._resolve_links_parent(ctx) + self.choices: list[Choice] = [] + + def resolve_links(self, ctx): + super().resolve_links_parent(ctx) self.choices = [ctx.e.choices[key] for key in self.choice_uuids - if key in ctx.e.choices.keys()] + if key in ctx.e.choices] for choice in self.choices: choice.parent = self @@ -903,23 +1021,25 @@ def load(data: dict, **options): class ListQuestion(Question): - def __init__(self, uuid, title, text, tag_uuids, reference_uuids, - expert_uuids, required_phase_uuid, followup_uuids, - annotations): + def __init__(self, uuid: str, title: str, text: str, + tag_uuids: list[str], reference_uuids: list[str], + expert_uuids: list[str], required_phase_uuid: str | None, + followup_uuids: list[str], annotations: AnnotationsT): super().__init__(uuid, 'ListQuestion', title, text, tag_uuids, reference_uuids, expert_uuids, required_phase_uuid, annotations) - self.followup_uuids = followup_uuids # type: list[str] - self.followups = list() # type: list[Question] + self.followup_uuids = followup_uuids + + self.followups: list[Question] = [] - def _resolve_links(self, ctx): - super()._resolve_links_parent(ctx) + def resolve_links(self, ctx): + super().resolve_links_parent(ctx) self.followups = [ctx.e.questions[key] for key in self.followup_uuids - if key in ctx.e.questions.keys()] + if key in ctx.e.questions] for followup in self.followups: followup.parent = self - followup._resolve_links(ctx) + followup.resolve_links(ctx) @staticmethod def load(data: dict, **options): @@ -938,18 +1058,21 @@ def load(data: dict, **options): class IntegrationQuestion(Question): - def __init__(self, uuid, title, text, tag_uuids, reference_uuids, props, - expert_uuids, required_phase_uuid, integration_uuid, - annotations): + def __init__(self, uuid: str, title: str, text: str | None, + tag_uuids: list[str], reference_uuids: list[str], + expert_uuids: list[str], required_phase_uuid: str | None, + integration_uuid: str | None, props: dict[str, str], + annotations: AnnotationsT): super().__init__(uuid, 'IntegrationQuestion', title, text, tag_uuids, reference_uuids, expert_uuids, required_phase_uuid, annotations) - self.props = props # type: dict[str, str] - self.integration_uuid = integration_uuid # type: str - self.integration = None # type: Optional[Integration] + self.props = props + self.integration_uuid = integration_uuid - def _resolve_links(self, ctx): - super()._resolve_links_parent(ctx) + self.integration: Integration | None = None + + def resolve_links(self, ctx): + super().resolve_links_parent(ctx) self.integration = ctx.e.integrations.get( self.integration_uuid, ApiIntegration.default(), @@ -973,17 +1096,18 @@ def load(data: dict, **options): class ItemSelectQuestion(Question): - def __init__(self, uuid, title, text, tag_uuids, reference_uuids, - expert_uuids, required_phase_uuid, list_question_uuid, - annotations): + def __init__(self, uuid: str, title: str, text: str | None, + tag_uuids: list[str], reference_uuids: list[str], + expert_uuids: list[str], required_phase_uuid: str | None, + list_question_uuid: str | None, annotations: AnnotationsT): super().__init__(uuid, 'ItemSelectQuestion', title, text, tag_uuids, reference_uuids, expert_uuids, required_phase_uuid, annotations) - self.list_question_uuid = list_question_uuid # type: str - self.list_question = None # type: Optional[ListQuestion] + self.list_question_uuid = list_question_uuid + self.list_question = None - def _resolve_links(self, ctx): - super()._resolve_links_parent(ctx) + def resolve_links(self, ctx): + super().resolve_links_parent(ctx) self.list_question = ctx.e.questions.get(self.list_question_uuid, None) @staticmethod @@ -1003,14 +1127,16 @@ def load(data: dict, **options): class Chapter: - def __init__(self, uuid, title, text, question_uuids, annotations): - self.uuid = uuid # type: str - self.title = title # type: str - self.text = text # type: Optional[str] - self.question_uuids = question_uuids # type: list[str] - self.questions = list() # type: list[Question] - self.reports = list() # type: list[ReportItem] - self.annotations = annotations # type: AnnotationsT + def __init__(self, uuid: str, title: str, text: str | None, + question_uuids: list[str], annotations: AnnotationsT): + self.uuid = uuid + self.title = title + self.text = text + self.question_uuids = question_uuids + self.annotations = annotations + + self.questions: list[Question] = [] + self.reports: list[ReportItem] = [] @property def a(self): @@ -1021,13 +1147,13 @@ def __eq__(self, other): return False return other.uuid == self.uuid - def _resolve_links(self, ctx): + def resolve_links(self, ctx): self.questions = [ctx.e.questions[key] for key in self.question_uuids - if key in ctx.e.questions.keys()] + if key in ctx.e.questions] for question in self.questions: question.parent = self - question._resolve_links(ctx) + question.resolve_links(ctx) @staticmethod def load(data: dict, **options): @@ -1093,18 +1219,18 @@ def _load_reply(path: str, data: dict, **options): class KnowledgeModelEntities: def __init__(self): - self.chapters = dict() # type: dict[str, Chapter] - self.questions = dict() # type: dict[str, Question] - self.answers = dict() # type: dict[str, Answer] - self.choices = dict() # type: dict[str, Choice] - self.resource_collections = dict() # type: dict[str, ResourceCollection] - self.resource_pages = dict() # type: dict[str, ResourcePage] - self.references = dict() # type: dict[str, Reference] - self.experts = dict() # type: dict[str, Expert] - self.tags = dict() # type: dict[str, Tag] - self.metrics = dict() # type: dict[str, Metric] - self.phases = dict() # type: dict[str, Phase] - self.integrations = dict() # type: dict[str, Integration] + self.chapters: dict[str, Chapter] = {} + self.questions: dict[str, Question] = {} + self.answers: dict[str, Answer] = {} + self.choices: dict[str, Choice] = {} + self.resource_collections: dict[str, ResourceCollection] = {} + self.resource_pages: dict[str, ResourcePage] = {} + self.references: dict[str, Reference] = {} + self.experts: dict[str, Expert] = {} + self.tags: dict[str, Tag] = {} + self.metrics: dict[str, Metric] = {} + self.phases: dict[str, Phase] = {} + self.integrations: dict[str, Integration] = {} @staticmethod def load(data: dict, **options): @@ -1138,24 +1264,26 @@ def load(data: dict, **options): class KnowledgeModel: - def __init__(self, uuid, chapter_uuids, tag_uuids, metric_uuids, - phase_uuids, integration_uuids, resource_collection_uuids, - entities, annotations): - self.uuid = uuid # type: str - self.entities = entities # type: KnowledgeModelEntities - self.chapter_uuids = chapter_uuids # type: list[str] - self.chapters = list() # type: list[Chapter] - self.tag_uuids = tag_uuids # type: list[str] - self.tags = list() # type: list[Tag] - self.metric_uuids = metric_uuids # type: list[str] - self.metrics = list() # type: list[Metric] - self.phase_uuids = phase_uuids # type: list[str] - self.phases = list() # type: list[Phase] - self.resource_collection_uuids = resource_collection_uuids # type: list[str] - self.resource_collections = list() # type: list[ResourceCollection] - self.integration_uuids = integration_uuids # type: list[str] - self.integrations = list() # type: list[Integration] - self.annotations = annotations # type: AnnotationsT + def __init__(self, uuid: str, chapter_uuids: list[str], tag_uuids: list[str], + metric_uuids: list[str], phase_uuids: list[str], integration_uuids: list[str], + resource_collection_uuids: list[str], entities: KnowledgeModelEntities, + annotations: AnnotationsT): + self.uuid = uuid + self.entities = entities + self.chapter_uuids = chapter_uuids + self.tag_uuids = tag_uuids + self.metric_uuids = metric_uuids + self.phase_uuids = phase_uuids + self.resource_collection_uuids = resource_collection_uuids + self.integration_uuids = integration_uuids + self.annotations = annotations + + self.chapters: list[Chapter] = [] + self.tags: list[Tag] = [] + self.metrics: list[Metric] = [] + self.phases: list[Phase] = [] + self.resource_collections: list[ResourceCollection] = [] + self.integrations: list[Integration] = [] @property def a(self): @@ -1165,31 +1293,31 @@ def a(self): def e(self): return self.entities - def _resolve_links(self, ctx): + def resolve_links(self, ctx): self.chapters = [ctx.e.chapters[key] for key in self.chapter_uuids - if key in ctx.e.chapters.keys()] + if key in ctx.e.chapters] self.tags = [ctx.e.tags[key] for key in self.tag_uuids - if key in ctx.e.tags.keys()] + if key in ctx.e.tags] self.metrics = [ctx.e.metrics[key] for key in self.metric_uuids - if key in ctx.e.metrics.keys()] + if key in ctx.e.metrics] self.phases = [ctx.e.phases[key] for key in self.phase_uuids - if key in ctx.e.phases.keys()] + if key in ctx.e.phases] self.resource_collections = [ctx.e.resource_collections[key] for key in self.resource_collection_uuids - if key in ctx.e.resource_collections.keys()] + if key in ctx.e.resource_collections] self.integrations = [ctx.e.integrations[key] for key in self.integration_uuids - if key in ctx.e.integrations.keys()] + if key in ctx.e.integrations] for index, phase in enumerate(self.phases, start=1): phase.order = index for chapter in self.chapters: - chapter._resolve_links(ctx) + chapter.resolve_links(ctx) for resource_collection in self.resource_collections: - resource_collection._resolve_links(ctx) + resource_collection.resolve_links(ctx) @staticmethod def load(data: dict, **options): @@ -1208,8 +1336,8 @@ def load(data: dict, **options): class ContextConfig: - def __init__(self, client_url): - self.client_url = client_url # type: str + def __init__(self, client_url: str | None): + self.client_url = client_url @staticmethod def load(data: dict, **options): @@ -1220,14 +1348,14 @@ def load(data: dict, **options): class Document: - def __init__(self, uuid, name, document_template_id, format_uuid, - created_by, created_at): - self.uuid = uuid # type: str - self.name = name # type: str - self.document_template_id = document_template_id # type: str - self.format_uuid = format_uuid # type: str - self.created_by = created_by # type: Optional[User] - self.created_at = created_at # type: datetime.datetime + def __init__(self, uuid: str, name: str, document_template_id: str, format_uuid: str, + created_by: User | None, created_at: datetime.datetime): + self.uuid = uuid + self.name = name + self.document_template_id = document_template_id + self.format_uuid = format_uuid + self.created_by = created_by + self.created_at = created_at @staticmethod def load(data: dict, **options): @@ -1241,39 +1369,18 @@ def load(data: dict, **options): ) -class SimpleAuthor: - - def __init__(self, uuid, first_name, last_name, image_url, gravatar_hash): - self.uuid = uuid # type: str - self.first_name = first_name # type: str - self.last_name = last_name # type: str - self.image_url = image_url # type: Optional[str] - self.gravatar_hash = gravatar_hash # type: Optional[str] - - @staticmethod - def load(data: Optional[dict], **options): - if data is None: - return None - return SimpleAuthor( - uuid=data['uuid'], - first_name=data['firstName'], - last_name=data['lastName'], - image_url=data['imageUrl'], - gravatar_hash=data['gravatarHash'], - ) - - class QuestionnaireVersion: - def __init__(self, uuid, event_uuid, name, description, - created_at, updated_at, created_by): - self.uuid = uuid # type: str - self.event_uuid = event_uuid # type: str - self.name = name # type: str - self.description = description # type: str - self.created_at = created_at # type: datetime.datetime - self.updated_at = updated_at # type: datetime.datetime - self.created_by = created_by # type: Optional[SimpleAuthor] + def __init__(self, uuid: str, event_uuid: str, name: str, description: str | None, + created_at: datetime.datetime, updated_at: datetime.datetime, + created_by: SimpleAuthor | None): + self.uuid = uuid + self.event_uuid = event_uuid + self.name = name + self.description = description + self.created_at = created_at + self.updated_at = updated_at + self.created_by = created_by @staticmethod def load(data: dict, **options): @@ -1293,52 +1400,55 @@ class RepliesContainer: def __init__(self, replies: dict[str, Reply]): self.replies = replies - def __getitem__(self, path: str) -> Optional[Reply]: + def __getitem__(self, path: str) -> Reply | None: return self.get(path) def __len__(self) -> int: return len(self.replies) - def get(self, path: str, default=None) -> Optional[Reply]: + def get(self, path: str, default=None) -> Reply | None: return self.replies.get(path, default) - def iterate_by_prefix(self, path_prefix: str) -> Iterable[Reply]: + def iterate_by_prefix(self, path_prefix: str) -> typing.Iterable[Reply]: return (r for path, r in self.replies.items() if path.startswith(path_prefix)) - def iterate_by_suffix(self, path_suffix: str) -> Iterable[Reply]: + def iterate_by_suffix(self, path_suffix: str) -> typing.Iterable[Reply]: return (r for path, r in self.replies.items() if path.endswith(path_suffix)) - def values(self) -> Iterable[Reply]: + def values(self) -> typing.Iterable[Reply]: return self.replies.values() - def keys(self) -> Iterable[str]: - return self.replies.keys() + def keys(self) -> typing.Iterable[str]: + return self.replies - def items(self) -> ItemsView[str, Reply]: + def items(self) -> typing.ItemsView[str, Reply]: return self.replies.items() class Questionnaire: - def __init__(self, uuid, name, description, created_by, phase_uuid, - created_at, updated_at): - self.uuid = uuid # type: str - self.name = name # type: str - self.description = description # type: str - self.version = None # type: Optional[QuestionnaireVersion] - self.versions = list() # type: list[QuestionnaireVersion] - self.todos = list() # type: list[str] - self.created_by = created_by # type: User - self.phase_uuid = phase_uuid # type: Optional[str] - self.phase = PHASE_NEVER # type: Phase - self.project_tags = list() # type: list[str] - self.replies = RepliesContainer(dict()) # type: RepliesContainer + def __init__(self, uuid: str, name: str, description: str | None, + created_by: User, phase_uuid: str | None, + created_at: datetime.datetime, updated_at: datetime.datetime): + self.uuid = uuid + self.name = name + self.description = description + self.created_by = created_by + self.phase_uuid = phase_uuid + self.project_tags = [] self.created_at = created_at self.updated_at = updated_at - def _resolve_links(self, ctx): + self.version: QuestionnaireVersion | None = None + self.versions: list[QuestionnaireVersion] = [] + self.todos: list[str] = [] + self.phase: Phase = PHASE_NEVER + + self.replies: RepliesContainer = RepliesContainer(replies={}) + + def resolve_links(self, ctx): for reply in self.replies.values(): - reply._resolve_links(ctx) + reply.resolve_links(ctx) @staticmethod def load(data: dict, **options): @@ -1369,15 +1479,17 @@ def load(data: dict, **options): class Package: - def __init__(self, org_id, km_id, version, versions, name, description, created_at): - self.organization_id = org_id # type: str - self.km_id = km_id # type: str - self.version = version # type: str - self.id = f'{org_id}:{km_id}:{version}' - self.versions = versions # type: list[str] - self.name = name # type: str - self.description = description # type: str - self.created_at = created_at # type: datetime.datetime + def __init__(self, org_id: str, km_id: str, version: str, versions: list[str], + name: str, description: str, created_at: datetime.datetime): + self.organization_id = org_id + self.km_id = km_id + self.version = version + self.versions = versions + self.name = name + self.description = description + self.created_at = created_at + + self.id: str = f'{org_id}:{km_id}:{version}' @property def org_id(self): @@ -1398,10 +1510,10 @@ def load(data: dict, **options): class ReportIndication: - def __init__(self, indication_type, answered, unanswered): - self.indication_type = indication_type # type: str - self.answered = answered # type: int - self.unanswered = unanswered # type: int + def __init__(self, indication_type: str, answered: int, unanswered: int): + self.indication_type = indication_type + self.answered = answered + self.unanswered = unanswered @property def total(self) -> int: @@ -1432,13 +1544,14 @@ def load(data: dict, **options): class ReportMetric: - def __init__(self, measure, metric_uuid): - self.measure = measure # type: float - self.metric_uuid = metric_uuid # type: str - self.metric = None # type: Optional[Metric] + def __init__(self, measure: float, metric_uuid: str): + self.measure = measure + self.metric_uuid = metric_uuid + + self.metric: Metric | None = None - def _resolve_links(self, ctx): - if self.metric_uuid in ctx.e.metrics.keys(): + def resolve_links(self, ctx): + if self.metric_uuid in ctx.e.metrics: self.metric = ctx.e.metrics[self.metric_uuid] @staticmethod @@ -1451,16 +1564,18 @@ def load(data: dict, **options): class ReportItem: - def __init__(self, indications, metrics, chapter_uuid): - self.indications = indications # type: list[ReportIndication] - self.metrics = metrics # type: list[ReportMetric] - self.chapter_uuid = chapter_uuid # type: Optional[str] - self.chapter = None # type: Optional[Chapter] + def __init__(self, indications: list[ReportIndication], metrics: list[ReportMetric], + chapter_uuid: str | None): + self.indications = indications + self.metrics = metrics + self.chapter_uuid = chapter_uuid - def _resolve_links(self, ctx): + self.chapter: Chapter | None = None + + def resolve_links(self, ctx): for m in self.metrics: - m._resolve_links(ctx) - if self.chapter_uuid is not None and self.chapter_uuid in ctx.e.chapters.keys(): + m.resolve_links(ctx) + if self.chapter_uuid is not None and self.chapter_uuid in ctx.e.chapters: self.chapter = ctx.e.chapters[self.chapter_uuid] if self.chapter is not None: self.chapter.reports.append(self) @@ -1478,17 +1593,19 @@ def load(data: dict, **options): class Report: - def __init__(self, uuid, created_at, updated_at, chapter_reports, total_report): - self.uuid = uuid # type: str - self.created_at = created_at # type: datetime.datetime - self.updated_at = updated_at # type: datetime.datetime - self.total_report = total_report # type: ReportItem - self.chapter_reports = chapter_reports # type: list[ReportItem] + def __init__(self, uuid: str, created_at: datetime.datetime, + updated_at: datetime.datetime, chapter_reports: list[ReportItem], + total_report: ReportItem): + self.uuid = uuid + self.created_at = created_at + self.updated_at = updated_at + self.total_report = total_report + self.chapter_reports = chapter_reports - def _resolve_links(self, ctx): - self.total_report._resolve_links(ctx) + def resolve_links(self, ctx): + self.total_report.resolve_links(ctx) for report in self.chapter_reports: - report._resolve_links(ctx) + report.resolve_links(ctx) @staticmethod def load(data: dict, **options): @@ -1502,70 +1619,18 @@ def load(data: dict, **options): ) -class User: - - def __init__(self, uuid, first_name, last_name, email, role, created_at, - updated_at, affiliation, permissions, sources, image_url): - self.uuid = uuid # type: str - self.first_name = first_name # type: str - self.last_name = last_name # type: str - self.email = email # type: str - self.role = role # type: str - self.image_url = image_url # type: Optional[str] - self.affiliation = affiliation # type: Optional[str] - self.permissions = permissions # type: list[str] - self.sources = sources # type: list[str] - self.created_at = created_at # type: datetime.datetime - self.updated_at = updated_at # type: datetime.datetime - - @staticmethod - def load(data: dict, **options): - if data is None: - return None - return User( - uuid=data['uuid'], - first_name=data['firstName'], - last_name=data['lastName'], - email=data['email'], - role=data['role'], - image_url=data['imageUrl'], - affiliation=data['affiliation'], - permissions=data['permissions'], - sources=data['sources'], - created_at=_datetime(data['createdAt']), - updated_at=_datetime(data['updatedAt']), - ) - - -class Organization: - - def __init__(self, org_id, name, description, affiliations): - self.id = org_id # type: str - self.name = name # type: str - self.description = description # type: Optional[str] - self.affiliations = affiliations # type: list[str] - - @staticmethod - def load(data: dict, **options): - return Organization( - org_id=data['organizationId'], - name=data['name'], - description=data['description'], - affiliations=data['affiliations'], - ) - - class UserGroup: - def __init__(self, uuid, name, description, private, - created_at, updated_at): - self.uuid = uuid # type: str - self.name = name # type: str - self.description = description # type: Optional[str] - self.private = private # type: bool - self.members = list() # type: list[UserGroupMember] - self.created_at = created_at # type: datetime.datetime - self.updated_at = updated_at # type: datetime.datetime + def __init__(self, uuid: str, name: str, description: str | None, private: bool, + created_at: datetime.datetime, updated_at: datetime.datetime): + self.uuid = uuid + self.name = name + self.description = description + self.private = private + self.created_at = created_at + self.updated_at = updated_at + + self.members: list[UserGroupMember] = [] @staticmethod def load(data: dict, **options): @@ -1584,14 +1649,14 @@ def load(data: dict, **options): class UserGroupMember: - def __init__(self, uuid, first_name, last_name, gravatar_hash, - image_url, membership_type): - self.uuid = uuid # type: str - self.first_name = first_name # type: str - self.last_name = last_name # type: str - self.gravatar_hash = gravatar_hash # type: str - self.image_url = image_url # type: Optional[str] - self.membership_type = membership_type # type: str + def __init__(self, uuid: str, first_name: str, last_name: str, gravatar_hash: str, + image_url: str | None, membership_type: str): + self.uuid = uuid + self.first_name = first_name + self.last_name = last_name + self.gravatar_hash = gravatar_hash + self.image_url = image_url + self.membership_type = membership_type @staticmethod def load(data: dict, **options): @@ -1610,9 +1675,9 @@ def load(data: dict, **options): class DocumentContextUserPermission: - def __init__(self, user, permissions): - self.user = user # type: Optional[User] - self.permissions = permissions # type: list[str] + def __init__(self, user: User | None, permissions: list[str]): + self.user = user + self.permissions = permissions @property def is_viewer(self): @@ -1640,9 +1705,9 @@ def load(data: dict, **options): class DocumentContextUserGroupPermission: - def __init__(self, group, permissions): - self.group = group # type: Optional[UserGroup] - self.permissions = permissions # type: list[str] + def __init__(self, group: UserGroup | None, permissions: list[str]): + self.group = group + self.permissions = permissions @property def is_viewer(self): @@ -1684,7 +1749,7 @@ def __init__(self, ctx, **options): self.document = Document.load(ctx['document'], **options) self.package = Package.load(ctx['package'], **options) self.organization = Organization.load(ctx['organization'], **options) - self.current_phase = PHASE_NEVER # type: Phase + self.current_phase: Phase = PHASE_NEVER self.users = [DocumentContextUserPermission.load(d, **options) for d in ctx['users']] @@ -1719,14 +1784,14 @@ def doc(self) -> Document: def replies(self) -> RepliesContainer: return self.questionnaire.replies - def _resolve_links(self): + def resolve_links(self): phase_uuid = self.questionnaire.phase_uuid - if phase_uuid is not None and phase_uuid in self.e.phases.keys(): + if phase_uuid is not None and phase_uuid in self.e.phases: self.current_phase = self.e.phases[phase_uuid] self.questionnaire.phase = self.current_phase - self.km._resolve_links(self) - self.report._resolve_links(self) - self.questionnaire._resolve_links(self) + self.km.resolve_links(self) + self.report.resolve_links(self) + self.questionnaire.resolve_links(self) rv = ReplyVisitor(self) rv.visit() @@ -1738,8 +1803,8 @@ def _resolve_links(self): class ReplyVisitor: def __init__(self, context: DocumentContext): - self.item_titles = dict() # type: dict[str, str] - self._set_also = dict() # type: dict[str, list[str]] + self.item_titles: dict[str, str] = {} + self._set_also: dict[str, list[str]] = {} self.context = context def visit(self): @@ -1755,7 +1820,7 @@ def _visit_question(self, question: Question, path: str): if isinstance(question, ListQuestion): self._visit_list_question(question, new_path) elif isinstance(question, OptionsQuestion): - self._visit_options_question(question, new_path) + self._visit_options_question(new_path) def _visit_list_question(self, question: ListQuestion, path: str): reply = self.context.replies.get(path) @@ -1765,7 +1830,6 @@ def _visit_list_question(self, question: ListQuestion, path: str): self.item_titles[item_uuid] = f'Item {n}' item_path = f'{path}.{item_uuid}' - # title if len(question.followups) > 0: title_path = f'{item_path}.{question.followups[0].uuid}' title_reply = self.context.replies.get(title_path) @@ -1777,18 +1841,17 @@ def _visit_list_question(self, question: ListQuestion, path: str): self.item_titles[item_uuid] = non_empty_lines[0] elif title_reply is not None and isinstance(title_reply, ItemSelectReply): ref_item_uuid = title_reply.item_uuid - if ref_item_uuid in self.item_titles.keys(): + if ref_item_uuid in self.item_titles: self.item_titles[item_uuid] = self.item_titles[ref_item_uuid] else: self._set_also.setdefault(ref_item_uuid, []).append(item_uuid) for set_also in self._set_also.get(item_uuid, []): self.item_titles[set_also] = self.item_titles[item_uuid] - # followups for followup in question.followups: self._visit_question(followup, path=item_path) - def _visit_options_question(self, question: OptionsQuestion, path: str): + def _visit_options_question(self, path: str): reply = self.context.replies.get(path) if reply is None or not isinstance(reply, AnswerReply) or reply.answer is None: return diff --git a/packages/dsw-document-worker/dsw/document_worker/model/http.py b/packages/dsw-document-worker/dsw/document_worker/model/http.py index 8476d091..4e75d0c0 100644 --- a/packages/dsw-document-worker/dsw/document_worker/model/http.py +++ b/packages/dsw-document-worker/dsw/document_worker/model/http.py @@ -17,16 +17,13 @@ def _prepare_for_request(self): def get(self, url, params=None, **kwargs) -> requests.Response: self._prepare_for_request() - kwargs.update(timeout=self.timeout) - resp = requests.get(url=url, params=params, **kwargs) + resp = requests.get(url=url, params=params, timeout=self.timeout, **kwargs) return resp def post(self, url, data=None, json=None, **kwargs) -> requests.Response: self._prepare_for_request() - kwargs.update(timeout=self.timeout) - return requests.post(url=url, data=data, json=json, **kwargs) + return requests.post(url=url, data=data, json=json, timeout=self.timeout, **kwargs) def request(self, method: str, url: str, **kwargs) -> requests.Response: self._prepare_for_request() - kwargs.update(timeout=self.timeout) - return requests.request(method=method, url=url, **kwargs) + return requests.request(method=method, url=url, timeout=self.timeout, **kwargs) diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/filters.py b/packages/dsw-document-worker/dsw/document_worker/templates/filters.py index 1f53547d..74a1e9ee 100644 --- a/packages/dsw-document-worker/dsw/document_worker/templates/filters.py +++ b/packages/dsw-document-worker/dsw/document_worker/templates/filters.py @@ -1,12 +1,12 @@ import datetime +import logging +import re +import typing + import dateutil.parser as dp import jinja2 -import logging import markupsafe import markdown -import re - -from typing import Any, Union, Optional from ..exceptions import JobException from ..model import DocumentContext @@ -22,10 +22,10 @@ def extendMarkdown(self, md): class DSWMarkdownProcessor(markdown.preprocessors.Preprocessor): + LI_RE = re.compile(r'^[ ]*((\d+\.)|[*+-])[ ]+.*') def __init__(self, md): super().__init__(md) - self.LI_RE = re.compile(r'^[ ]*((\d+\.)|[*+-])[ ]+.*') def run(self, lines): prev_li = False @@ -54,7 +54,7 @@ def run(self, lines): class _JinjaEnv: def __init__(self): - self._env = None # type: Optional[jinja2.Environment] + self._env: jinja2.Environment | None = None @property def env(self) -> jinja2.Environment: @@ -76,12 +76,12 @@ def get_template(self, template_str: str) -> jinja2.Template: _alphabet_size = len(_alphabet) _base_jinja_loader = jinja2.BaseLoader() _j2_env = _JinjaEnv() -_empty_dict = dict() # type: dict[str, Any] +_empty_dict: dict[str, typing.Any] = {} _romans = [(1000, 'M'), (900, 'CM'), (500, 'D'), (400, 'CD'), (100, 'C'), (90, 'XC'), (50, 'L'), (40, 'XL'), (10, 'X'), (9, 'IX'), (5, 'V'), (4, 'IV'), (1, 'I')] -def datetime_format(iso_timestamp: Union[None, datetime.datetime, str], fmt: str): +def datetime_format(iso_timestamp: None | datetime.datetime | str, fmt: str): if iso_timestamp is None: return '' if not isinstance(iso_timestamp, datetime.datetime): @@ -133,7 +133,7 @@ def _has_value(reply: dict) -> bool: return bool(reply) and ('value' in reply.keys()) and ('value' in reply['value'].keys()) -def _get_value(reply: dict) -> Any: +def _get_value(reply: dict) -> typing.Any: return reply['value']['value'] @@ -201,7 +201,7 @@ def to_context_obj(ctx, **options) -> DocumentContext: LOG.debug('DocumentContext object requested') result = DocumentContext(ctx, **options) LOG.debug('DocumentContext object created') - result._resolve_links() + result.resolve_links() LOG.debug('DocumentContext object links resolved') return result diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/formats.py b/packages/dsw-document-worker/dsw/document_worker/templates/formats.py index 1962c300..4c5a98fb 100644 --- a/packages/dsw-document-worker/dsw/document_worker/templates/formats.py +++ b/packages/dsw-document-worker/dsw/document_worker/templates/formats.py @@ -23,7 +23,7 @@ def __init__(self, template, metadata: dict): self._verify_metadata(metadata) self.uuid = self._trace = metadata[FormatField.UUID] self.name = metadata[FormatField.NAME] - LOG.info(f'Setting up format "{self.name}" ({self._trace})') + LOG.info('Setting up format "%s" (%s)', self.name, self._trace) self.steps = self._create_steps(metadata) if len(self.steps) < 1: self.template.raise_exc(f'Format {self.name} has no steps') diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/steps/base.py b/packages/dsw-document-worker/dsw/document_worker/templates/steps/base.py index 220bcd35..7fe8c7a6 100644 --- a/packages/dsw-document-worker/dsw/document_worker/templates/steps/base.py +++ b/packages/dsw-document-worker/dsw/document_worker/templates/steps/base.py @@ -22,26 +22,29 @@ class Step: def __init__(self, template, options: dict[str, str]): self.template = template self.options = options - extras_str = self.options.get(self.OPTION_EXTRAS, '') # type: str - self.extras = set(extras_str.split(',')) # type: set[str] + + extras_str: str = self.options.get(self.OPTION_EXTRAS, '') + self.extras: set[str] = set(extras_str.split(',')) def requires_via_extras(self, requirement: str) -> bool: return requirement in self.extras def execute_first(self, context: dict) -> DocumentFile: + # pylint: disable=unused-argument return self.raise_exc('Called execute_follow on Step class') def execute_follow(self, document: DocumentFile, context: dict) -> DocumentFile: + # pylint: disable=unused-argument return self.raise_exc('Called execute_follow on Step class') def raise_exc(self, message: str): raise FormatStepException(message) -STEPS = dict() +STEPS: dict[str, type[Step]] = {} -def register_step(name: str, step_class: type): +def register_step(name: str, step_class: type[Step]): STEPS[name.lower()] = step_class diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/steps/conversion.py b/packages/dsw-document-worker/dsw/document_worker/templates/steps/conversion.py index c318b67a..85673df5 100644 --- a/packages/dsw-document-worker/dsw/document_worker/templates/steps/conversion.py +++ b/packages/dsw-document-worker/dsw/document_worker/templates/steps/conversion.py @@ -119,7 +119,7 @@ def __init__(self, template, options: dict): @staticmethod def _extract_filter_names(filters: str) -> list[str]: - names = list() + names: list[str] = [] for name in filters.split(','): name = name.strip() if name: diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/steps/excel.py b/packages/dsw-document-worker/dsw/document_worker/templates/steps/excel.py index 1f9032af..7fe9d681 100644 --- a/packages/dsw-document-worker/dsw/document_worker/templates/steps/excel.py +++ b/packages/dsw-document-worker/dsw/document_worker/templates/steps/excel.py @@ -1,23 +1,22 @@ import base64 import datetime -import dateutil.parser import io import json import pathlib +import typing +import dateutil.parser import xlsxwriter from xlsxwriter.chart import Chart from xlsxwriter.format import Format from xlsxwriter.worksheet import Worksheet -from typing import Any - from ...documents import DocumentFile, FileFormats from .base import Step, register_step, TMP_DIR, FormatStepException -_EMPTY_DICT = {} # type: dict[str, Any] -_EMPTY_LIST = [] # type: list[Any] +_EMPTY_DICT: dict[str, typing.Any] = {} +_EMPTY_LIST: list[typing.Any] = [] def _b64img2io(b64bytes: str) -> io.BytesIO: @@ -115,9 +114,9 @@ class WorkbookBuilder: def __init__(self, workbook: xlsxwriter.Workbook): self.workbook = workbook - self.sheets = list() # type: list[Worksheet] - self.formats = dict() # type: dict[str, Format] - self.charts = dict() # type: dict[str, Chart] + self.sheets: list[Worksheet] = [] + self.formats: dict[str, Format] = {} + self.charts: dict[str, Chart] = {} def _add_workbook_options(self, data: dict): # customized options not in regular 'constructor options' @@ -258,7 +257,8 @@ def _add_data_to_worksheet(self, worksheet: Worksheet, data: dict): item_type = item.get('type', None) if item_type is None: continue - elif item_type == 'cell': + + if item_type == 'cell': self._add_data_cell(worksheet, item) elif item_type == 'row': self._add_data_row(worksheet, item) @@ -420,7 +420,7 @@ def _add_data_image(worksheet: Worksheet, item: dict): bytes_io = io.BytesIO() if 'b64bytes' in item.keys(): if 'options' not in item.keys(): - item['options'] = dict() + item['options'] = {} bytes_io = _b64img2io(item['b64bytes']) item['options']['image_data'] = bytes_io if 'cell' in item.keys(): @@ -500,8 +500,8 @@ def _fix_footer_header_images(options): options[key] = _b64img2io(options[key]) @classmethod - def _setup_worksheet_common(cls, worksheet: Worksheet, data: dict): - data = data.get('options', None) + def _setup_worksheet_common(cls, worksheet: Worksheet, container: dict): + data: dict | None = container.get('options', None) if data is None: return if data.get('select', False): @@ -523,8 +523,8 @@ def _setup_worksheet_common(cls, worksheet: Worksheet, data: dict): worksheet.set_page_view() cls._setup_worksheet_print(worksheet, data) - def _setup_worksheet_data(self, worksheet: Worksheet, data: dict): - data = data.get('options', None) + def _setup_worksheet_data(self, worksheet: Worksheet, container: dict): + data: dict | None = container.get('options', None) if data is None: return self._setup_worksheet_basic(worksheet, data) diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/steps/template.py b/packages/dsw-document-worker/dsw/document_worker/templates/steps/template.py index 8ac5e38c..3590a3bb 100644 --- a/packages/dsw-document-worker/dsw/document_worker/templates/steps/template.py +++ b/packages/dsw-document-worker/dsw/document_worker/templates/steps/template.py @@ -1,11 +1,9 @@ -import typing as t +import json +import typing import jinja2 import jinja2.exceptions import jinja2.sandbox -import json - -from typing import Any from ...consts import DEFAULT_ENCODING from ...context import Context @@ -30,7 +28,7 @@ def execute_follow(self, document: DocumentFile, context: dict) -> DocumentFile: class JinjaEnvironment(jinja2.sandbox.SandboxedEnvironment): - def is_safe_attribute(self, obj: t.Any, attr: str, value: t.Any) -> bool: + def is_safe_attribute(self, obj: typing.Any, attr: str, value: typing.Any) -> bool: if attr in ['os', 'subprocess', 'eval', 'exec', 'popen', 'system']: return False if attr == '__setitem__' and isinstance(obj, dict): @@ -38,32 +36,14 @@ def is_safe_attribute(self, obj: t.Any, attr: str, value: t.Any) -> bool: return super().is_safe_attribute(obj, attr, value) -class Jinja2Step(Step): - NAME = 'jinja' - DEFAULT_FORMAT = FileFormats.HTML - - OPTION_ROOT_FILE = 'template' - OPTION_CONTENT_TYPE = 'content-type' - OPTION_EXTENSION = 'extension' +class JinjaPoweredStep(Step): OPTION_JINJA_EXT = 'jinja-ext' OPTION_I18N_DIR = 'i18n-dir' OPTION_I18N_DOMAIN = 'i18n-domain' OPTION_I18N_LANG = 'i18n-lang' - def _jinja_exception_msg(self, e: jinja2.exceptions.TemplateSyntaxError): - lines = [ - 'Failed loading Jinja2 template due to syntax error:', - f'- {e.message}', - f'- Filename: {e.name}', - f'- Line number: {e.lineno}', - ] - return '\n'.join(lines) - - def __init__(self, template, options: dict): + def __init__(self, template, options): super().__init__(template, options) - self.root_file = self.options[self.OPTION_ROOT_FILE] - self.content_type = self.options.get(self.OPTION_CONTENT_TYPE, self.DEFAULT_FORMAT.content_type) - self.extension = self.options.get(self.OPTION_EXTENSION, self.DEFAULT_FORMAT.file_extension) self.jinja_ext = frozenset( map(lambda x: x.strip(), self.options.get(self.OPTION_JINJA_EXT, '').split(',')) ) @@ -71,7 +51,6 @@ def __init__(self, template, options: dict): self.i18n_domain = self.options.get(self.OPTION_I18N_DOMAIN, 'default') self.i18n_lang = self.options.get(self.OPTION_I18N_LANG, None) - self.output_format = FileFormat(self.extension, self.content_type, self.extension) try: self.j2_env = JinjaEnvironment( loader=jinja2.FileSystemLoader(searchpath=template.template_dir), @@ -86,21 +65,30 @@ def __init__(self, template, options: dict): self.j2_env.add_extension('jinja2.ext.debug') self._apply_policies(options) self._add_j2_enhancements() - self.j2_root_template = self.j2_env.get_template(self.root_file) except jinja2.exceptions.TemplateSyntaxError as e: self.raise_exc(self._jinja_exception_msg(e)) except Exception as e: self.raise_exc(f'Failed loading Jinja2 template: {e}') + + def _jinja_exception_msg(self, e: jinja2.exceptions.TemplateSyntaxError): + lines = [ + 'Failed loading Jinja2 template due to syntax error:', + f'- {e.message}', + f'- Filename: {e.name}', + f'- Line number: {e.lineno}', + ] + return '\n'.join(lines) + def _apply_policies(self, options: dict): # https://jinja.palletsprojects.com/en/3.0.x/api/#policies - policies = { + policies: dict[str, typing.Any] = { 'policy.urlize.target': '_blank', 'json.dumps_kwargs': { 'allow_nan': False, 'ensure_ascii': False, }, - } # type: dict[str,Any] + } if 'policy.truncate.leeway' in options: policies['truncate.leeway'] = options['policy.truncate.leeway'] if 'policy.urlize.rel' in options: @@ -143,7 +131,6 @@ def _add_j2_enhancements(self): from ..tests import tests from ...model.http import RequestsWrapper import rdflib - import json self.j2_env.filters.update(filters) self.j2_env.tests.update(tests) template_cfg = Context.get().app.cfg.templates.get_config( @@ -151,13 +138,37 @@ def _add_j2_enhancements(self): ) self.j2_env.globals.update({'rdflib': rdflib, 'json': json}) if template_cfg is not None: - global_vars = {'secrets': template_cfg.secrets} # type: dict[str,Any] + global_vars: dict[str, typing.Any] = {'secrets': template_cfg.secrets} if template_cfg.requests.enabled: global_vars['requests'] = RequestsWrapper( template_cfg=template_cfg, ) self.j2_env.globals.update(global_vars) + + +class Jinja2Step(JinjaPoweredStep): + NAME = 'jinja' + DEFAULT_FORMAT = FileFormats.HTML + + OPTION_ROOT_FILE = 'template' + OPTION_CONTENT_TYPE = 'content-type' + OPTION_EXTENSION = 'extension' + + def __init__(self, template, options: dict): + super().__init__(template, options) + self.root_file = self.options[self.OPTION_ROOT_FILE] + self.content_type = self.options.get(self.OPTION_CONTENT_TYPE, self.DEFAULT_FORMAT.content_type) + self.extension = self.options.get(self.OPTION_EXTENSION, self.DEFAULT_FORMAT.file_extension) + + self.output_format = FileFormat(self.extension, self.content_type, self.extension) + try: + self.j2_root_template = self.j2_env.get_template(self.root_file) + except jinja2.exceptions.TemplateSyntaxError as e: + self.raise_exc(self._jinja_exception_msg(e)) + except Exception as e: + self.raise_exc(f'Failed loading Jinja2 template: {e}') + def _execute(self, **jinja_args): def asset_fetcher(file_name): return self.template.fetch_asset(file_name) diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/steps/word.py b/packages/dsw-document-worker/dsw/document_worker/templates/steps/word.py index 657584a8..57efa4fc 100644 --- a/packages/dsw-document-worker/dsw/document_worker/templates/steps/word.py +++ b/packages/dsw-document-worker/dsw/document_worker/templates/steps/word.py @@ -1,67 +1,28 @@ import pathlib -import jinja2 import shutil import zipfile -from typing import Any, Optional +import jinja2 from ...consts import DEFAULT_ENCODING -from ...context import Context from ...documents import DocumentFile, FileFormats -from .base import Step, register_step, TMP_DIR +from .base import register_step, TMP_DIR +from .template import JinjaPoweredStep -class EnrichDocxStep(Step): +class EnrichDocxStep(JinjaPoweredStep): NAME = 'enrich-docx' INPUT_FORMAT = FileFormats.DOCX OUTPUT_FORMAT = FileFormats.DOCX - def _jinja_exception_msg(self, e: jinja2.exceptions.TemplateSyntaxError): - lines = [ - 'Failed loading Jinja2 template due to syntax error:', - f'- {e.message}', - f'- Filename: {e.name}', - f'- Line number: {e.lineno}', - ] - return '\n'.join(lines) - def __init__(self, template, options: dict): super().__init__(template, options) self.rewrites = {k[8:]: v for k, v in options.items() if k.startswith('rewrite:')} - # TODO: shared part with Jinja2Step - try: - self.j2_env = jinja2.Environment( - loader=jinja2.FileSystemLoader(searchpath=template.template_dir), - extensions=['jinja2.ext.do'], - ) - self._add_j2_enhancements() - except jinja2.exceptions.TemplateSyntaxError as e: - self.raise_exc(self._jinja_exception_msg(e)) - except Exception as e: - self.raise_exc(f'Failed loading Jinja2 template: {e}') - - def _add_j2_enhancements(self): - # TODO: shared part with Jinja2Step - from ..filters import filters - from ..tests import tests - from ...model.http import RequestsWrapper - self.j2_env.filters.update(filters) - self.j2_env.tests.update(tests) - template_cfg = Context.get().app.cfg.templates.get_config( - self.template.template_id, - ) - if template_cfg is not None: - global_vars = {'secrets': template_cfg.secrets} # type: dict[str, Any] - if template_cfg.requests.enabled: - global_vars['requests'] = RequestsWrapper( - template_cfg=template_cfg, - ) - self.j2_env.globals.update(global_vars) def _render_rewrite(self, rewrite_template: str, context: dict, - existing_content: Optional[str]) -> str: + existing_content: str | None) -> str: try: j2_template = self.j2_env.get_template(rewrite_template) return j2_template.render( @@ -76,16 +37,17 @@ def _render_rewrite(self, rewrite_template: str, context: dict, def _static_rewrite(self, rewrite_file: str) -> str: try: - path = self.template.template_dir / rewrite_file # type: pathlib.Path + path: pathlib.Path = self.template.template_dir / rewrite_file return path.read_text(encoding=DEFAULT_ENCODING) except Exception as e: self.raise_exc(f'Failed loading Jinja2 template: {e}') return '' - def _get_rewrite(self, rewrite: str, context: dict, existing_content: Optional[str]) -> str: + def _get_rewrite(self, rewrite: str, context: dict, + existing_content: str | None) -> str: if rewrite.startswith('static:'): return self._static_rewrite(rewrite[7:]) - elif rewrite.startswith('render:'): + if rewrite.startswith('render:'): return self._render_rewrite(rewrite[7:], context, existing_content) return '' diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/templates.py b/packages/dsw-document-worker/dsw/document_worker/templates/templates.py index 32054333..f9ed39e2 100644 --- a/packages/dsw-document-worker/dsw/document_worker/templates/templates.py +++ b/packages/dsw-document-worker/dsw/document_worker/templates/templates.py @@ -4,8 +4,6 @@ import pathlib import shutil -from typing import Optional - from dsw.database.database import DBDocumentTemplate, \ DBDocumentTemplateFile, DBDocumentTemplateAsset @@ -50,9 +48,9 @@ def src_value(self): class TemplateComposite: def __init__(self, db_template, db_files, db_assets): - self.template = db_template # type: DBDocumentTemplate - self.files = db_files # type: dict[str, DBDocumentTemplateFile] - self.assets = db_assets # type: dict[str, DBDocumentTemplateAsset] + self.template: DBDocumentTemplate = db_template + self.files: dict[str, DBDocumentTemplateFile] = db_files + self.assets: dict[str, DBDocumentTemplateAsset] = db_assets class Template: @@ -64,7 +62,7 @@ def __init__(self, tenant_uuid: str, template_dir: pathlib.Path, self.last_used = datetime.datetime.now(tz=datetime.UTC) self.db_template = db_template self.template_id = self.db_template.template.id - self.formats = dict() # type: dict[str, Format] + self.formats: dict[str, Format] = {} self.asset_prefix = f'templates/{self.db_template.template.id}' if Context.get().app.cfg.cloud.multi_tenant: self.asset_prefix = f'{self.tenant_uuid}/{self.asset_prefix}' @@ -72,8 +70,8 @@ def __init__(self, tenant_uuid: str, template_dir: pathlib.Path, def raise_exc(self, message: str): raise TemplateException(self.template_id, message) - def fetch_asset(self, file_name: str) -> Optional[Asset]: - LOG.info(f'Fetching asset "{file_name}"') + def fetch_asset(self, file_name: str) -> Asset | None: + LOG.info('Fetching asset "%s"', file_name) file_path = self.template_dir / file_name asset = None for a in self.db_template.assets.values(): @@ -81,7 +79,7 @@ def fetch_asset(self, file_name: str) -> Optional[Asset]: asset = a break if asset is None or not file_path.exists(): - LOG.error(f'Asset "{file_name}" not found') + LOG.error('Asset "%s" not found', file_name) return None return Asset( asset_uuid=asset.uuid, @@ -94,16 +92,16 @@ def asset_path(self, filename: str) -> str: return str(self.template_dir / filename) def _store_asset(self, asset: DBDocumentTemplateAsset): - LOG.debug(f'Storing asset {asset.uuid} ({asset.file_name})') + LOG.debug('Storing asset %s (%s)', asset.uuid, asset.file_name) remote_path = f'{self.asset_prefix}/{asset.uuid}' local_path = self.template_dir / asset.file_name local_path.parent.mkdir(parents=True, exist_ok=True) result = Context.get().app.s3.download_file(remote_path, local_path) if not result: - LOG.error(f'Asset "{local_path.name}" cannot be retrieved') + LOG.error('Asset "%s" cannot be retrieved', local_path.name) def _store_file(self, file: DBDocumentTemplateFile): - LOG.debug(f'Storing file {file.uuid} ({file.file_name})') + LOG.debug('Storing file %s (%s)', file.uuid, file.file_name) local_path = self.template_dir / file.file_name local_path.parent.mkdir(parents=True, exist_ok=True) local_path.write_text( @@ -112,45 +110,45 @@ def _store_file(self, file: DBDocumentTemplateFile): ) def _delete_asset(self, asset: DBDocumentTemplateAsset): - LOG.debug(f'Deleting asset {asset.uuid} ({asset.file_name})') + LOG.debug('Deleting asset %s (%s)', asset.uuid, asset.file_name) local_path = self.template_dir / asset.file_name local_path.unlink(missing_ok=True) def _delete_file(self, file: DBDocumentTemplateFile): - LOG.debug(f'Deleting file {file.uuid} ({file.file_name})') + LOG.debug('Deleting file %s (%s)', file.uuid, file.file_name) local_path = self.template_dir / file.file_name local_path.unlink(missing_ok=True) def _update_asset(self, asset: DBDocumentTemplateAsset): - LOG.debug(f'Updating asset {asset.uuid} ({asset.file_name})') + LOG.debug('Updating asset %s (%s)', asset.uuid, asset.file_name) old_asset = self.db_template.assets[asset.uuid] local_path = self.template_dir / asset.file_name if old_asset.updated_at == asset.updated_at and local_path.exists(): - LOG.debug(f'- Asset {asset.uuid} ({asset.file_name}) did not change') + LOG.debug('- Asset %s (%s) did not change', asset.uuid, asset.file_name) return self._store_asset(asset) def _update_file(self, file: DBDocumentTemplateFile): - LOG.debug(f'Updating file {file.uuid} ({file.file_name})') + LOG.debug('Updating file %s (%s)', file.uuid, file.file_name) old_file = self.db_template.files[file.uuid] local_path = self.template_dir / file.file_name if old_file.updated_at == file.updated_at and local_path.exists(): - LOG.debug(f'- File {file.uuid} ({file.file_name}) did not change') + LOG.debug('- File %s (%s) did not change', file.uuid, file.file_name) return self._store_file(file) def prepare_all_template_files(self): - LOG.info(f'Storing all files of template {self.template_id} locally') + LOG.info('Storing all files of template %s locally', self.template_id) for file in self.db_template.files.values(): self._store_file(file) def prepare_all_template_assets(self): - LOG.info(f'Storing all assets of template {self.template_id} locally') + LOG.info('Storing all assets of template %s locally', self.template_id) for asset in self.db_template.assets.values(): self._store_asset(asset) def prepare_fs(self): - LOG.info(f'Preparing directory for template {self.template_id}') + LOG.info('Preparing directory for template %s', self.template_id) if self.template_dir.exists(): shutil.rmtree(self.template_dir) self.template_dir.mkdir(parents=True) @@ -165,7 +163,7 @@ def _resolve_change(old_keys: frozenset[str], new_keys: frozenset[str]): return to_add, to_del, to_chk def update_template_files(self, db_files: dict[str, DBDocumentTemplateFile]): - LOG.info(f'Updating files of template {self.template_id}') + LOG.info('Updating files of template %s', self.template_id) to_add, to_del, to_chk = self._resolve_change( old_keys=frozenset(self.db_template.files.keys()), new_keys=frozenset(db_files.keys()), @@ -179,7 +177,7 @@ def update_template_files(self, db_files: dict[str, DBDocumentTemplateFile]): self.db_template.files = db_files def update_template_assets(self, db_assets: dict[str, DBDocumentTemplateAsset]): - LOG.info(f'Updating assets of template {self.template_id}') + LOG.info('Updating assets of template %s', self.template_id) to_add, to_del, to_chk = self._resolve_change( old_keys=frozenset(self.db_template.assets.keys()), new_keys=frozenset(db_assets.keys()), @@ -231,15 +229,15 @@ def get(cls) -> 'TemplateRegistry': return cls._instance def __init__(self): - self._templates = dict() # type: dict[str, dict[str, Template]] + self._templates: dict[str, dict[str, Template]] = {} def has_template(self, tenant_uuid: str, template_id: str) -> bool: - return tenant_uuid in self._templates.keys() and \ - template_id in self._templates[tenant_uuid].keys() + return tenant_uuid in self._templates and \ + template_id in self._templates[tenant_uuid] def _set_template(self, tenant_uuid: str, template_id: str, template: Template): - if tenant_uuid not in self._templates.keys(): - self._templates[tenant_uuid] = dict() + if tenant_uuid not in self._templates: + self._templates[tenant_uuid] = {} self._templates[tenant_uuid][template_id] = template def get_template(self, tenant_uuid: str, template_id: str) -> Template: @@ -264,10 +262,10 @@ def _refresh_template(self, tenant_uuid: str, template_id: str, def prepare_template(self, tenant_uuid: str, template_id: str) -> Template: ctx = Context.get() - query_args = dict( - template_id=template_id, - tenant_uuid=tenant_uuid, - ) + query_args = { + 'template_id': template_id, + 'tenant_uuid': tenant_uuid, + } db_template = ctx.app.db.fetch_template(**query_args) if db_template is None: raise RuntimeError(f'Template {template_id} not found in database') diff --git a/packages/dsw-document-worker/dsw/document_worker/worker.py b/packages/dsw-document-worker/dsw/document_worker/worker.py index da15f31f..e9adcaf9 100644 --- a/packages/dsw-document-worker/dsw/document_worker/worker.py +++ b/packages/dsw-document-worker/dsw/document_worker/worker.py @@ -1,10 +1,10 @@ import datetime -import dateutil.parser import functools import logging import pathlib +import typing -from typing import Optional +import dateutil.parser from dsw.command_queue import CommandWorker, CommandQueue from dsw.config.sentry import SentryReporter @@ -36,11 +36,12 @@ def handled_step(job, *args, **kwargs): return func(job, *args, **kwargs) except Exception as e: LOG.debug('Handling exception', exc_info=True) - raise create_job_exception( + new_exception = create_job_exception( job_id=job.doc_uuid, message=message, exc=e, ) + raise new_exception from e return handled_step return decorator @@ -49,15 +50,15 @@ class Job: def __init__(self, command: PersistentCommand, document_uuid: str): self.ctx = Context.get() - self.template = None # type: Optional[Template] - self.format = None # type: Optional[Format] - self.tenant_uuid = command.tenant_uuid # type: str - self.doc_uuid = document_uuid # type: str - self.doc_context = command.body # type: dict - self.doc = None # type: Optional[DBDocument] - self.final_file = None # type: Optional[DocumentFile] - self.tenant_config = None # type: Optional[DBTenantConfig] - self.tenant_limits = None # type: Optional[DBTenantLimits] + self.template: Template | None = None + self.format: Format | None = None + self.tenant_uuid: str = command.tenant_uuid + self.doc_uuid: str = document_uuid + self.doc_context: dict = command.body + self.doc: DBDocument | None = None + self.final_file: DocumentFile | None = None + self.tenant_config: DBTenantConfig | None = None + self.tenant_limits: DBTenantLimits | None = None @property def safe_doc(self) -> DBDocument: @@ -89,8 +90,8 @@ def get_document(self): SentryReporter.set_context('format', '') SentryReporter.set_context('document', '') if self.tenant_uuid != NULL_UUID: - LOG.info(f'Limiting to tenant with UUID: {self.tenant_uuid}') - LOG.info(f'Getting the document "{self.doc_uuid}" details from DB') + LOG.info('Limiting to tenant with UUID: %s', self.tenant_uuid) + LOG.info('Getting the document "%s" details from DB', self.doc_uuid) self.doc = self.ctx.app.db.fetch_document( document_uuid=self.doc_uuid, tenant_uuid=self.tenant_uuid, @@ -101,10 +102,10 @@ def get_document(self): message='Document record not found in database', ) self.doc.retrieved_at = datetime.datetime.now(tz=datetime.UTC) - LOG.info(f'Job "{self.doc_uuid}" details received') + LOG.info('Job "%s" details received', self.doc_uuid) # verify state state = self.doc.state - LOG.info(f'Original state of job is {state}') + LOG.info('Original state of job is %s', state) if state == DocumentState.FINISHED: raise create_job_exception( job_id=self.doc_uuid, @@ -119,7 +120,8 @@ def get_document(self): def prepare_template(self): template_id = self.safe_doc.document_template_id format_uuid = self.safe_doc.format_uuid - LOG.info(f'Document uses template {template_id} with format {format_uuid}') + LOG.info('Document uses template %s with format %s', + template_id, format_uuid) # update Sentry info SentryReporter.set_context('template', template_id) SentryReporter.set_context('format', format_uuid) @@ -136,7 +138,7 @@ def prepare_template(self): self.template = template def _enrich_context(self): - extras = dict() + extras: dict[str, typing.Any] = {} if self.safe_format.requires_via_extras('submissions'): submissions = self.ctx.app.db.fetch_questionnaire_submissions( questionnaire_uuid=self.safe_doc.questionnaire_uuid, @@ -182,16 +184,17 @@ def build_document(self): def store_document(self): s3_id = self.ctx.app.s3.identification final_file = self.safe_final_file - LOG.info(f'Preparing S3 bucket {s3_id}') + LOG.info('Preparing S3 bucket %s', s3_id) self.ctx.app.s3.ensure_bucket() - LOG.info(f'Storing document to S3 bucket {s3_id}') + LOG.info('Storing document to S3 bucket %s', s3_id) self.ctx.app.s3.store_document( tenant_uuid=self.tenant_uuid, file_name=self.doc_uuid, content_type=final_file.object_content_type, data=final_file.content, ) - LOG.info(f'Document {self.doc_uuid} stored in S3 bucket {s3_id}') + LOG.info('Document %s stored in S3 bucket %s', + self.doc_uuid, s3_id) @handle_job_step('Failed to finalize document generation') def finalize(self): @@ -213,7 +216,7 @@ def finalize(self): ), document_uuid=self.doc_uuid, ) - LOG.info(f'Document {self.doc_uuid} record finalized') + LOG.info('Document %s record finalized', self.doc_uuid) def set_job_state(self, state: str, message: str) -> bool: return self.ctx.app.db.update_document_state( @@ -227,7 +230,8 @@ def try_set_job_state(self, state: str, message: str) -> bool: return self.set_job_state(state, message) except Exception as e: SentryReporter.capture_exception(e) - LOG.warning(f'Tried to set state of {self.doc_uuid} to {state} but failed: {e}') + LOG.warning('Tried to set state of %s to %s but failed: %s', + self.doc_uuid, state, str(e)) return False def _run(self): @@ -256,10 +260,10 @@ def run(self): if self._sentry_job_exception(): SentryReporter.capture_exception(e) if self.try_set_job_state(DocumentState.FAILED, e.db_message()): - LOG.info(f'Set state to {DocumentState.FAILED}') + LOG.info('Set state to FAILED') else: - LOG.error(f'Could not set state to {DocumentState.FAILED}') - raise RuntimeError(f'Could not set state to {DocumentState.FAILED}') + LOG.error('Could not set state to FAILED') + raise RuntimeError('Could not set state to FAILED') from e except Exception as e: SentryReporter.capture_exception(e) job_exc = create_job_exception( @@ -270,10 +274,10 @@ def run(self): LOG.error(job_exc.log_message()) LOG.info('Failed with unexpected error', exc_info=e) if self.try_set_job_state(DocumentState.FAILED, job_exc.db_message()): - LOG.info(f'Set state to {DocumentState.FAILED}') + LOG.info('Set state to FAILED') else: - LOG.warning(f'Could not set state to {DocumentState.FAILED}') - raise RuntimeError(f'Could not set state to {DocumentState.FAILED}') + LOG.warning('Could not set state to FAILED') + raise RuntimeError('Could not set state to FAILED') from e class DocumentWorker(CommandWorker): @@ -281,7 +285,7 @@ class DocumentWorker(CommandWorker): def __init__(self, config: DocumentWorkerConfig, workdir: pathlib.Path): self.config = config self._init_context(workdir=workdir) - self.current_job = None # type: Job | None + self.current_job: Job | None = None def _init_context(self, workdir: pathlib.Path): Context.initialize( @@ -308,8 +312,8 @@ def _init_context(self, workdir: pathlib.Path): @staticmethod def _update_component_info(): built_at = dateutil.parser.parse(BUILD_INFO.built_at) - LOG.info(f'Updating component info ({BUILD_INFO.version}, ' - f'{built_at.isoformat(timespec="seconds")})') + LOG.info('Updating component info (%s, %s)', + BUILD_INFO.version, built_at.isoformat(timespec="seconds")) Context.get().app.db.update_component_info( name=COMPONENT_NAME, version=BUILD_INFO.version, @@ -353,7 +357,7 @@ def work(self, cmd: PersistentCommand): Context.get().update_trace_id(cmd.uuid) Context.get().update_document_id(document_uuid) SentryReporter.set_context('cmd_uuid', cmd.uuid) - LOG.info(f'Running job #{cmd.uuid}') + LOG.info('Running job %s', cmd.uuid) self.current_job = Job(command=cmd, document_uuid=document_uuid) self.current_job.run() self.current_job = None diff --git a/packages/dsw-document-worker/pyproject.toml b/packages/dsw-document-worker/pyproject.toml index 4d2e94db..b4de1009 100644 --- a/packages/dsw-document-worker/pyproject.toml +++ b/packages/dsw-document-worker/pyproject.toml @@ -17,11 +17,11 @@ classifiers = [ 'License :: OSI Approved :: Apache Software License', 'Operating System :: POSIX :: Linux', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Text Processing', ] -requires-python = '>=3.10, <4' +requires-python = '>=3.11, <4' dependencies = [ 'click', 'Jinja2', diff --git a/packages/dsw-mailer/dsw/mailer/cli.py b/packages/dsw-mailer/dsw/mailer/cli.py index 52690b8a..2e5086a1 100644 --- a/packages/dsw-mailer/dsw/mailer/cli.py +++ b/packages/dsw-mailer/dsw/mailer/cli.py @@ -1,8 +1,9 @@ -import click # type: ignore import json import pathlib +import typing +import sys -from typing import IO +import click from dsw.config.parser import MissingConfigurationError @@ -16,7 +17,7 @@ def load_config_str(config_str: str) -> MailerConfig: parser = MailerConfigParser() if not parser.can_read(config_str): click.echo('Error: Cannot parse config file', err=True) - exit(1) + sys.exit(1) try: parser.read_string(config_str) @@ -25,14 +26,14 @@ def load_config_str(config_str: str) -> MailerConfig: click.echo('Error: Missing configuration', err=True) for missing_item in e.missing: click.echo(f' - {missing_item}', err=True) - exit(1) + sys.exit(1) config = parser.config config.log.apply() return config -def validate_config(ctx, param, value: IO | None): +def validate_config(ctx, param, value: typing.IO | None): content = '' if value is not None: content = value.read() @@ -40,14 +41,14 @@ def validate_config(ctx, param, value: IO | None): return load_config_str(content) -def extract_message_request(ctx, param, value: IO): +def extract_message_request(ctx, param, value: typing.IO): data = json.load(value) try: return MessageRequest.load_from_file(data) except Exception as e: click.echo('Error: Cannot parse message request', err=True) click.echo(f'{type(e).__name__}: {str(e)}') - exit(1) + sys.exit(1) @click.group(name='dsw-mailer', help='Mailer for sending emails from DSW') @@ -74,7 +75,7 @@ def cli(ctx, config: MailerConfig, workdir: str): type=click.File('r', encoding=DEFAULT_ENCODING)) def send(ctx, msg_request: MessageRequest, config: MailerConfig): from .mailer import Mailer - mailer = ctx.obj['mailer'] # type: Mailer + mailer: Mailer = ctx.obj['mailer'] mailer.send(rq=msg_request, cfg=config.mail) @@ -82,7 +83,7 @@ def send(ctx, msg_request: MessageRequest, config: MailerConfig): @click.pass_context def run(ctx): from .mailer import Mailer - mailer = ctx.obj['mailer'] # type: Mailer + mailer: Mailer = ctx.obj['mailer'] mailer.run() diff --git a/packages/dsw-mailer/dsw/mailer/context.py b/packages/dsw-mailer/dsw/mailer/context.py index e7c6aac0..8551853e 100644 --- a/packages/dsw-mailer/dsw/mailer/context.py +++ b/packages/dsw-mailer/dsw/mailer/context.py @@ -1,13 +1,10 @@ import pathlib -from typing import TYPE_CHECKING +from dsw.database import Database +from .config import MailerConfig from .templates import TemplateRegistry -if TYPE_CHECKING: - from .config import MailerConfig - from dsw.database import Database - class ContextNotInitializedError(RuntimeError): @@ -17,10 +14,10 @@ def __init__(self): class AppContext: - def __init__(self, db, cfg, workdir): - self.db = db # type: Database - self.cfg = cfg # type: MailerConfig - self.workdir = workdir # type: pathlib.Path + def __init__(self, db: Database, cfg: MailerConfig, workdir: pathlib.Path): + self.db = db + self.cfg = cfg + self.workdir = workdir class JobContext: @@ -47,7 +44,7 @@ def reset_ids(self): class Context: - _instance = None # type: _Context | None + _instance: _Context | None = None @classmethod def get(cls) -> _Context: diff --git a/packages/dsw-mailer/dsw/mailer/mailer.py b/packages/dsw-mailer/dsw/mailer/mailer.py index ca8b35fe..cfaba315 100644 --- a/packages/dsw-mailer/dsw/mailer/mailer.py +++ b/packages/dsw-mailer/dsw/mailer/mailer.py @@ -1,11 +1,12 @@ import datetime -import dateutil.parser import logging import math import pathlib import time import urllib.parse +import dateutil.parser + from dsw.command_queue import CommandWorker, CommandQueue from dsw.config.sentry import SentryReporter from dsw.database.database import Database @@ -55,8 +56,8 @@ def _init_context(self, workdir: pathlib.Path): @staticmethod def _update_component_info(): built_at = dateutil.parser.parse(BUILD_INFO.built_at) - LOG.info(f'Updating component info ({BUILD_INFO.version}, ' - f'{built_at.isoformat(timespec="seconds")})') + LOG.info('Updating component info (%s, %s)', + BUILD_INFO.version, built_at.isoformat(timespec="seconds")) Context.get().app.db.update_component_info( name=COMPONENT_NAME, version=BUILD_INFO.version, @@ -89,31 +90,31 @@ def run_once(self): queue = self._run_preparation() queue.run_once() - def work(self, cmd: PersistentCommand): + def work(self, command: PersistentCommand): # update Sentry info SentryReporter.set_context('template', '-') - SentryReporter.set_context('cmd_uuid', cmd.uuid) - Context.get().update_trace_id(cmd.uuid) + SentryReporter.set_context('cmd_uuid', command.uuid) + Context.get().update_trace_id(command.uuid) # work app_ctx = Context.get().app - mc = MailerCommand.load(cmd) + mc = MailerCommand.load(command) rq = mc.to_request( - msg_id=cmd.uuid, + msg_id=command.uuid, trigger='PersistentComment', ) # get tenant config from DB - tenant_cfg = app_ctx.db.get_tenant_config(tenant_uuid=cmd.tenant_uuid) - LOG.debug(f'Tenant config from DB: {tenant_cfg}') + tenant_cfg = app_ctx.db.get_tenant_config(tenant_uuid=command.tenant_uuid) + LOG.debug('Tenant config from DB: %s', tenant_cfg) if tenant_cfg is not None: rq.style.from_dict(tenant_cfg.look_and_feel) # get mailer config from DB mail_cfg = merge_mail_configs( cfg=self.cfg, - db_cfg=app_ctx.db.get_mail_config(tenant_uuid=cmd.tenant_uuid), + db_cfg=app_ctx.db.get_mail_config(tenant_uuid=command.tenant_uuid), ) - LOG.debug(f'Mail config from DB: {mail_cfg}') + LOG.debug('Mail config from DB: %s', mail_cfg) # client URL - rq.client_url = cmd.body.get('clientUrl', app_ctx.cfg.general.client_url) + rq.client_url = command.body.get('clientUrl', app_ctx.cfg.general.client_url) rq.domain = urllib.parse.urlparse(rq.client_url).hostname # update Sentry info SentryReporter.set_context('template', rq.template_name) @@ -131,15 +132,15 @@ def process_exception(self, e: BaseException): SentryReporter.capture_exception(e) def send(self, rq: MessageRequest, cfg: MailConfig): - LOG.info(f'Sending request: {rq.template_name} ({rq.id})') + LOG.info('Sending request: %s (%s)', rq.template_name, rq.id) # get template if not self.ctx.templates.has_template_for(rq): raise RuntimeError(f'Template not found: {rq.template_name}') # render - LOG.info(f'Rendering message: {rq.template_name}') + LOG.info('Rendering message: %s', rq.template_name) msg = self.ctx.templates.render(rq, cfg) # send - LOG.info(f'Sending message: {rq.template_name}') + LOG.info('Sending message: %s', rq.template_name) send(msg, cfg) LOG.info('Message sent successfully') @@ -149,7 +150,7 @@ class RateLimiter: def __init__(self, window: int, count: int): self.window = window self.count = count - self.hits = [] # type: list[float] + self.hits: list[float] = [] def hit(self): if self.window == 0: @@ -165,14 +166,14 @@ def hit(self): LOG.info('Reached rate limit') sleep_time = math.ceil(self.window - now + self.hits[0]) if sleep_time > 1: - LOG.info(f'Will sleep now for {sleep_time} second') + LOG.info('Will sleep now for %s seconds', sleep_time) time.sleep(sleep_time) class MailerCommand: - def __init__(self, recipients: list[str], mode: str, template: str, ctx: dict, - tenant_uuid: str, cmd_uuid: str): + def __init__(self, recipients: list[str], mode: str, template: str, + ctx: dict, tenant_uuid: str, cmd_uuid: str): self.mode = mode self.template = template self.recipients = recipients @@ -204,7 +205,7 @@ def load(cmd: PersistentCommand) -> 'MailerCommand': if cmd.component != CMD_COMPONENT: raise RuntimeError('Tried to process non-mailer command') if cmd.function != CMD_FUNCTION: - raise RuntimeError(f'Unsupported function: {cmd.function}') + raise RuntimeError('Unsupported function: %s', cmd.function) try: return MailerCommand( mode=cmd.body['mode'], @@ -215,4 +216,4 @@ def load(cmd: PersistentCommand) -> 'MailerCommand': cmd_uuid=cmd.uuid, ) except KeyError as e: - raise RuntimeError(f'Cannot parse command: {str(e)}') + raise RuntimeError(f'Cannot parse command: {str(e)}') from e diff --git a/packages/dsw-mailer/dsw/mailer/model.py b/packages/dsw-mailer/dsw/mailer/model.py index 722f419e..036e2dc1 100644 --- a/packages/dsw-mailer/dsw/mailer/model.py +++ b/packages/dsw-mailer/dsw/mailer/model.py @@ -14,8 +14,7 @@ def contrast_ratio(color1: 'Color', color2: 'Color') -> float: l2 = color2.luminance + 0.05 if l1 > l2: return l1 / l2 - else: - return l2 / l1 + return l2 / l1 def __init__(self, color_hex: str = '#000000', default: str = '#000000'): color_hex = self.parse_color_to_hex(color_hex) or default @@ -46,8 +45,7 @@ def _luminance_component(component: int): c = component / 255 if c <= 0.03928: return c / 12.92 - else: - return ((c + 0.055) / 1.055) ** 2.4 + return ((c + 0.055) / 1.055) ** 2.4 r = _luminance_component(self.red) g = _luminance_component(self.green) @@ -66,8 +64,7 @@ def is_light(self): def contrast_color(self) -> 'Color': if self.contrast_ratio(self, Color('#ffffff')) > 3: return Color('#ffffff') - else: - return Color('#000000') + return Color('#000000') def __str__(self): return self.hex @@ -83,7 +80,7 @@ def __init__(self, logo_url: str | None, primary_color: str, self.illustrations_color = Color(illustrations_color, Color.DEFAULT_ILLUSTRATIONS_HEX) def from_dict(self, data: dict | None): - data = data or dict() + data = data or {} if data.get('logoUrl', None) is not None: self.logo_url = data.get('logoUrl') if data.get('primaryColor', None) is not None: @@ -134,7 +131,7 @@ def __init__(self, part_type: str, file: str): self.content_type = '' self.encoding = '' - def _update_from_data(self, data: dict): + def update_from_data(self, data: dict): for field in self.FIELDS: target_field = field.replace('-', '_') if field in data.keys(): @@ -150,7 +147,7 @@ def load_from_file(data: dict) -> 'TemplateDescriptorPart': part_type=data.get('type', 'unknown'), file=data.get('file', ''), ) - part._update_from_data(data) + part.update_from_data(data) return part @@ -222,21 +219,21 @@ def load_from_file(data: dict) -> 'MessageRequest': class MailMessage: def __init__(self): - self.from_mail = '' # type: str - self.from_name = None # type: str | None - self.recipients = list() # type: list[str] - self.subject = '' # type: str - self.plain_body = None # type: str | None - self.html_body = None # type: str | None - self.html_images = list() # type: list[MailAttachment] - self.attachments = list() # type: list[MailAttachment] - self.msg_id = None # type: str | None - self.msg_domain = None # type: str | None - self.language = 'en' # type: str - self.importance = 'normal' # type: str - self.sensitivity = None # type: str | None - self.priority = None # type: str | None - self.client_url = '' # type: str + self.from_mail: str = '' + self.from_name: str | None = None + self.recipients: list[str] = [] + self.subject: str = '' + self.plain_body: str | None = None + self.html_body: str | None = None + self.html_images: list[MailAttachment] = [] + self.attachments: list[MailAttachment] = [] + self.msg_id: str | None = None + self.msg_domain: str | None = None + self.language: str = 'en' + self.importance: str = 'normal' + self.sensitivity: str | None = None + self.priority: str | None = None + self.client_url: str = '' class MailAttachment: diff --git a/packages/dsw-mailer/dsw/mailer/sender/amazon_ses.py b/packages/dsw-mailer/dsw/mailer/sender/amazon_ses.py index 647ac2ae..ae384660 100644 --- a/packages/dsw-mailer/dsw/mailer/sender/amazon_ses.py +++ b/packages/dsw-mailer/dsw/mailer/sender/amazon_ses.py @@ -1,6 +1,7 @@ -import boto3 import logging +import boto3 + from .base import BaseMailSender from ..config import MailConfig from ..model import MailMessage @@ -19,7 +20,8 @@ def validate_config(cfg: MailConfig): raise ValueError('Missing region for Amazon SES') def send(self, message: MailMessage): - LOG.info(f'Sending via Amazon SES (region {self.cfg.amazon_ses.region})') + LOG.info('Sending via Amazon SES (region %s)', + self.cfg.amazon_ses.region) self._send(message, self.cfg) def _send(self, mail: MailMessage, cfg: MailConfig): diff --git a/packages/dsw-mailer/dsw/mailer/sender/base.py b/packages/dsw-mailer/dsw/mailer/sender/base.py index 4ff86200..f256d526 100644 --- a/packages/dsw-mailer/dsw/mailer/sender/base.py +++ b/packages/dsw-mailer/dsw/mailer/sender/base.py @@ -1,8 +1,6 @@ import abc import datetime -import dkim import logging -import pathvalidate from email import encoders from email.mime.base import MIMEBase @@ -10,6 +8,9 @@ from email.mime.text import MIMEText from email.utils import formataddr, format_datetime, make_msgid +import dkim +import pathvalidate + from ..config import MailConfig from ..consts import DEFAULT_ENCODING from ..model import MailMessage, MailAttachment @@ -147,4 +148,3 @@ def validate_config(cfg: MailConfig): def send(self, message: MailMessage): LOG.info('No provider configured, not sending anything') - return diff --git a/packages/dsw-mailer/dsw/mailer/sender/smtp.py b/packages/dsw-mailer/dsw/mailer/sender/smtp.py index 0dc6a477..6a1082eb 100644 --- a/packages/dsw-mailer/dsw/mailer/sender/smtp.py +++ b/packages/dsw-mailer/dsw/mailer/sender/smtp.py @@ -1,10 +1,11 @@ import logging import smtplib import ssl -import tenacity from email.utils import formataddr +import tenacity + from .base import BaseMailSender from ..config import MailConfig from ..model import MailMessage @@ -32,7 +33,8 @@ def validate_config(cfg: MailConfig): after=tenacity.after_log(LOG, logging.DEBUG), ) def send(self, message: MailMessage): - LOG.info(f'Sending via SMTP (server {self.cfg.smtp.host}:{self.cfg.smtp.port})') + LOG.info('Sending via SMTP (server %s:%s)', + self.cfg.smtp.host, self.cfg.smtp.port) if self.cfg.smtp.is_ssl: self._send_smtp_ssl(mail=message) else: diff --git a/packages/dsw-mailer/dsw/mailer/templates.py b/packages/dsw-mailer/dsw/mailer/templates.py index 8f1858f2..f3915ffa 100644 --- a/packages/dsw-mailer/dsw/mailer/templates.py +++ b/packages/dsw-mailer/dsw/mailer/templates.py @@ -1,17 +1,18 @@ import datetime +import json +import logging +import pathlib +import re + import dateutil.parser import jinja2 import jinja2.sandbox -import json -import logging import markdown import markupsafe -import pathlib -import re from .config import MailerConfig, MailConfig from .consts import DEFAULT_ENCODING -from .model import MailMessage, MailAttachment, MessageRequest,\ +from .model import MailMessage, MailAttachment, MessageRequest, \ TemplateDescriptor, TemplateDescriptorPart @@ -27,8 +28,8 @@ def __init__(self, name: str, descriptor: TemplateDescriptor, self.descriptor = descriptor self.html_template = html_template self.plain_template = plain_template - self.attachments = list() # type: list[MailAttachment] - self.html_images = list() # type: list[MailAttachment] + self.attachments: list[MailAttachment] = [] + self.html_images: list[MailAttachment] = [] def render(self, rq: MessageRequest, mail_name: str | None, mail_from: str) -> MailMessage: ctx = rq.ctx @@ -71,7 +72,7 @@ def __init__(self, cfg: MailerConfig, workdir: pathlib.Path): loader=jinja2.FileSystemLoader(searchpath=workdir), extensions=['jinja2.ext.do'], ) - self.templates = dict() # type: dict[str, MailTemplate] + self.templates: dict[str, MailTemplate] = {} self._set_filters() self._load_templates() @@ -110,16 +111,16 @@ def _load_descriptor(path: pathlib.Path) -> TemplateDescriptor | None: data = json.loads(path.read_text(encoding=DEFAULT_ENCODING)) return TemplateDescriptor.load_from_file(data) except Exception as e: - LOG.warning(f'Cannot load template descriptor at {str(path)}' - f'due to: {str(e)}') + LOG.warning('Cannot load template descriptor at %s: %s', + path.as_posix(), str(e)) return None def _load_template(self, path: pathlib.Path, descriptor: TemplateDescriptor) -> MailTemplate | None: html_template = None plain_template = None - attachments = list() - html_images = list() + attachments = [] + html_images = [] for part in descriptor.parts: if part.type == 'html': html_template = self._load_jinja2(path / part.file) @@ -130,8 +131,8 @@ def _load_template(self, path: pathlib.Path, elif part.type == 'html_image': html_images.append(self._load_attachment(path, part)) if html_template is None and plain_template is None: - LOG.warning(f'Template "{descriptor.id}" from {str(path)}' - f'does not have HTML nor Plain part - skipping') + LOG.warning('Template "%s" from %s has no HTML nor Plain part - skipping', + descriptor.id, path.as_posix()) return None template = MailTemplate( name=path.name, @@ -152,11 +153,12 @@ def _load_templates(self): template = self._load_template(path, descriptor) if template is None: continue - LOG.info(f'Loaded template "{descriptor.id}" from {str(path)}') + LOG.info('Loaded template "%s" from %s', + descriptor.id, path.as_posix()) self.templates[descriptor.id] = template def has_template_for(self, rq: MessageRequest) -> bool: - return rq.template_name in self.templates.keys() + return rq.template_name in self.templates def render(self, rq: MessageRequest, cfg: MailConfig) -> MailMessage: used_cfg = cfg or self.cfg.mail @@ -183,9 +185,10 @@ def extendMarkdown(self, md): class DSWMarkdownProcessor(markdown.preprocessors.Preprocessor): + LI_RE = re.compile(r'^[ ]*((\d+\.)|[*+-])[ ]+.*') + def __init__(self, md): super().__init__(md) - self.LI_RE = re.compile(r'^[ ]*((\d+\.)|[*+-])[ ]+.*') def run(self, lines): prev_li = False diff --git a/packages/dsw-mailer/pyproject.toml b/packages/dsw-mailer/pyproject.toml index 9022a1e2..c9878c00 100644 --- a/packages/dsw-mailer/pyproject.toml +++ b/packages/dsw-mailer/pyproject.toml @@ -16,12 +16,12 @@ classifiers = [ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Communications :: Email', 'Topic :: Text Processing', ] -requires-python = '>=3.10, <4' +requires-python = '>=3.11, <4' dependencies = [ 'boto3', 'click', diff --git a/packages/dsw-models/dsw/models/km/events.py b/packages/dsw-models/dsw/models/km/events.py index ca83c13f..444dcac3 100644 --- a/packages/dsw-models/dsw/models/km/events.py +++ b/packages/dsw-models/dsw/models/km/events.py @@ -1,10 +1,9 @@ import abc - -from typing import Generic, Optional, TypeVar, Any +import typing # https://github.com/ds-wizard/engine-backend/blob/develop/engine-shared/src/Shared/Model/Event/ -T = TypeVar('T') +T = typing.TypeVar('T') class MetricMeasure: @@ -30,9 +29,9 @@ def from_dict(data: dict) -> 'MetricMeasure': ) -class EventField(Generic[T]): +class EventField(typing.Generic[T]): - def __init__(self, changed: bool, value: Optional[T]): + def __init__(self, changed: bool, value: T | None): self.changed = changed self.value = value @@ -172,7 +171,7 @@ def to_dict(self) -> dict: return result -EVENT_TYPES = {} # type: dict[str, Any] +EVENT_TYPES: dict[str, typing.Any] = {} def event_class(cls): @@ -262,7 +261,7 @@ class AddChapterEvent(_KMAddEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - title: str, text: Optional[str]): + title: str, text: str | None): super().__init__( event_uuid=event_uuid, entity_uuid=entity_uuid, @@ -302,7 +301,7 @@ class EditChapterEvent(_KMEditEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], - title: EventField[str], text: EventField[Optional[str]], + title: EventField[str], text: EventField[str | None], question_uuids: EventField[list[str]]): super().__init__( event_uuid=event_uuid, @@ -371,7 +370,7 @@ class AddQuestionEvent(_KMAddEvent, abc.ABC): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - title: str, text: Optional[str], required_phase_uuid: Optional[str], + title: str, text: str | None, required_phase_uuid: str | None, tag_uuids: list[str]): super().__init__( event_uuid=event_uuid, @@ -418,7 +417,7 @@ class AddOptionsQuestionEvent(AddQuestionEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - title: str, text: Optional[str], required_phase_uuid: Optional[str], + title: str, text: str | None, required_phase_uuid: str | None, tag_uuids: list[str]): super().__init__( event_uuid=event_uuid, @@ -462,7 +461,7 @@ class AddMultiChoiceQuestionEvent(AddQuestionEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - title: str, text: Optional[str], required_phase_uuid: Optional[str], + title: str, text: str | None, required_phase_uuid: str | None, tag_uuids: list[str]): super().__init__( event_uuid=event_uuid, @@ -506,7 +505,7 @@ class AddListQuestionEvent(AddQuestionEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - title: str, text: Optional[str], required_phase_uuid: Optional[str], + title: str, text: str | None, required_phase_uuid: str | None, tag_uuids: list[str]): super().__init__( event_uuid=event_uuid, @@ -550,7 +549,7 @@ class AddValueQuestionEvent(AddQuestionEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - title: str, text: Optional[str], required_phase_uuid: Optional[str], + title: str, text: str | None, required_phase_uuid: str | None, tag_uuids: list[str], value_type: str): super().__init__( event_uuid=event_uuid, @@ -597,7 +596,7 @@ class AddIntegrationQuestionEvent(AddQuestionEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - title: str, text: Optional[str], required_phase_uuid: Optional[str], + title: str, text: str | None, required_phase_uuid: str | None, tag_uuids: list[str], integration_uuid: str, props: dict[str, str]): super().__init__( event_uuid=event_uuid, @@ -648,8 +647,8 @@ class EditQuestionEvent(_KMEditEvent, abc.ABC): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], - title: EventField[str], text: EventField[Optional[str]], - required_phase_uuid: EventField[Optional[str]], + title: EventField[str], text: EventField[str | None], + required_phase_uuid: EventField[str | None], tag_uuids: EventField[list[str]], expert_uuids: EventField[list[str]], reference_uuids: EventField[list[str]]): super().__init__( @@ -701,8 +700,8 @@ class EditOptionsQuestionEvent(EditQuestionEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], - title: EventField[str], text: EventField[Optional[str]], - required_phase_uuid: EventField[Optional[str]], + title: EventField[str], text: EventField[str | None], + required_phase_uuid: EventField[str | None], tag_uuids: EventField[list[str]], expert_uuids: EventField[list[str]], reference_uuids: EventField[list[str]], answer_uuids: EventField[list[str]]): @@ -758,8 +757,8 @@ class EditMultiChoiceQuestionEvent(EditQuestionEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], - title: EventField[str], text: EventField[Optional[str]], - required_phase_uuid: EventField[Optional[str]], + title: EventField[str], text: EventField[str | None], + required_phase_uuid: EventField[str | None], tag_uuids: EventField[list[str]], expert_uuids: EventField[list[str]], reference_uuids: EventField[list[str]], choice_uuids: EventField[list[str]]): @@ -815,8 +814,8 @@ class EditListQuestionEvent(EditQuestionEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], - title: EventField[str], text: EventField[Optional[str]], - required_phase_uuid: EventField[Optional[str]], + title: EventField[str], text: EventField[str | None], + required_phase_uuid: EventField[str | None], tag_uuids: EventField[list[str]], expert_uuids: EventField[list[str]], reference_uuids: EventField[list[str]], item_template_question_uuids: EventField[list[str]]): @@ -872,8 +871,8 @@ class EditValueQuestionEvent(EditQuestionEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], - title: EventField[str], text: EventField[Optional[str]], - required_phase_uuid: EventField[Optional[str]], + title: EventField[str], text: EventField[str | None], + required_phase_uuid: EventField[str | None], tag_uuids: EventField[list[str]], expert_uuids: EventField[list[str]], reference_uuids: EventField[list[str]], value_type: EventField[str]): super().__init__( @@ -928,8 +927,8 @@ class EditIntegrationQuestionEvent(EditQuestionEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], - title: EventField[str], text: EventField[Optional[str]], - required_phase_uuid: EventField[Optional[str]], + title: EventField[str], text: EventField[str | None], + required_phase_uuid: EventField[str | None], tag_uuids: EventField[list[str]], expert_uuids: EventField[list[str]], reference_uuids: EventField[list[str]], integration_uuid: EventField[str], props: EventField[dict[str, str]]): @@ -1011,7 +1010,7 @@ class AddAnswerEvent(_KMAddEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - label: str, advice: Optional[str], metric_measures: list[MetricMeasure]): + label: str, advice: str | None, metric_measures: list[MetricMeasure]): super().__init__( event_uuid=event_uuid, entity_uuid=entity_uuid, @@ -1054,7 +1053,7 @@ class EditAnswerEvent(_KMEditEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], - label: EventField[str], advice: EventField[Optional[str]], + label: EventField[str], advice: EventField[str | None], follow_up_uuids: EventField[list[str]], metric_measures: EventField[list[MetricMeasure]]): super().__init__( @@ -1673,7 +1672,7 @@ class AddTagEvent(_KMAddEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - name: str, description: Optional[str], color: str): + name: str, description: str | None, color: str): super().__init__( event_uuid=event_uuid, entity_uuid=entity_uuid, @@ -1716,7 +1715,7 @@ class EditTagEvent(_KMEditEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], - name: EventField[str], description: EventField[Optional[str]], + name: EventField[str], description: EventField[str | None], color: EventField[str]): super().__init__( event_uuid=event_uuid, @@ -1785,8 +1784,8 @@ class AddIntegrationEvent(_KMAddEvent, abc.ABC): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - integration_id: str, name: str, props: list[str], logo: Optional[str], - item_url: Optional[str]): + integration_id: str, name: str, props: list[str], logo: str | None, + item_url: str | None): super().__init__( event_uuid=event_uuid, entity_uuid=entity_uuid, @@ -1828,10 +1827,10 @@ class AddApiIntegrationEvent(AddIntegrationEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - integration_id: str, name: str, props: list[str], logo: Optional[str], - item_url: Optional[str], rq_method: str, rq_url: str, + integration_id: str, name: str, props: list[str], logo: str | None, + item_url: str | None, rq_method: str, rq_url: str, rq_headers: list[MapEntry], rq_body: str, rq_empty_search: bool, - rs_list_field: Optional[str], rs_item_id: Optional[str], + rs_list_field: str | None, rs_item_id: str | None, rs_item_template: str): super().__init__( event_uuid=event_uuid, @@ -1901,8 +1900,8 @@ class AddWidgetIntegrationEvent(AddIntegrationEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - integration_id: str, name: str, props: list[str], logo: Optional[str], - item_url: Optional[str], widget_url: str): + integration_id: str, name: str, props: list[str], logo: str | None, + item_url: str | None, widget_url: str): super().__init__( event_uuid=event_uuid, entity_uuid=entity_uuid, @@ -1952,8 +1951,8 @@ class EditIntegrationEvent(_KMEditEvent, abc.ABC): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], integration_id: EventField[str], name: EventField[str], - props: EventField[list[str]], logo: EventField[Optional[str]], - item_url: EventField[Optional[str]]): + props: EventField[list[str]], logo: EventField[str | None], + item_url: EventField[str | None]): super().__init__( event_uuid=event_uuid, entity_uuid=entity_uuid, @@ -1996,11 +1995,11 @@ class EditApiIntegrationEvent(EditIntegrationEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], integration_id: EventField[str], name: EventField[str], - props: EventField[list[str]], logo: EventField[Optional[str]], - item_url: EventField[Optional[str]], rq_method: EventField[str], + props: EventField[list[str]], logo: EventField[str | None], + item_url: EventField[str | None], rq_method: EventField[str], rq_url: EventField[str], rq_headers: EventField[list[MapEntry]], rq_body: EventField[str], rq_empty_search: EventField[bool], - rs_list_field: EventField[Optional[str]], rs_item_id: EventField[Optional[str]], + rs_list_field: EventField[str | None], rs_item_id: EventField[str | None], rs_item_template: EventField[str]): super().__init__( event_uuid=event_uuid, @@ -2077,8 +2076,8 @@ class EditWidgetIntegrationEvent(EditIntegrationEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], integration_id: EventField[str], name: EventField[str], - props: EventField[list[str]], logo: EventField[Optional[str]], - item_url: EventField[Optional[str]], widget_url: EventField[str]): + props: EventField[list[str]], logo: EventField[str | None], + item_url: EventField[str | None], widget_url: EventField[str]): super().__init__( event_uuid=event_uuid, entity_uuid=entity_uuid, @@ -2152,7 +2151,7 @@ class AddMetricEvent(_KMAddEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - title: str, abbreviation: Optional[str], description: Optional[str]): + title: str, abbreviation: str | None, description: str | None): super().__init__( event_uuid=event_uuid, entity_uuid=entity_uuid, @@ -2195,8 +2194,8 @@ class EditMetricEvent(_KMEditEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], - title: EventField[str], abbreviation: EventField[Optional[str]], - description: EventField[Optional[str]]): + title: EventField[str], abbreviation: EventField[str | None], + description: EventField[str | None]): super().__init__( event_uuid=event_uuid, entity_uuid=entity_uuid, @@ -2264,7 +2263,7 @@ class AddPhaseEvent(_KMAddEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: list[MapEntry], - title: str, description: Optional[str]): + title: str, description: str | None): super().__init__( event_uuid=event_uuid, entity_uuid=entity_uuid, @@ -2304,7 +2303,7 @@ class EditPhaseEvent(_KMEditEvent): def __init__(self, event_uuid: str, entity_uuid: str, parent_uuid: str, created_at: str, annotations: EventField[list[MapEntry]], - title: EventField[str], description: EventField[Optional[str]]): + title: EventField[str], description: EventField[str | None]): super().__init__( event_uuid=event_uuid, entity_uuid=entity_uuid, diff --git a/packages/dsw-models/dsw/models/km/package.py b/packages/dsw-models/dsw/models/km/package.py index edc4019e..ca40b4ca 100644 --- a/packages/dsw-models/dsw/models/km/package.py +++ b/packages/dsw-models/dsw/models/km/package.py @@ -1,14 +1,12 @@ from .events import _KMEvent, Event -from typing import Optional - class Package: def __init__(self, km_id: str, org_id: str, version: str, name: str, metamodel_version: int, description: str, license: str, - readme: str, created_at: str, fork_pkg_id: Optional[str], - merge_pkg_id: Optional[str], prev_pkg_id: Optional[str]): + readme: str, created_at: str, fork_pkg_id: str | None, + merge_pkg_id: str | None, prev_pkg_id: str | None): self.km_id = km_id self.org_id = org_id self.version = version @@ -21,7 +19,7 @@ def __init__(self, km_id: str, org_id: str, version: str, name: str, self.fork_pkg_id = fork_pkg_id self.merge_pkg_id = merge_pkg_id self.prev_pkg_id = prev_pkg_id - self.events = list() # type: list[_KMEvent] + self.events: list[_KMEvent] = list() @property def id(self): diff --git a/packages/dsw-models/pyproject.toml b/packages/dsw-models/pyproject.toml index 09ec46d8..fb46ec86 100644 --- a/packages/dsw-models/pyproject.toml +++ b/packages/dsw-models/pyproject.toml @@ -16,12 +16,12 @@ classifiers = [ 'Development Status :: 4 - Beta', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Text Processing', 'Topic :: Utilities', ] -requires-python = '>=3.10, <4' +requires-python = '>=3.11, <4' dependencies = [ ] diff --git a/packages/dsw-storage/dsw/storage/s3storage.py b/packages/dsw-storage/dsw/storage/s3storage.py index ac0b8a9f..f084547e 100644 --- a/packages/dsw-storage/dsw/storage/s3storage.py +++ b/packages/dsw-storage/dsw/storage/s3storage.py @@ -1,13 +1,12 @@ import contextlib import io import logging -import minio # type: ignore -import minio.error # type: ignore import pathlib import tempfile -import tenacity -from typing import Optional +import minio +import minio.error +import tenacity from dsw.config.model import S3Config @@ -71,7 +70,7 @@ def ensure_bucket(self): ) def store_document(self, tenant_uuid: str, file_name: str, content_type: str, data: bytes, - metadata: Optional[dict] = None): + metadata: dict | None = None): object_name = f'{DOCUMENTS_DIR}/{file_name}' if self.multi_tenant: object_name = f'{tenant_uuid}/{object_name}' @@ -114,7 +113,7 @@ def download_file(self, file_name: str, target_path: pathlib.Path) -> bool: ) def store_object(self, tenant_uuid: str, object_name: str, content_type: str, data: bytes, - metadata: Optional[dict] = None): + metadata: dict | None = None): if self.multi_tenant: object_name = f'{tenant_uuid}/{object_name}' with io.BytesIO(data) as file: diff --git a/packages/dsw-storage/pyproject.toml b/packages/dsw-storage/pyproject.toml index 9352be94..db3163ef 100644 --- a/packages/dsw-storage/pyproject.toml +++ b/packages/dsw-storage/pyproject.toml @@ -16,12 +16,12 @@ classifiers = [ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Communications :: File Sharing', 'Topic :: Utilities', ] -requires-python = '>=3.10, <4' +requires-python = '>=3.11, <4' dependencies = [ 'minio', 'tenacity', diff --git a/packages/dsw-tdk/pyproject.toml b/packages/dsw-tdk/pyproject.toml index 922fb8e9..0a4317e8 100644 --- a/packages/dsw-tdk/pyproject.toml +++ b/packages/dsw-tdk/pyproject.toml @@ -17,13 +17,13 @@ classifiers = [ 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: Apache Software License', 'Programming Language :: Python', - 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'Topic :: Internet :: WWW/HTTP', 'Topic :: Utilities', ] -requires-python = '>=3.9, <4' +requires-python = '>=3.10, <4' dependencies = [ 'aiohttp', 'click', diff --git a/scripts/clean.sh b/scripts/clean.sh new file mode 100755 index 00000000..cc71533c --- /dev/null +++ b/scripts/clean.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env sh +set -e + +for PKG in $(ls packages); do + echo "Cleaning $PKG" + rm -rf "packages/$PKG/build" + rm -rf "packages/$PKG/env" + find "packages/$PKG" | grep -E "(.egg-info$)" | xargs rm -rf + find "packages/$PKG" | grep -E "(/__pycache__$|\.pyc$|\.pyo$)" | xargs rm -rf +done