From 3c50d4a9c393029e3e6e26e724d3273c14496bda Mon Sep 17 00:00:00 2001 From: otherJL0 Date: Mon, 4 Oct 2021 11:18:01 -0400 Subject: [PATCH 1/4] Add pylint, pre-commit, and formatting tools (isort, black) to pyproject dev dependencies list --- pyproject.toml | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e8da8d6..7bd88f5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,10 +46,23 @@ server = ["starlette"] [tool.poetry.dev-dependencies] parameterized = "*" +pylint = "*" +pre-commit = "*" +black = "*" +isort = "*" + +[tool.isort] +profile = "black" + + +[tool.black] +line-length = 88 +target-version = ["py36"] +skip-string-normalization = true [build-system] requires = ["poetry-core>=1.0.0"] build-backend = "poetry.core.masonry.api" [tool.poetry.scripts] -preql = 'preql.__main__:main' \ No newline at end of file +preql = 'preql.__main__:main' From 06a26fd95c60f01d6001ff079d2826f56414913d Mon Sep 17 00:00:00 2001 From: otherJL0 Date: Mon, 4 Oct 2021 11:35:10 -0400 Subject: [PATCH 2/4] Updating lockfile --- poetry.lock | 809 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 809 insertions(+) create mode 100644 poetry.lock diff --git a/poetry.lock b/poetry.lock new file mode 100644 index 0000000..6ba1e60 --- /dev/null +++ b/poetry.lock @@ -0,0 +1,809 @@ +[[package]] +name = "appdirs" +version = "1.4.4" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "arrow" +version = "1.2.0" +description = "Better dates & times for Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +python-dateutil = ">=2.7.0" +typing-extensions = {version = "*", markers = "python_version < \"3.8\""} + +[[package]] +name = "astroid" +version = "2.8.0" +description = "An abstract syntax tree for Python with inference support." +category = "dev" +optional = false +python-versions = "~=3.6" + +[package.dependencies] +lazy-object-proxy = ">=1.4.0" +typed-ast = {version = ">=1.4.0,<1.5", markers = "implementation_name == \"cpython\" and python_version < \"3.8\""} +typing-extensions = {version = ">=3.10", markers = "python_version < \"3.10\""} +wrapt = ">=1.11,<1.13" + +[[package]] +name = "backports.entry-points-selectable" +version = "1.1.0" +description = "Compatibility shim providing selectable entry points for older implementations" +category = "dev" +optional = false +python-versions = ">=2.7" + +[package.dependencies] +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=4.6)", "pytest-flake8", "pytest-cov", "pytest-black (>=0.3.7)", "pytest-mypy", "pytest-checkdocs (>=2.4)", "pytest-enabler (>=1.0.1)"] + +[[package]] +name = "black" +version = "20.8b1" +description = "The uncompromising code formatter." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +appdirs = "*" +click = ">=7.1.2" +dataclasses = {version = ">=0.6", markers = "python_version < \"3.7\""} +mypy-extensions = ">=0.4.3" +pathspec = ">=0.6,<1" +regex = ">=2020.1.8" +toml = ">=0.10.1" +typed-ast = ">=1.4.0" +typing-extensions = ">=3.7.4" + +[package.extras] +colorama = ["colorama (>=0.4.3)"] +d = ["aiohttp (>=3.3.2)", "aiohttp-cors"] + +[[package]] +name = "cfgv" +version = "3.0.0" +description = "Validate configuration and produce human readable error messages." +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "click" +version = "8.0.1" +description = "Composable command line interface toolkit" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} + +[[package]] +name = "colorama" +version = "0.4.4" +description = "Cross-platform colored terminal text." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" + +[[package]] +name = "commonmark" +version = "0.9.1" +description = "Python parser for the CommonMark Markdown spec" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] + +[[package]] +name = "dataclasses" +version = "0.8" +description = "A backport of the dataclasses module for Python 3.6" +category = "main" +optional = false +python-versions = ">=3.6, <3.7" + +[[package]] +name = "distlib" +version = "0.3.3" +description = "Distribution utilities" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "dsnparse" +version = "0.1.15" +description = "parse dsn urls" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "filelock" +version = "3.3.0" +description = "A platform independent file lock." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["furo (>=2021.8.17b43)", "sphinx (>=4.1)", "sphinx-autodoc-typehints (>=1.12)"] +testing = ["covdefaults (>=1.2.0)", "coverage (>=4)", "pytest (>=4)", "pytest-cov", "pytest-timeout (>=1.4.2)"] + +[[package]] +name = "identify" +version = "1.6.2" +description = "File identification library for Python" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" + +[package.extras] +license = ["editdistance"] + +[[package]] +name = "importlib-metadata" +version = "4.8.1" +description = "Read metadata from Python packages" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} +zipp = ">=0.5" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +perf = ["ipython"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pep517", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy", "importlib-resources (>=1.3)"] + +[[package]] +name = "importlib-resources" +version = "5.2.2" +description = "Read resources from Python packages" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "pytest-black (>=0.3.7)", "pytest-mypy"] + +[[package]] +name = "isort" +version = "5.8.0" +description = "A Python utility / library to sort Python imports." +category = "dev" +optional = false +python-versions = ">=3.6,<4.0" + +[package.extras] +pipfile_deprecated_finder = ["pipreqs", "requirementslib"] +requirements_deprecated_finder = ["pipreqs", "pip-api"] +colors = ["colorama (>=0.4.3,<0.5.0)"] + +[[package]] +name = "lark-parser" +version = "0.11.3" +description = "a modern parsing library" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +atomic_cache = ["atomicwrites"] +nearley = ["js2py"] +regex = ["regex"] + +[[package]] +name = "lazy-object-proxy" +version = "1.6.0" +description = "A fast and thorough lazy object proxy." +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[[package]] +name = "mccabe" +version = "0.6.1" +description = "McCabe checker, plugin for flake8" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "mypy-extensions" +version = "0.4.3" +description = "Experimental type system extensions for programs checked with the mypy typechecker." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "mysqlclient" +version = "2.0.3" +description = "Python interface to MySQL" +category = "main" +optional = true +python-versions = ">=3.5" + +[[package]] +name = "nodeenv" +version = "1.6.0" +description = "Node.js virtual environment builder" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "parameterized" +version = "0.8.1" +description = "Parameterized testing with any Python test framework" +category = "dev" +optional = false +python-versions = "*" + +[package.extras] +dev = ["jinja2"] + +[[package]] +name = "pathspec" +version = "0.9.0" +description = "Utility library for gitignore style pattern matching of file paths." +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[[package]] +name = "platformdirs" +version = "2.4.0" +description = "A small Python module for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["Sphinx (>=4)", "furo (>=2021.7.5b38)", "proselint (>=0.10.2)", "sphinx-autodoc-typehints (>=1.12)"] +test = ["appdirs (==1.4.4)", "pytest (>=6)", "pytest-cov (>=2.7)", "pytest-mock (>=3.6)"] + +[[package]] +name = "pre-commit" +version = "2.1.1" +description = "A framework for managing and maintaining multi-language pre-commit hooks." +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +cfgv = ">=2.0.0" +identify = ">=1.0.0" +importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +importlib-resources = {version = "*", markers = "python_version < \"3.7\""} +nodeenv = ">=0.11.1" +pyyaml = ">=5.1" +toml = "*" +virtualenv = ">=15.2" + +[[package]] +name = "prompt-toolkit" +version = "3.0.3" +description = "Library for building powerful interactive command lines in Python" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.dependencies] +wcwidth = "*" + +[[package]] +name = "psycopg2" +version = "2.9.1" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +category = "main" +optional = true +python-versions = ">=3.6" + +[[package]] +name = "pygments" +version = "2.10.0" +description = "Pygments is a syntax highlighting package written in Python." +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "pylint" +version = "2.11.1" +description = "python code static checker" +category = "dev" +optional = false +python-versions = "~=3.6" + +[package.dependencies] +astroid = ">=2.8.0,<2.9" +colorama = {version = "*", markers = "sys_platform == \"win32\""} +isort = ">=4.2.5,<6" +mccabe = ">=0.6,<0.7" +platformdirs = ">=2.2.0" +toml = ">=0.7.1" +typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} + +[[package]] +name = "python-dateutil" +version = "2.8.2" +description = "Extensions to the standard Python datetime module" +category = "main" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" + +[package.dependencies] +six = ">=1.5" + +[[package]] +name = "pyyaml" +version = "5.4.1" +description = "YAML parser and emitter for Python" +category = "dev" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[[package]] +name = "regex" +version = "2021.9.30" +description = "Alternative regular expression module, to replace re." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "rich" +version = "10.11.0" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" +optional = false +python-versions = ">=3.6,<4.0" + +[package.dependencies] +colorama = ">=0.4.0,<0.5.0" +commonmark = ">=0.9.0,<0.10.0" +dataclasses = {version = ">=0.7,<0.9", markers = "python_version >= \"3.6\" and python_version < \"3.7\""} +pygments = ">=2.6.0,<3.0.0" +typing-extensions = {version = ">=3.7.4,<4.0.0", markers = "python_version < \"3.8\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] + +[[package]] +name = "runtype" +version = "0.1.16" +description = "Type dispatch and validation for run-time Python" +category = "main" +optional = false +python-versions = ">=3.6,<4.0" + +[package.dependencies] +dataclasses = {version = "*", markers = "python_version >= \"3.6\" and python_version < \"3.7\""} + +[[package]] +name = "six" +version = "1.16.0" +description = "Python 2 and 3 compatibility utilities" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "starlette" +version = "0.14.2" +description = "The little ASGI library that shines." +category = "main" +optional = true +python-versions = ">=3.6" + +[package.extras] +full = ["aiofiles", "graphene", "itsdangerous", "jinja2", "python-multipart", "pyyaml", "requests"] + +[[package]] +name = "toml" +version = "0.10.2" +description = "Python Library for Tom's Obvious, Minimal Language" +category = "dev" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" + +[[package]] +name = "typed-ast" +version = "1.4.3" +description = "a fork of Python 2 and 3 ast modules with type comment support" +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "typing-extensions" +version = "3.10.0.2" +description = "Backported and Experimental Type Hints for Python 3.5+" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "virtualenv" +version = "20.8.1" +description = "Virtual Python Environment builder" +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" + +[package.dependencies] +"backports.entry-points-selectable" = ">=1.0.4" +distlib = ">=0.3.1,<1" +filelock = ">=3.0.0,<4" +importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +importlib-resources = {version = ">=1.0", markers = "python_version < \"3.7\""} +platformdirs = ">=2,<3" +six = ">=1.9.0,<2" + +[package.extras] +docs = ["proselint (>=0.10.2)", "sphinx (>=3)", "sphinx-argparse (>=0.2.5)", "sphinx-rtd-theme (>=0.4.3)", "towncrier (>=19.9.0rc1)"] +testing = ["coverage (>=4)", "coverage-enable-subprocess (>=1)", "flaky (>=3)", "pytest (>=4)", "pytest-env (>=0.6.2)", "pytest-freezegun (>=0.4.1)", "pytest-mock (>=2)", "pytest-randomly (>=1)", "pytest-timeout (>=1)", "packaging (>=20.0)"] + +[[package]] +name = "wcwidth" +version = "0.2.5" +description = "Measures the displayed width of unicode strings in a terminal" +category = "main" +optional = false +python-versions = "*" + +[[package]] +name = "wrapt" +version = "1.12.1" +description = "Module for decorators, wrappers and monkey patching." +category = "dev" +optional = false +python-versions = "*" + +[[package]] +name = "zipp" +version = "3.6.0" +description = "Backport of pathlib-compatible object wrapper for zip files" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +docs = ["sphinx", "jaraco.packaging (>=8.2)", "rst.linker (>=1.9)"] +testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy"] + +[extras] +mysql = ["mysqlclient"] +pgsql = ["psycopg2"] +server = ["starlette"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.6" +content-hash = "cdf38396fe54e4141bf65795501317ddba310e391427a3211802159d324b4ad8" + +[metadata.files] +appdirs = [ + {file = "appdirs-1.4.4-py2.py3-none-any.whl", hash = "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"}, + {file = "appdirs-1.4.4.tar.gz", hash = "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41"}, +] +arrow = [ + {file = "arrow-1.2.0-py3-none-any.whl", hash = "sha256:8fb7d9d3d4bf90e49e734c22fa077bdd0964135c4b8120de2510575a8d1f620c"}, + {file = "arrow-1.2.0.tar.gz", hash = "sha256:16fc29bbd9e425e3eb0fef3018297910a0f4568f21116fc31771e2760a50e074"}, +] +astroid = [ + {file = "astroid-2.8.0-py3-none-any.whl", hash = "sha256:dcc06f6165f415220013801642bd6c9808a02967070919c4b746c6864c205471"}, + {file = "astroid-2.8.0.tar.gz", hash = "sha256:fe81f80c0b35264acb5653302ffbd935d394f1775c5e4487df745bf9c2442708"}, +] +"backports.entry-points-selectable" = [ + {file = "backports.entry_points_selectable-1.1.0-py2.py3-none-any.whl", hash = "sha256:a6d9a871cde5e15b4c4a53e3d43ba890cc6861ec1332c9c2428c92f977192acc"}, + {file = "backports.entry_points_selectable-1.1.0.tar.gz", hash = "sha256:988468260ec1c196dab6ae1149260e2f5472c9110334e5d51adcb77867361f6a"}, +] +black = [ + {file = "black-20.8b1.tar.gz", hash = "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"}, +] +cfgv = [ + {file = "cfgv-3.0.0-py2.py3-none-any.whl", hash = "sha256:f22b426ed59cd2ab2b54ff96608d846c33dfb8766a67f0b4a6ce130ce244414f"}, + {file = "cfgv-3.0.0.tar.gz", hash = "sha256:04b093b14ddf9fd4d17c53ebfd55582d27b76ed30050193c14e560770c5360eb"}, +] +click = [ + {file = "click-8.0.1-py3-none-any.whl", hash = "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"}, + {file = "click-8.0.1.tar.gz", hash = "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a"}, +] +colorama = [ + {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, + {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, +] +commonmark = [ + {file = "commonmark-0.9.1-py2.py3-none-any.whl", hash = "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9"}, + {file = "commonmark-0.9.1.tar.gz", hash = "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60"}, +] +dataclasses = [ + {file = "dataclasses-0.8-py3-none-any.whl", hash = "sha256:0201d89fa866f68c8ebd9d08ee6ff50c0b255f8ec63a71c16fda7af82bb887bf"}, + {file = "dataclasses-0.8.tar.gz", hash = "sha256:8479067f342acf957dc82ec415d355ab5edb7e7646b90dc6e2fd1d96ad084c97"}, +] +distlib = [ + {file = "distlib-0.3.3-py2.py3-none-any.whl", hash = "sha256:c8b54e8454e5bf6237cc84c20e8264c3e991e824ef27e8f1e81049867d861e31"}, + {file = "distlib-0.3.3.zip", hash = "sha256:d982d0751ff6eaaab5e2ec8e691d949ee80eddf01a62eaa96ddb11531fe16b05"}, +] +dsnparse = [ + {file = "dsnparse-0.1.15.tar.gz", hash = "sha256:2ac5705b17cb28e8b115053c2d51cf3321dc2041b1d75e2db6157e05146d0fba"}, +] +filelock = [ + {file = "filelock-3.3.0-py3-none-any.whl", hash = "sha256:bbc6a0382fe8ec4744ecdf6683a2e07f65eb10ff1aff53fc02a202565446cde0"}, + {file = "filelock-3.3.0.tar.gz", hash = "sha256:8c7eab13dc442dc249e95158bcc12dec724465919bdc9831fdbf0660f03d1785"}, +] +identify = [ + {file = "identify-1.6.2-py2.py3-none-any.whl", hash = "sha256:8f9879b5b7cca553878d31548a419ec2f227d3328da92fe8202bc5e546d5cbc3"}, + {file = "identify-1.6.2.tar.gz", hash = "sha256:1c2014f6985ed02e62b2e6955578acf069cb2c54859e17853be474bfe7e13bed"}, +] +importlib-metadata = [ + {file = "importlib_metadata-4.8.1-py3-none-any.whl", hash = "sha256:b618b6d2d5ffa2f16add5697cf57a46c76a56229b0ed1c438322e4e95645bd15"}, + {file = "importlib_metadata-4.8.1.tar.gz", hash = "sha256:f284b3e11256ad1e5d03ab86bb2ccd6f5339688ff17a4d797a0fe7df326f23b1"}, +] +importlib-resources = [ + {file = "importlib_resources-5.2.2-py3-none-any.whl", hash = "sha256:2480d8e07d1890056cb53c96e3de44fead9c62f2ba949b0f2e4c4345f4afa977"}, + {file = "importlib_resources-5.2.2.tar.gz", hash = "sha256:a65882a4d0fe5fbf702273456ba2ce74fe44892c25e42e057aca526b702a6d4b"}, +] +isort = [ + {file = "isort-5.8.0-py3-none-any.whl", hash = "sha256:2bb1680aad211e3c9944dbce1d4ba09a989f04e238296c87fe2139faa26d655d"}, + {file = "isort-5.8.0.tar.gz", hash = "sha256:0a943902919f65c5684ac4e0154b1ad4fac6dcaa5d9f3426b732f1c8b5419be6"}, +] +lark-parser = [ + {file = "lark-parser-0.11.3.tar.gz", hash = "sha256:e29ca814a98bb0f81674617d878e5f611cb993c19ea47f22c80da3569425f9bd"}, +] +lazy-object-proxy = [ + {file = "lazy-object-proxy-1.6.0.tar.gz", hash = "sha256:489000d368377571c6f982fba6497f2aa13c6d1facc40660963da62f5c379726"}, + {file = "lazy_object_proxy-1.6.0-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:c6938967f8528b3668622a9ed3b31d145fab161a32f5891ea7b84f6b790be05b"}, + {file = "lazy_object_proxy-1.6.0-cp27-cp27m-win32.whl", hash = "sha256:ebfd274dcd5133e0afae738e6d9da4323c3eb021b3e13052d8cbd0e457b1256e"}, + {file = "lazy_object_proxy-1.6.0-cp27-cp27m-win_amd64.whl", hash = "sha256:ed361bb83436f117f9917d282a456f9e5009ea12fd6de8742d1a4752c3017e93"}, + {file = "lazy_object_proxy-1.6.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d900d949b707778696fdf01036f58c9876a0d8bfe116e8d220cfd4b15f14e741"}, + {file = "lazy_object_proxy-1.6.0-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:5743a5ab42ae40caa8421b320ebf3a998f89c85cdc8376d6b2e00bd12bd1b587"}, + {file = "lazy_object_proxy-1.6.0-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:bf34e368e8dd976423396555078def5cfc3039ebc6fc06d1ae2c5a65eebbcde4"}, + {file = "lazy_object_proxy-1.6.0-cp36-cp36m-win32.whl", hash = "sha256:b579f8acbf2bdd9ea200b1d5dea36abd93cabf56cf626ab9c744a432e15c815f"}, + {file = "lazy_object_proxy-1.6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:4f60460e9f1eb632584c9685bccea152f4ac2130e299784dbaf9fae9f49891b3"}, + {file = "lazy_object_proxy-1.6.0-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:d7124f52f3bd259f510651450e18e0fd081ed82f3c08541dffc7b94b883aa981"}, + {file = "lazy_object_proxy-1.6.0-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:22ddd618cefe54305df49e4c069fa65715be4ad0e78e8d252a33debf00f6ede2"}, + {file = "lazy_object_proxy-1.6.0-cp37-cp37m-win32.whl", hash = "sha256:9d397bf41caad3f489e10774667310d73cb9c4258e9aed94b9ec734b34b495fd"}, + {file = "lazy_object_proxy-1.6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:24a5045889cc2729033b3e604d496c2b6f588c754f7a62027ad4437a7ecc4837"}, + {file = "lazy_object_proxy-1.6.0-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:17e0967ba374fc24141738c69736da90e94419338fd4c7c7bef01ee26b339653"}, + {file = "lazy_object_proxy-1.6.0-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:410283732af311b51b837894fa2f24f2c0039aa7f220135192b38fcc42bd43d3"}, + {file = "lazy_object_proxy-1.6.0-cp38-cp38-win32.whl", hash = "sha256:85fb7608121fd5621cc4377a8961d0b32ccf84a7285b4f1d21988b2eae2868e8"}, + {file = "lazy_object_proxy-1.6.0-cp38-cp38-win_amd64.whl", hash = "sha256:d1c2676e3d840852a2de7c7d5d76407c772927addff8d742b9808fe0afccebdf"}, + {file = "lazy_object_proxy-1.6.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:b865b01a2e7f96db0c5d12cfea590f98d8c5ba64ad222300d93ce6ff9138bcad"}, + {file = "lazy_object_proxy-1.6.0-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:4732c765372bd78a2d6b2150a6e99d00a78ec963375f236979c0626b97ed8e43"}, + {file = "lazy_object_proxy-1.6.0-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:9698110e36e2df951c7c36b6729e96429c9c32b3331989ef19976592c5f3c77a"}, + {file = "lazy_object_proxy-1.6.0-cp39-cp39-win32.whl", hash = "sha256:1fee665d2638491f4d6e55bd483e15ef21f6c8c2095f235fef72601021e64f61"}, + {file = "lazy_object_proxy-1.6.0-cp39-cp39-win_amd64.whl", hash = "sha256:f5144c75445ae3ca2057faac03fda5a902eff196702b0a24daf1d6ce0650514b"}, +] +mccabe = [ + {file = "mccabe-0.6.1-py2.py3-none-any.whl", hash = "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42"}, + {file = "mccabe-0.6.1.tar.gz", hash = "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f"}, +] +mypy-extensions = [ + {file = "mypy_extensions-0.4.3-py2.py3-none-any.whl", hash = "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d"}, + {file = "mypy_extensions-0.4.3.tar.gz", hash = "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"}, +] +mysqlclient = [ + {file = "mysqlclient-2.0.3-cp36-cp36m-win_amd64.whl", hash = "sha256:3381ca1a4f37ff1155fcfde20836b46416d66531add8843f6aa6d968982731c3"}, + {file = "mysqlclient-2.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0ac0dd759c4ca02c35a9fedc24bc982cf75171651e8187c2495ec957a87dfff7"}, + {file = "mysqlclient-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:71c4b330cf2313bbda0307fc858cc9055e64493ba9bf28454d25cf8b3ee8d7f5"}, + {file = "mysqlclient-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:fc575093cf81b6605bed84653e48b277318b880dc9becf42dd47fa11ffd3e2b6"}, + {file = "mysqlclient-2.0.3.tar.gz", hash = "sha256:f6ebea7c008f155baeefe16c56cd3ee6239f7a5a9ae42396c2f1860f08a7c432"}, +] +nodeenv = [ + {file = "nodeenv-1.6.0-py2.py3-none-any.whl", hash = "sha256:621e6b7076565ddcacd2db0294c0381e01fd28945ab36bcf00f41c5daf63bef7"}, + {file = "nodeenv-1.6.0.tar.gz", hash = "sha256:3ef13ff90291ba2a4a7a4ff9a979b63ffdd00a464dbe04acf0ea6471517a4c2b"}, +] +parameterized = [ + {file = "parameterized-0.8.1-py2.py3-none-any.whl", hash = "sha256:9cbb0b69a03e8695d68b3399a8a5825200976536fe1cb79db60ed6a4c8c9efe9"}, + {file = "parameterized-0.8.1.tar.gz", hash = "sha256:41bbff37d6186430f77f900d777e5bb6a24928a1c46fb1de692f8b52b8833b5c"}, +] +pathspec = [ + {file = "pathspec-0.9.0-py2.py3-none-any.whl", hash = "sha256:7d15c4ddb0b5c802d161efc417ec1a2558ea2653c2e8ad9c19098201dc1c993a"}, + {file = "pathspec-0.9.0.tar.gz", hash = "sha256:e564499435a2673d586f6b2130bb5b95f04a3ba06f81b8f895b651a3c76aabb1"}, +] +platformdirs = [ + {file = "platformdirs-2.4.0-py3-none-any.whl", hash = "sha256:8868bbe3c3c80d42f20156f22e7131d2fb321f5bc86a2a345375c6481a67021d"}, + {file = "platformdirs-2.4.0.tar.gz", hash = "sha256:367a5e80b3d04d2428ffa76d33f124cf11e8fff2acdaa9b43d545f5c7d661ef2"}, +] +pre-commit = [ + {file = "pre_commit-2.1.1-py2.py3-none-any.whl", hash = "sha256:09ebe467f43ce24377f8c2f200fe3cd2570d328eb2ce0568c8e96ce19da45fa6"}, + {file = "pre_commit-2.1.1.tar.gz", hash = "sha256:f8d555e31e2051892c7f7b3ad9f620bd2c09271d87e9eedb2ad831737d6211eb"}, +] +prompt-toolkit = [ + {file = "prompt_toolkit-3.0.3-py3-none-any.whl", hash = "sha256:c93e53af97f630f12f5f62a3274e79527936ed466f038953dfa379d4941f651a"}, + {file = "prompt_toolkit-3.0.3.tar.gz", hash = "sha256:a402e9bf468b63314e37460b68ba68243d55b2f8c4d0192f85a019af3945050e"}, +] +psycopg2 = [ + {file = "psycopg2-2.9.1-cp36-cp36m-win32.whl", hash = "sha256:7f91312f065df517187134cce8e395ab37f5b601a42446bdc0f0d51773621854"}, + {file = "psycopg2-2.9.1-cp36-cp36m-win_amd64.whl", hash = "sha256:830c8e8dddab6b6716a4bf73a09910c7954a92f40cf1d1e702fb93c8a919cc56"}, + {file = "psycopg2-2.9.1-cp37-cp37m-win32.whl", hash = "sha256:89409d369f4882c47f7ea20c42c5046879ce22c1e4ea20ef3b00a4dfc0a7f188"}, + {file = "psycopg2-2.9.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7640e1e4d72444ef012e275e7b53204d7fab341fb22bc76057ede22fe6860b25"}, + {file = "psycopg2-2.9.1-cp38-cp38-win32.whl", hash = "sha256:079d97fc22de90da1d370c90583659a9f9a6ee4007355f5825e5f1c70dffc1fa"}, + {file = "psycopg2-2.9.1-cp38-cp38-win_amd64.whl", hash = "sha256:2c992196719fadda59f72d44603ee1a2fdcc67de097eea38d41c7ad9ad246e62"}, + {file = "psycopg2-2.9.1-cp39-cp39-win32.whl", hash = "sha256:2087013c159a73e09713294a44d0c8008204d06326006b7f652bef5ace66eebb"}, + {file = "psycopg2-2.9.1-cp39-cp39-win_amd64.whl", hash = "sha256:bf35a25f1aaa8a3781195595577fcbb59934856ee46b4f252f56ad12b8043bcf"}, + {file = "psycopg2-2.9.1.tar.gz", hash = "sha256:de5303a6f1d0a7a34b9d40e4d3bef684ccc44a49bbe3eb85e3c0bffb4a131b7c"}, +] +pygments = [ + {file = "Pygments-2.10.0-py3-none-any.whl", hash = "sha256:b8e67fe6af78f492b3c4b3e2970c0624cbf08beb1e493b2c99b9fa1b67a20380"}, + {file = "Pygments-2.10.0.tar.gz", hash = "sha256:f398865f7eb6874156579fdf36bc840a03cab64d1cde9e93d68f46a425ec52c6"}, +] +pylint = [ + {file = "pylint-2.11.1-py3-none-any.whl", hash = "sha256:0f358e221c45cbd4dad2a1e4b883e75d28acdcccd29d40c76eb72b307269b126"}, + {file = "pylint-2.11.1.tar.gz", hash = "sha256:2c9843fff1a88ca0ad98a256806c82c5a8f86086e7ccbdb93297d86c3f90c436"}, +] +python-dateutil = [ + {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, + {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, +] +pyyaml = [ + {file = "PyYAML-5.4.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:3b2b1824fe7112845700f815ff6a489360226a5609b96ec2190a45e62a9fc922"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win32.whl", hash = "sha256:129def1b7c1bf22faffd67b8f3724645203b79d8f4cc81f674654d9902cb4393"}, + {file = "PyYAML-5.4.1-cp27-cp27m-win_amd64.whl", hash = "sha256:4465124ef1b18d9ace298060f4eccc64b0850899ac4ac53294547536533800c8"}, + {file = "PyYAML-5.4.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:bb4191dfc9306777bc594117aee052446b3fa88737cd13b7188d0e7aa8162185"}, + {file = "PyYAML-5.4.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:6c78645d400265a062508ae399b60b8c167bf003db364ecb26dcab2bda048253"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:4e0583d24c881e14342eaf4ec5fbc97f934b999a6828693a99157fde912540cc"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:72a01f726a9c7851ca9bfad6fd09ca4e090a023c00945ea05ba1638c09dc3347"}, + {file = "PyYAML-5.4.1-cp36-cp36m-manylinux2014_s390x.whl", hash = "sha256:895f61ef02e8fed38159bb70f7e100e00f471eae2bc838cd0f4ebb21e28f8541"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win32.whl", hash = "sha256:3bd0e463264cf257d1ffd2e40223b197271046d09dadf73a0fe82b9c1fc385a5"}, + {file = "PyYAML-5.4.1-cp36-cp36m-win_amd64.whl", hash = "sha256:e4fac90784481d221a8e4b1162afa7c47ed953be40d31ab4629ae917510051df"}, + {file = "PyYAML-5.4.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5accb17103e43963b80e6f837831f38d314a0495500067cb25afab2e8d7a4018"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:e1d4970ea66be07ae37a3c2e48b5ec63f7ba6804bdddfdbd3cfd954d25a82e63"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:cb333c16912324fd5f769fff6bc5de372e9e7a202247b48870bc251ed40239aa"}, + {file = "PyYAML-5.4.1-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:fe69978f3f768926cfa37b867e3843918e012cf83f680806599ddce33c2c68b0"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win32.whl", hash = "sha256:dd5de0646207f053eb0d6c74ae45ba98c3395a571a2891858e87df7c9b9bd51b"}, + {file = "PyYAML-5.4.1-cp37-cp37m-win_amd64.whl", hash = "sha256:08682f6b72c722394747bddaf0aa62277e02557c0fd1c42cb853016a38f8dedf"}, + {file = "PyYAML-5.4.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d2d9808ea7b4af864f35ea216be506ecec180628aced0704e34aca0b040ffe46"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:8c1be557ee92a20f184922c7b6424e8ab6691788e6d86137c5d93c1a6ec1b8fb"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:fd7f6999a8070df521b6384004ef42833b9bd62cfee11a09bda1079b4b704247"}, + {file = "PyYAML-5.4.1-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:bfb51918d4ff3d77c1c856a9699f8492c612cde32fd3bcd344af9be34999bfdc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win32.whl", hash = "sha256:fa5ae20527d8e831e8230cbffd9f8fe952815b2b7dae6ffec25318803a7528fc"}, + {file = "PyYAML-5.4.1-cp38-cp38-win_amd64.whl", hash = "sha256:0f5f5786c0e09baddcd8b4b45f20a7b5d61a7e7e99846e3c799b05c7c53fa696"}, + {file = "PyYAML-5.4.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:294db365efa064d00b8d1ef65d8ea2c3426ac366c0c4368d930bf1c5fb497f77"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:74c1485f7707cf707a7aef42ef6322b8f97921bd89be2ab6317fd782c2d53183"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:d483ad4e639292c90170eb6f7783ad19490e7a8defb3e46f97dfe4bacae89122"}, + {file = "PyYAML-5.4.1-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:fdc842473cd33f45ff6bce46aea678a54e3d21f1b61a7750ce3c498eedfe25d6"}, + {file = "PyYAML-5.4.1-cp39-cp39-win32.whl", hash = "sha256:49d4cdd9065b9b6e206d0595fee27a96b5dd22618e7520c33204a4a3239d5b10"}, + {file = "PyYAML-5.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:c20cfa2d49991c8b4147af39859b167664f2ad4561704ee74c1de03318e898db"}, + {file = "PyYAML-5.4.1.tar.gz", hash = "sha256:607774cbba28732bfa802b54baa7484215f530991055bb562efbed5b2f20a45e"}, +] +regex = [ + {file = "regex-2021.9.30-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:66696c8336a1b5d1182464f3af3427cc760118f26d0b09a2ddc16a976a4d2637"}, + {file = "regex-2021.9.30-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4d87459ad3ab40cd8493774f8a454b2e490d8e729e7e402a0625867a983e4e02"}, + {file = "regex-2021.9.30-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78cf6a1e023caf5e9a982f5377414e1aeac55198831b852835732cfd0a0ca5ff"}, + {file = "regex-2021.9.30-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:255791523f80ea8e48e79af7120b4697ef3b74f6886995dcdb08c41f8e516be0"}, + {file = "regex-2021.9.30-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e502f8d4e5ef714bcc2c94d499684890c94239526d61fdf1096547db91ca6aa6"}, + {file = "regex-2021.9.30-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4907fb0f9b9309a5bded72343e675a252c2589a41871874feace9a05a540241e"}, + {file = "regex-2021.9.30-cp310-cp310-win32.whl", hash = "sha256:3be40f720af170a6b20ddd2ad7904c58b13d2b56f6734ee5d09bbdeed2fa4816"}, + {file = "regex-2021.9.30-cp310-cp310-win_amd64.whl", hash = "sha256:c2b180ed30856dfa70cfe927b0fd38e6b68198a03039abdbeb1f2029758d87e7"}, + {file = "regex-2021.9.30-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:e6f2d2f93001801296fe3ca86515eb04915472b5380d4d8752f09f25f0b9b0ed"}, + {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fa7ba9ab2eba7284e0d7d94f61df7af86015b0398e123331362270d71fab0b9"}, + {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28040e89a04b60d579c69095c509a4f6a1a5379cd865258e3a186b7105de72c6"}, + {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f588209d3e4797882cd238195c175290dbc501973b10a581086b5c6bcd095ffb"}, + {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:42952d325439ef223e4e9db7ee6d9087b5c68c5c15b1f9de68e990837682fc7b"}, + {file = "regex-2021.9.30-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:cae4099031d80703954c39680323dabd87a69b21262303160776aa0e55970ca0"}, + {file = "regex-2021.9.30-cp36-cp36m-win32.whl", hash = "sha256:0de8ad66b08c3e673b61981b9e3626f8784d5564f8c3928e2ad408c0eb5ac38c"}, + {file = "regex-2021.9.30-cp36-cp36m-win_amd64.whl", hash = "sha256:b345ecde37c86dd7084c62954468a4a655fd2d24fd9b237949dd07a4d0dd6f4c"}, + {file = "regex-2021.9.30-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6f08187136f11e430638c2c66e1db091105d7c2e9902489f0dbc69b44c222b4"}, + {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b55442650f541d195a535ccec33078c78a9521973fb960923da7515e9ed78fa6"}, + {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87e9c489aa98f50f367fb26cc9c8908d668e9228d327644d7aa568d47e456f47"}, + {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:e2cb7d4909ed16ed35729d38af585673f1f0833e73dfdf0c18e5be0061107b99"}, + {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d0861e7f6325e821d5c40514c551fd538b292f8cc3960086e73491b9c5d8291d"}, + {file = "regex-2021.9.30-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:81fdc90f999b2147fc62e303440c424c47e5573a9b615ed5d43a5b832efcca9e"}, + {file = "regex-2021.9.30-cp37-cp37m-win32.whl", hash = "sha256:8c1ad61fa024195136a6b7b89538030bd00df15f90ac177ca278df9b2386c96f"}, + {file = "regex-2021.9.30-cp37-cp37m-win_amd64.whl", hash = "sha256:e3770781353a4886b68ef10cec31c1f61e8e3a0be5f213c2bb15a86efd999bc4"}, + {file = "regex-2021.9.30-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9c065d95a514a06b92a5026766d72ac91bfabf581adb5b29bc5c91d4b3ee9b83"}, + {file = "regex-2021.9.30-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9925985be05d54b3d25fd6c1ea8e50ff1f7c2744c75bdc4d3b45c790afa2bcb3"}, + {file = "regex-2021.9.30-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:470f2c882f2672d8eeda8ab27992aec277c067d280b52541357e1acd7e606dae"}, + {file = "regex-2021.9.30-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ad0517df22a97f1da20d8f1c8cb71a5d1997fa383326b81f9cf22c9dadfbdf34"}, + {file = "regex-2021.9.30-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9e30838df7bfd20db6466fd309d9b580d32855f8e2c2e6d74cf9da27dcd9b63"}, + {file = "regex-2021.9.30-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5b34d2335d6aedec7dcadd3f8283b9682fadad8b9b008da8788d2fce76125ebe"}, + {file = "regex-2021.9.30-cp38-cp38-win32.whl", hash = "sha256:e07049cece3462c626d650e8bf42ddbca3abf4aa08155002c28cb6d9a5a281e2"}, + {file = "regex-2021.9.30-cp38-cp38-win_amd64.whl", hash = "sha256:37868075eda024470bd0feab872c692ac4ee29db1e14baec103257bf6cc64346"}, + {file = "regex-2021.9.30-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d331f238a7accfbbe1c4cd1ba610d4c087b206353539331e32a8f05345c74aec"}, + {file = "regex-2021.9.30-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6348a7ab2a502cbdd0b7fd0496d614007489adb7361956b38044d1d588e66e04"}, + {file = "regex-2021.9.30-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce7b1cca6c23f19bee8dc40228d9c314d86d1e51996b86f924aca302fc8f8bf9"}, + {file = "regex-2021.9.30-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1f1125bc5172ab3a049bc6f4b9c0aae95a2a2001a77e6d6e4239fa3653e202b5"}, + {file = "regex-2021.9.30-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:638e98d069b14113e8afba6a54d1ca123f712c0d105e67c1f9211b2a825ef926"}, + {file = "regex-2021.9.30-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9a0b0db6b49da7fa37ca8eddf9f40a8dbc599bad43e64f452284f37b6c34d91c"}, + {file = "regex-2021.9.30-cp39-cp39-win32.whl", hash = "sha256:9910869c472e5a6728680ca357b5846546cbbd2ab3ad5bef986ef0bc438d0aa6"}, + {file = "regex-2021.9.30-cp39-cp39-win_amd64.whl", hash = "sha256:3b71213ec3bad9a5a02e049f2ec86b3d7c3e350129ae0f4e2f99c12b5da919ed"}, + {file = "regex-2021.9.30.tar.gz", hash = "sha256:81e125d9ba54c34579e4539a967e976a3c56150796674aec318b1b2f49251be7"}, +] +rich = [ + {file = "rich-10.11.0-py3-none-any.whl", hash = "sha256:44bb3f9553d00b3c8938abf89828df870322b9ba43caf3b12bb7758debdc6dec"}, + {file = "rich-10.11.0.tar.gz", hash = "sha256:016fa105f34b69c434e7f908bb5bd7fefa9616efdb218a2917117683a6394ce5"}, +] +runtype = [ + {file = "runtype-0.1.16-py3-none-any.whl", hash = "sha256:fbca54523ac66729ec90afde78ea8f402cc86246be11e3f7967279e4ac56daaf"}, + {file = "runtype-0.1.16.tar.gz", hash = "sha256:0bb3277b17b828ffe838a9997fd50a3417998e7c5930c4ad1f58fb4b8a253bcd"}, +] +six = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] +starlette = [ + {file = "starlette-0.14.2-py3-none-any.whl", hash = "sha256:3c8e48e52736b3161e34c9f0e8153b4f32ec5d8995a3ee1d59410d92f75162ed"}, + {file = "starlette-0.14.2.tar.gz", hash = "sha256:7d49f4a27f8742262ef1470608c59ddbc66baf37c148e938c7038e6bc7a998aa"}, +] +toml = [ + {file = "toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] +typed-ast = [ + {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:2068531575a125b87a41802130fa7e29f26c09a2833fea68d9a40cf33902eba6"}, + {file = "typed_ast-1.4.3-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:c907f561b1e83e93fad565bac5ba9c22d96a54e7ea0267c708bffe863cbe4075"}, + {file = "typed_ast-1.4.3-cp35-cp35m-manylinux2014_aarch64.whl", hash = "sha256:1b3ead4a96c9101bef08f9f7d1217c096f31667617b58de957f690c92378b528"}, + {file = "typed_ast-1.4.3-cp35-cp35m-win32.whl", hash = "sha256:dde816ca9dac1d9c01dd504ea5967821606f02e510438120091b84e852367428"}, + {file = "typed_ast-1.4.3-cp35-cp35m-win_amd64.whl", hash = "sha256:777a26c84bea6cd934422ac2e3b78863a37017618b6e5c08f92ef69853e765d3"}, + {file = "typed_ast-1.4.3-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:f8afcf15cc511ada719a88e013cec87c11aff7b91f019295eb4530f96fe5ef2f"}, + {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_i686.whl", hash = "sha256:52b1eb8c83f178ab787f3a4283f68258525f8d70f778a2f6dd54d3b5e5fb4341"}, + {file = "typed_ast-1.4.3-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:01ae5f73431d21eead5015997ab41afa53aa1fbe252f9da060be5dad2c730ace"}, + {file = "typed_ast-1.4.3-cp36-cp36m-manylinux2014_aarch64.whl", hash = "sha256:c190f0899e9f9f8b6b7863debfb739abcb21a5c054f911ca3596d12b8a4c4c7f"}, + {file = "typed_ast-1.4.3-cp36-cp36m-win32.whl", hash = "sha256:398e44cd480f4d2b7ee8d98385ca104e35c81525dd98c519acff1b79bdaac363"}, + {file = "typed_ast-1.4.3-cp36-cp36m-win_amd64.whl", hash = "sha256:bff6ad71c81b3bba8fa35f0f1921fb24ff4476235a6e94a26ada2e54370e6da7"}, + {file = "typed_ast-1.4.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0fb71b8c643187d7492c1f8352f2c15b4c4af3f6338f21681d3681b3dc31a266"}, + {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:760ad187b1041a154f0e4d0f6aae3e40fdb51d6de16e5c99aedadd9246450e9e"}, + {file = "typed_ast-1.4.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:5feca99c17af94057417d744607b82dd0a664fd5e4ca98061480fd8b14b18d04"}, + {file = "typed_ast-1.4.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:95431a26309a21874005845c21118c83991c63ea800dd44843e42a916aec5899"}, + {file = "typed_ast-1.4.3-cp37-cp37m-win32.whl", hash = "sha256:aee0c1256be6c07bd3e1263ff920c325b59849dc95392a05f258bb9b259cf39c"}, + {file = "typed_ast-1.4.3-cp37-cp37m-win_amd64.whl", hash = "sha256:9ad2c92ec681e02baf81fdfa056fe0d818645efa9af1f1cd5fd6f1bd2bdfd805"}, + {file = "typed_ast-1.4.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:b36b4f3920103a25e1d5d024d155c504080959582b928e91cb608a65c3a49e1a"}, + {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:067a74454df670dcaa4e59349a2e5c81e567d8d65458d480a5b3dfecec08c5ff"}, + {file = "typed_ast-1.4.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:7538e495704e2ccda9b234b82423a4038f324f3a10c43bc088a1636180f11a41"}, + {file = "typed_ast-1.4.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:af3d4a73793725138d6b334d9d247ce7e5f084d96284ed23f22ee626a7b88e39"}, + {file = "typed_ast-1.4.3-cp38-cp38-win32.whl", hash = "sha256:f2362f3cb0f3172c42938946dbc5b7843c2a28aec307c49100c8b38764eb6927"}, + {file = "typed_ast-1.4.3-cp38-cp38-win_amd64.whl", hash = "sha256:dd4a21253f42b8d2b48410cb31fe501d32f8b9fbeb1f55063ad102fe9c425e40"}, + {file = "typed_ast-1.4.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f328adcfebed9f11301eaedfa48e15bdece9b519fb27e6a8c01aa52a17ec31b3"}, + {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:2c726c276d09fc5c414693a2de063f521052d9ea7c240ce553316f70656c84d4"}, + {file = "typed_ast-1.4.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:cae53c389825d3b46fb37538441f75d6aecc4174f615d048321b716df2757fb0"}, + {file = "typed_ast-1.4.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:b9574c6f03f685070d859e75c7f9eeca02d6933273b5e69572e5ff9d5e3931c3"}, + {file = "typed_ast-1.4.3-cp39-cp39-win32.whl", hash = "sha256:209596a4ec71d990d71d5e0d312ac935d86930e6eecff6ccc7007fe54d703808"}, + {file = "typed_ast-1.4.3-cp39-cp39-win_amd64.whl", hash = "sha256:9c6d1a54552b5330bc657b7ef0eae25d00ba7ffe85d9ea8ae6540d2197a3788c"}, + {file = "typed_ast-1.4.3.tar.gz", hash = "sha256:fb1bbeac803adea29cedd70781399c99138358c26d05fcbd23c13016b7f5ec65"}, +] +typing-extensions = [ + {file = "typing_extensions-3.10.0.2-py2-none-any.whl", hash = "sha256:d8226d10bc02a29bcc81df19a26e56a9647f8b0a6d4a83924139f4a8b01f17b7"}, + {file = "typing_extensions-3.10.0.2-py3-none-any.whl", hash = "sha256:f1d25edafde516b146ecd0613dabcc61409817af4766fbbcfb8d1ad4ec441a34"}, + {file = "typing_extensions-3.10.0.2.tar.gz", hash = "sha256:49f75d16ff11f1cd258e1b988ccff82a3ca5570217d7ad8c5f48205dd99a677e"}, +] +virtualenv = [ + {file = "virtualenv-20.8.1-py2.py3-none-any.whl", hash = "sha256:10062e34c204b5e4ec5f62e6ef2473f8ba76513a9a617e873f1f8fb4a519d300"}, + {file = "virtualenv-20.8.1.tar.gz", hash = "sha256:bcc17f0b3a29670dd777d6f0755a4c04f28815395bca279cdcb213b97199a6b8"}, +] +wcwidth = [ + {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, + {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, +] +wrapt = [ + {file = "wrapt-1.12.1.tar.gz", hash = "sha256:b62ffa81fb85f4332a4f609cab4ac40709470da05643a082ec1eb88e6d9b97d7"}, +] +zipp = [ + {file = "zipp-3.6.0-py3-none-any.whl", hash = "sha256:9fe5ea21568a0a70e50f273397638d39b03353731e6cbbb3fd8502a33fec40bc"}, + {file = "zipp-3.6.0.tar.gz", hash = "sha256:71c644c5369f4a6e07636f0aa966270449561fcea2e3d6747b8d23efaa9d7832"}, +] From 97b47cb193c6aabda9b3901711e88ecd3df58b0c Mon Sep 17 00:00:00 2001 From: otherJL0 Date: Mon, 4 Oct 2021 11:38:08 -0400 Subject: [PATCH 3/4] Adding pre-commit yaml --- .pre-commit-config.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..da84f9f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,28 @@ +repos: + - repo: local + hooks: + - id: system + name: isort + stages: [commit] + entry: poetry run isort preql/ + types: [python] + pass_filenames: false + language: system + - repo: local + hooks: + - id: system + name: black + stages: [commit] + entry: poetry run black preql/ + types: [python] + pass_filenames: false + language: system + - repo: local + hooks: + - id: system + name: pylint + stages: [commit] + entry: poetry run pylint preql/ + types: [python] + pass_filenames: false + language: system From 958a8dfd3a040f9c40fa394a8bfc3295f32a3019 Mon Sep 17 00:00:00 2001 From: otherJL0 Date: Mon, 4 Oct 2021 11:46:26 -0400 Subject: [PATCH 4/4] Adding tests to pre-commit --- .gitignore | 1 + .pre-commit-config.yaml | 13 +- preql/__init__.py | 1 - preql/__main__.py | 70 +++-- preql/_base_imports.py | 2 +- preql/api.py | 51 ++-- preql/context.py | 6 +- preql/core/autocomplete.py | 68 +++-- preql/core/base.py | 4 +- preql/core/casts.py | 81 ++++-- preql/core/compile_binops.py | 119 +++++++-- preql/core/compiler.py | 286 +++++++++++++------- preql/core/display.py | 76 ++++-- preql/core/evaluate.py | 440 ++++++++++++++++++++++--------- preql/core/exceptions.py | 15 +- preql/core/interp_common.py | 32 ++- preql/core/interpreter.py | 45 ++-- preql/core/parser.py | 57 ++-- preql/core/pql_ast.py | 88 +++++-- preql/core/pql_functions.py | 449 +++++++++++++++++++++----------- preql/core/pql_objects.py | 135 ++++++---- preql/core/pql_types.py | 90 ++++--- preql/core/sql.py | 315 +++++++++++++++------- preql/core/sql_import_result.py | 91 +++++-- preql/core/state.py | 51 ++-- preql/core/types_impl.py | 29 ++- preql/docstring/autodoc.py | 54 +++- preql/docstring/docstring.py | 39 +-- preql/docstring/type_docs.py | 86 +++--- preql/jup_kernel/__main__.py | 1 + preql/jup_kernel/install.py | 30 ++- preql/jup_kernel/kernel.py | 74 +++--- preql/loggers.py | 15 +- preql/repl.py | 111 ++++---- preql/settings.py | 27 +- preql/sql_interface.py | 267 +++++++++++-------- preql/utils.py | 67 +++-- 37 files changed, 2237 insertions(+), 1149 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index da84f9f..c02bb2f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,7 +5,6 @@ repos: name: isort stages: [commit] entry: poetry run isort preql/ - types: [python] pass_filenames: false language: system - repo: local @@ -14,15 +13,21 @@ repos: name: black stages: [commit] entry: poetry run black preql/ - types: [python] pass_filenames: false language: system - repo: local hooks: - id: system name: pylint - stages: [commit] + stages: [manual] entry: poetry run pylint preql/ - types: [python] + pass_filenames: false + language: system + - repo: local + hooks: + - id: system + name: test + stages: [push] + entry: poetry run python -m tests minimal pass_filenames: false language: system diff --git a/preql/__init__.py b/preql/__init__.py index e9c83eb..8319527 100644 --- a/preql/__init__.py +++ b/preql/__init__.py @@ -1,5 +1,4 @@ from . import _base_imports - from .api import Preql, T from .core.exceptions import Signal diff --git a/preql/__main__.py b/preql/__main__.py index c464501..4793915 100644 --- a/preql/__main__.py +++ b/preql/__main__.py @@ -1,26 +1,51 @@ -import json import argparse -from pathlib import Path -from itertools import chain +import json import time +from itertools import chain +from pathlib import Path -from . import Preql, __version__, Signal -from . import settings +from . import Preql, Signal, __version__, settings parser = argparse.ArgumentParser(description='Preql command-line interface (aka REPL)') -parser.add_argument('-i', '--interactive', action='store_true', default=False, - help="enter interactive mode after running the script") +parser.add_argument( + '-i', + '--interactive', + action='store_true', + default=False, + help="enter interactive mode after running the script", +) parser.add_argument('-v', '--version', action='store_true', help="print version") -parser.add_argument('--install-jupyter', action='store_true', help="installs the Preql plugin for Jupyter notebook") -parser.add_argument('--print-sql', action='store_true', help="print the SQL code that's being executed") +parser.add_argument( + '--install-jupyter', + action='store_true', + help="installs the Preql plugin for Jupyter notebook", +) +parser.add_argument( + '--print-sql', action='store_true', help="print the SQL code that's being executed" +) parser.add_argument('-f', '--file', type=str, help='path to a Preql script to run') parser.add_argument('-m', '--module', type=str, help='name of a Preql module to run') -parser.add_argument('--time', action='store_true', help='displays how long the script ran') -parser.add_argument('-c', '--config', type=str, help='path to a JSON configuration file for Preql (default: ~/.preql_conf.json)') -parser.add_argument('database', type=str, nargs='?', default=None, - help="database url (postgres://user:password@host:port/db_name") -parser.add_argument('--python-traceback', action='store_true', - help="Show the Python traceback when an exception causes the interpreter to quit") +parser.add_argument( + '--time', action='store_true', help='displays how long the script ran' +) +parser.add_argument( + '-c', + '--config', + type=str, + help='path to a JSON configuration file for Preql (default: ~/.preql_conf.json)', +) +parser.add_argument( + 'database', + type=str, + nargs='?', + default=None, + help="database url (postgres://user:password@host:port/db_name", +) +parser.add_argument( + '--python-traceback', + action='store_true', + help="Show the Python traceback when an exception causes the interpreter to quit", +) def find_dot_preql(): @@ -30,6 +55,7 @@ def find_dot_preql(): if dot_preql.exists(): return dot_preql + def update_settings(path): config = json.load(path.open()) if 'debug' in config: @@ -37,6 +63,7 @@ def update_settings(path): if 'color_scheme' in config: settings.color_theme.update(config['color_scheme']) + def main(): args = parser.parse_args() @@ -45,11 +72,15 @@ def main(): if args.install_jupyter: from .jup_kernel.install import main as install_jupyter + install_jupyter([]) - print("Install successful. To start working, run 'jupyter notebook' and create a new Preql notebook.") + print( + "Install successful. To start working, run 'jupyter notebook' and create a new Preql notebook." + ) return from pathlib import Path + if args.config: update_settings(Path(args.config)) else: @@ -57,8 +88,6 @@ def main(): if config_path.exists(): update_settings(config_path) - - kw = {'print_sql': args.print_sql} if args.database: kw['db_uri'] = args.database @@ -93,7 +122,7 @@ def main(): end = time.time() if args.time: - print('Script took %.2f seconds to run' % (end -start)) + print('Script took %.2f seconds to run' % (end - start)) if interactive: p.load_all_tables() @@ -101,5 +130,6 @@ def main(): else: return error_code + if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/preql/_base_imports.py b/preql/_base_imports.py index f33449b..786081e 100644 --- a/preql/_base_imports.py +++ b/preql/_base_imports.py @@ -1 +1 @@ -from .core import interp_common, pql_types, pql_objects, types_impl \ No newline at end of file +from .core import interp_common, pql_objects, pql_types, types_impl diff --git a/preql/api.py b/preql/api.py index 5686140..c56d347 100644 --- a/preql/api.py +++ b/preql/api.py @@ -2,19 +2,18 @@ from functools import wraps from . import settings -from .core.pql_ast import pyvalue, Ast +from .core import display from .core import pql_objects as objects +from .core.exceptions import Signal from .core.interpreter import Interpreter +from .core.pql_ast import Ast, pyvalue from .core.pql_types import T from .sql_interface import create_engine from .utils import dsp -from .core.exceptions import Signal -from .core import display display.install_reprs() - def clean_signal(f): @wraps(f) def inner(*args, **kwargs): @@ -25,6 +24,7 @@ def inner(*args, **kwargs): return f(*args, **kwargs) except Signal as e: raise e.clean_copy() from None # Error from Preql + return inner @@ -53,6 +53,7 @@ def to_json(self): def to_pandas(self): "Returns table as a Pandas dataframe (requires pandas installed)" from pandas import DataFrame + return DataFrame(self) def __eq__(self, other): @@ -75,11 +76,13 @@ def __getitem__(self, index): if isinstance(index, slice): offset = index.start or 0 limit = (index.stop or len(self)) - offset - new_inst = self._interp.call_builtin_func('limit_offset', [self._inst, pyvalue(limit), pyvalue(offset)]) + new_inst = self._interp.call_builtin_func( + 'limit_offset', [self._inst, pyvalue(limit), pyvalue(offset)] + ) return TablePromise(self._interp, new_inst) # TODO different debug log level / mode - res ,= self._interp.cast_to_python(self[index:index+1]._inst) + (res,) = self._interp.cast_to_python(self[index : index + 1]._inst) return res def __repr__(self): @@ -87,7 +90,6 @@ def __repr__(self): return display.print_to_string(display.table_repr(self._inst), 'text') - @dsp def from_python(value: TablePromise): return value._inst @@ -101,7 +103,6 @@ def _prepare_instance_for_user(interp, inst): class _Delegate: - def __init__(self, pql, fname): self.fname = fname self.pql = pql @@ -109,9 +110,9 @@ def __init__(self, pql, fname): @clean_signal def __call__(self, *args, **kw): pql_args = [objects.from_python(a) for a in args] - pql_kwargs = {k:objects.from_python(v) for k,v in kw.items()} + pql_kwargs = {k: objects.from_python(v) for k, v in kw.items()} pql_res = self.pql._interp.call_func(self.fname, pql_args, pql_kwargs) - return self.pql._wrap_result( pql_res ) + return self.pql._wrap_result(pql_res) class Preql: @@ -126,7 +127,12 @@ class Preql: __name__ = "Preql" - def __init__(self, db_uri: str='sqlite://:memory:', print_sql: bool=settings.print_sql, auto_create: bool = False): + def __init__( + self, + db_uri: str = 'sqlite://:memory:', + print_sql: bool = settings.print_sql, + auto_create: bool = False, + ): """Initialize a new Preql instance Parameters: @@ -139,7 +145,9 @@ def __init__(self, db_uri: str='sqlite://:memory:', print_sql: bool=settings.pri self._display = display.RichDisplay() # self.engine.ping() - engine = create_engine(self._db_uri, print_sql=self._print_sql, auto_create=auto_create) + engine = create_engine( + self._db_uri, print_sql=self._print_sql, auto_create=auto_create + ) self._reset_interpreter(engine) def __repr__(self): @@ -156,12 +164,11 @@ def set_output_format(self, fmt): self._interp.state.state.display = self._display # TODO proper api - def _reset_interpreter(self, engine=None): if engine is None: engine = self._interp.state.db self._interp = Interpreter(engine, self._display) - self._interp._py_api = self # TODO proper api + self._interp._py_api = self # TODO proper api def close(self): self._interp.state.db.close() @@ -179,7 +186,7 @@ def __getattr__(self, fname): # return self._wrap_result( pql_res ) # return delegate else: - obj = self._interp.evaluate_obj( var ) + obj = self._interp.evaluate_obj(var) return self._wrap_result(obj) def __setattr__(self, name, value): @@ -192,11 +199,15 @@ def __setattr__(self, name, value): def _wrap_result(self, res): "Wraps Preql result in a Python-friendly object" if isinstance(res, Ast): - raise TypeError("Returned object cannot be converted into a Python representation") + raise TypeError( + "Returned object cannot be converted into a Python representation" + ) return _prepare_instance_for_user(self._interp, res) # TODO session, not state def _run_code(self, code, source_name='', args=None): - pql_args = {name: objects.from_python(value) for name, value in (args or {}).items()} + pql_args = { + name: objects.from_python(value) for name, value in (args or {}).items() + } return self._interp.execute_code(code + "\n", source_name, pql_args) @clean_signal @@ -228,6 +239,7 @@ def transaction(self): def start_repl(self, *args): "Run the interactive prompt" from .repl import start_repl + start_repl(self, *args) def commit(self): @@ -236,7 +248,6 @@ def commit(self): def rollback(self): return self._interp.state.db.rollback() - def import_pandas(self, **dfs): """Import pandas.DataFrame instances into SQL tables @@ -245,16 +256,12 @@ def import_pandas(self, **dfs): """ return self._interp.import_pandas(dfs) - def load_all_tables(self): return self._interp.load_all_tables() @property def interp(self): raise Exception("Reserved") - - - # def _functions(self): diff --git a/preql/context.py b/preql/context.py index 939b887..584125c 100644 --- a/preql/context.py +++ b/preql/context.py @@ -1,6 +1,7 @@ import threading from contextlib import contextmanager + class Context(threading.local): def __init__(self): self._ctx = [{}] @@ -28,9 +29,9 @@ def __call__(self, **attrs): assert attrs is _d - def test_threading(): - import time, random + import random + import time context = Context() @@ -44,7 +45,6 @@ def g(i): assert context.i == i print(i, end=', ') - for i in range(100): t = threading.Thread(target=f, args=(i,)) t.start() diff --git a/preql/core/autocomplete.py b/preql/core/autocomplete.py index 62abafa..02a7d29 100644 --- a/preql/core/autocomplete.py +++ b/preql/core/autocomplete.py @@ -1,17 +1,18 @@ -from lark import Token, UnexpectedCharacters, UnexpectedToken, ParseError +from lark import ParseError, Token, UnexpectedCharacters, UnexpectedToken +from preql.context import context from preql.loggers import ac_log from preql.utils import bfs_all_unique, dsp -from preql.context import context -from .exceptions import Signal, ReturnSignal, pql_SyntaxError -from .compiler import AutocompleteSuggestions -from .evaluate import evaluate, resolve -from .state import ThreadState, set_var, use_scope +from . import parser from . import pql_ast as ast from . import pql_objects as objects +from . import sql +from .compiler import AutocompleteSuggestions +from .evaluate import evaluate, resolve +from .exceptions import ReturnSignal, Signal, pql_SyntaxError from .pql_types import T -from . import sql, parser +from .state import ThreadState, set_var, use_scope @dsp @@ -19,49 +20,65 @@ def eval_autocomplete(x, go_inside): _res = evaluate(x) # assert isinstance(res, objects.Instance) + @dsp def eval_autocomplete(cb: ast.Statement, go_inside): raise NotImplementedError(cb) + @dsp def eval_autocomplete(t: ast.Try, go_inside): eval_autocomplete(t.try_, go_inside) catch_type = evaluate(t.catch_expr) - scope = {t.catch_name: Signal(catch_type, [], '')} if t.catch_name else {} + scope = ( + {t.catch_name: Signal(catch_type, [], '')} + if t.catch_name + else {} + ) with use_scope(scope): eval_autocomplete(t.catch_block, go_inside) + @dsp def eval_autocomplete(a: ast.InsertRows, go_inside): eval_autocomplete(a.value, go_inside) + + @dsp def eval_autocomplete(a: ast.Assert, go_inside): eval_autocomplete(a.cond, go_inside) + + @dsp def eval_autocomplete(a: ast.Print, go_inside): eval_autocomplete(a.value, go_inside) + @dsp def eval_autocomplete(x: ast.If, go_inside): eval_autocomplete(x.then, go_inside) if x.else_: eval_autocomplete(x.else_, go_inside) + @dsp def eval_autocomplete(x: ast.For, go_inside): with use_scope({x.var: None}): eval_autocomplete(x.do, go_inside) + @dsp def eval_autocomplete(x: ast.While, go_inside): eval_autocomplete(x.do, go_inside) + @dsp def eval_autocomplete(x: ast.SetValue, go_inside): - value = evaluate( x.value) + value = evaluate(x.value) if isinstance(x.name, ast.Name): set_var(x.name.name, value) + @dsp def eval_autocomplete(cb: ast.CodeBlock, go_inside): for s in cb.statements[:-1]: @@ -70,22 +87,26 @@ def eval_autocomplete(cb: ast.CodeBlock, go_inside): for s in cb.statements[-1:]: eval_autocomplete(s, go_inside) + @dsp def eval_autocomplete(td: ast.TableDefFromExpr, go_inside): expr = evaluate(td.expr) set_var(td.name, expr) + @dsp def eval_autocomplete(td: ast.TableDef, go_inside): t = resolve(td) - n ,= t.options['name'].parts + (n,) = t.options['name'].parts set_var(n, objects.TableInstance.make(sql.unknown, t, [])) + @dsp def eval_autocomplete(td: ast.StructDef, go_inside): t = resolve(td) set_var(t.name, t) + @dsp def eval_autocomplete(fd: ast.FuncDef, go_inside): f = fd.userfunc @@ -93,7 +114,7 @@ def eval_autocomplete(fd: ast.FuncDef, go_inside): try: if go_inside: - with use_scope({p.name:objects.unknown for p in f.params}): + with use_scope({p.name: objects.unknown for p in f.params}): try: eval_autocomplete(f.expr, go_inside) except ReturnSignal: @@ -103,14 +124,14 @@ def eval_autocomplete(fd: ast.FuncDef, go_inside): cb = ast.CodeBlock([ast.Return(objects.unknown)]) set_var(f.name, f.replace(expr=cb)) + @dsp def eval_autocomplete(r: ast.Return, go_inside): # Same as _execute - value = evaluate( r.value) + value = evaluate(r.value) raise ReturnSignal(value) - _closing_tokens = { 'RSQB': ']', 'RBRACE': '}', @@ -119,6 +140,7 @@ def eval_autocomplete(r: ast.Return, go_inside): '_NL': '\n', } + def _search_parser(parser): def expand(p): for choice in p.choices(): @@ -126,7 +148,7 @@ def expand(p): t = Token(choice, _closing_tokens[choice], 1, 1, 1, 1, 2, 2) try: new_p = p.feed_token(t) - except ParseError: # Illegal + except ParseError: # Illegal pass else: yield new_p @@ -135,6 +157,7 @@ def expand(p): if p.result: return p.result + def autocomplete_tree(parser): if not parser: return @@ -147,7 +170,7 @@ def autocomplete_tree(parser): t = Token('MARKER', '', 1, 1, 1, 1, 2, 2) try: res = parser.feed_token(t) - except ParseError: # Could still fail + except ParseError: # Could still fail return assert not res, res @@ -157,7 +180,8 @@ def autocomplete_tree(parser): KEYWORDS = 'table update delete new func try if else for throw catch print assert const in or and not one null false true return !in'.split() -KEYWORDS = {k:(100000, None) for k in KEYWORDS} +KEYWORDS = {k: (100000, None) for k in KEYWORDS} + class AcState(ThreadState): def get_var(self, name): @@ -173,14 +197,17 @@ def get_var(self, name): # return all_vars def get_all_vars_with_rank(self): - all_vars = {k:(10000, v) for k, v in self.get_var('__builtins__').namespace.items()} - all_vars.update( self.ns.get_all_vars_with_rank() ) + all_vars = { + k: (10000, v) for k, v in self.get_var('__builtins__').namespace.items() + } + all_vars.update(self.ns.get_all_vars_with_rank()) all_vars.update(KEYWORDS) return all_vars def replace(self, **kw): assert False + def _eval_autocomplete(ac_state, stmts): for stmt in stmts: try: @@ -188,6 +215,7 @@ def _eval_autocomplete(ac_state, stmts): except Signal as e: ac_log.exception(e) + def autocomplete(state, code, source=''): ac_state = AcState.clone(state) with context(state=ac_state): @@ -208,7 +236,7 @@ def autocomplete(state, code, source=''): try: eval_autocomplete(stmts[-1], True) except AutocompleteSuggestions as e: - ns ,= e.args + (ns,) = e.args return ns except Signal as e: ac_log.exception(e) @@ -216,4 +244,4 @@ def autocomplete(state, code, source=''): else: _eval_autocomplete(ac_state, stmts) - return ac_state.get_all_vars_with_rank() \ No newline at end of file + return ac_state.get_all_vars_with_rank() diff --git a/preql/core/base.py b/preql/core/base.py index 8c89ba4..627ac87 100644 --- a/preql/core/base.py +++ b/preql/core/base.py @@ -1,15 +1,13 @@ - class Object: def repr(self): return repr(self) def inline_repr(self): - return self.repr() + return self.repr() def rich_repr(self): return self.repr().replace('[', '\\[') - def all_attrs(self): return {} diff --git a/preql/core/casts.py b/preql/core/casts.py index 359f88f..f71f902 100644 --- a/preql/core/casts.py +++ b/preql/core/casts.py @@ -1,15 +1,19 @@ from . import pql_objects as objects from . import sql -from .pql_types import T, dp_type, ITEM_NAME -from .types_impl import kernel_type from .exceptions import Signal from .interp_common import call_builtin_func +from .pql_types import ITEM_NAME, T, dp_type +from .types_impl import kernel_type + @dp_type def _cast(inst_type, target_type, inst): if inst_type <= target_type: return inst - raise Signal.make(T.TypeError, None, f"Cast not implemented for {inst_type}->{target_type}") + raise Signal.make( + T.TypeError, None, f"Cast not implemented for {inst_type}->{target_type}" + ) + @dp_type def _cast(inst_type: T.list, target_type: T.list, inst): @@ -28,74 +32,113 @@ def _cast(inst_type: T.list, target_type: T.list, inst): @dp_type def _cast(inst_type: T.aggregated, target_type: T.list, inst): res = _cast(inst_type.elem, target_type.elem, inst) - return objects.aggregate(res) # ?? + return objects.aggregate(res) # ?? + @dp_type def _cast(inst_type: T.table, target_type: T.list, inst): t = inst.type if len(t.elems) != 1: - raise Signal.make(T.TypeError, None, f"Cannot cast {inst_type} to {target_type}. Too many columns") + raise Signal.make( + T.TypeError, + None, + f"Cannot cast {inst_type} to {target_type}. Too many columns", + ) if not inst_type.elem <= target_type.elem: - raise Signal.make(T.TypeError, None, f"Cannot cast {inst_type} to {target_type}. Elements not matching") - - (elem_name, elem_type) ,= inst_type.elems.items() - code = sql.Select(T.list[elem_type], inst.code, [sql.ColumnAlias(sql.Name(elem_type, elem_name), ITEM_NAME)]) + raise Signal.make( + T.TypeError, + None, + f"Cannot cast {inst_type} to {target_type}. Elements not matching", + ) + + ((elem_name, elem_type),) = inst_type.elems.items() + code = sql.Select( + T.list[elem_type], + inst.code, + [sql.ColumnAlias(sql.Name(elem_type, elem_name), ITEM_NAME)], + ) return objects.TableInstance.make(code, T.list[elem_type], [inst]) + @dp_type def _cast(inst_type: T.table, target_type: T.primitive, inst): t = inst.type if len(t.elems) != 1: - raise Signal.make(T.TypeError, None, f"Cannot cast {inst_type} to {target_type}. Expected exactly 1 column, instead got {len(t.elems)}") + raise Signal.make( + T.TypeError, + None, + f"Cannot cast {inst_type} to {target_type}. Expected exactly 1 column, instead got {len(t.elems)}", + ) if not inst_type.elem <= target_type: - raise Signal.make(T.TypeError, None, f"Cannot cast {inst_type} to {target_type}. Elements type doesn't match") + raise Signal.make( + T.TypeError, + None, + f"Cannot cast {inst_type} to {target_type}. Elements type doesn't match", + ) res = inst.localize() if len(res) != 1: - raise Signal.make(T.TypeError, None, f"Cannot cast {inst_type} to {target_type}. Expected exactly 1 row, instead got {len(res)}") - item ,= res + raise Signal.make( + T.TypeError, + None, + f"Cannot cast {inst_type} to {target_type}. Expected exactly 1 row, instead got {len(res)}", + ) + (item,) = res return objects.pyvalue_inst(item, inst_type.elem) - + @dp_type def _cast(_inst_type: T.t_id, _target_type: T.int, inst): return inst.replace(type=T.int) + @dp_type def _cast(_inst_type: T.int, target_type: T.t_id, inst): return inst.replace(type=target_type) + @dp_type def _cast(_inst_type: T.union[T.float, T.bool], _target_type: T.int, inst): code = sql.Cast(T.int, inst.code) return objects.Instance.make(code, T.int, [inst]) + @dp_type def _cast(_inst_type: T.number, _target_type: T.bool, inst): code = sql.Compare('!=', [inst.code, sql.make_value(0)]) return objects.Instance.make(code, T.bool, [inst]) + @dp_type def _cast(_inst_type: T.string, _target_type: T.bool, inst): code = sql.Compare('!=', [inst.code, sql.make_value('')]) return objects.Instance.make(code, T.bool, [inst]) + @dp_type def _cast(_inst_type: T.string, _target_type: T.text, inst): return inst.replace(type=T.text) + + @dp_type def _cast(_inst_type: T.text, _target_type: T.string, inst): return inst.replace(type=T.string) + + @dp_type -def _cast(_inst_type: T.string, _target_type: T.string, inst): # Disambiguate text<->string due to inheritance +def _cast( + _inst_type: T.string, _target_type: T.string, inst +): # Disambiguate text<->string due to inheritance return inst + @dp_type def _cast(_inst_type: T.union[T.int, T.bool], _target_type: T.float, inst): code = sql.Cast(T.float, inst.code) return objects.Instance.make(code, T.float, [inst]) + @dp_type def _cast(_inst_type: T.string, _target_type: T.int, inst): return call_builtin_func("_cast_string_to_int", [inst]) @@ -106,21 +149,27 @@ def _cast(_inst_type: T.string, _target_type: T.int, inst): # # XXX unsafe cast, bad strings won't throw an error # return objects.Instance.make(inst.code, T.datetime, [inst]) + @dp_type def _cast(_inst_type: T.primitive, _target_type: T.string, inst): code = sql.Cast(T.string, inst.code) return objects.Instance.make(code, T.string, [inst]) + @dp_type def _cast(_inst_type: T.t_relation, target_type: T.t_id, inst): # TODO verify same table? same type? return inst.replace(type=target_type) + @dp_type def _cast(inst_type: T.t_relation, target_type: T.int, inst): if inst.type.elem <= T.int: return inst.replace(type=target_type) - raise Signal.make(T.TypeError, None, f"Cast not implemented for {inst_type}->{target_type}") + raise Signal.make( + T.TypeError, None, f"Cast not implemented for {inst_type}->{target_type}" + ) + def cast(obj, t): res = _cast(kernel_type(obj.type), t, obj) diff --git a/preql/core/compile_binops.py b/preql/core/compile_binops.py index f540b88..64b8c49 100644 --- a/preql/core/compile_binops.py +++ b/preql/core/compile_binops.py @@ -1,33 +1,41 @@ import operator + from preql import settings -from .pql_types import T, dp_inst -from .interp_common import pyvalue_inst, call_builtin_func -from . import sql from . import pql_objects as objects +from . import sql +from .casts import cast from .exceptions import Signal +from .interp_common import call_builtin_func, pyvalue_inst from .pql_objects import make_instance, remove_phantom_type +from .pql_types import T, dp_inst from .state import get_var -from .casts import cast + ## Compare def compare(op, a, b): res = _compare(op, remove_phantom_type(a), remove_phantom_type(b)) return objects.inherit_phantom_type(res, [a, b]) + @dp_inst def _compare(op, a: T.any, b: T.any): - raise Signal.make(T.TypeError, op, f"Compare not implemented for {a.type} and {b.type}") + raise Signal.make( + T.TypeError, op, f"Compare not implemented for {a.type} and {b.type}" + ) @dp_inst def _compare(op, _a: T.nulltype, _b: T.nulltype): return pyvalue_inst(op in ('=', '<=', '>=')) + @dp_inst def _compare(_op, a: T.type, _b: T.nulltype): assert not a.type.maybe_null() return pyvalue_inst(False) + + @dp_inst def _compare(op, a: T.nulltype, b: T.type): return _compare(op, b, a) @@ -35,6 +43,7 @@ def _compare(op, a: T.nulltype, b: T.type): primitive_or_struct = T.union[T.primitive, T.struct] + @dp_inst def _compare(op, a: T.nulltype, b: primitive_or_struct): # TODO Enable this type-based optimization: @@ -44,6 +53,8 @@ def _compare(op, a: T.nulltype, b: primitive_or_struct): b = b.primary_key() code = sql.Compare(op, [a.code, b.code]) return objects.Instance.make(code, T.bool, [a, b]) + + @dp_inst def _compare(op, a: primitive_or_struct, b: T.nulltype): return _compare(op, b, a) @@ -52,9 +63,13 @@ def _compare(op, a: primitive_or_struct, b: T.nulltype): @dp_inst def _compare(_op, _a: T.unknown, _b: T.object): return objects.UnknownInstance() + + @dp_inst def _compare(_op, _a: T.object, _b: T.unknown): return objects.UnknownInstance() + + @dp_inst def _compare(_op, _a: T.unknown, _b: T.unknown): return objects.UnknownInstance() @@ -66,32 +81,49 @@ def _prepare_to_compare(op, a, b): return pyvalue_inst(False) elif op == '!=': return pyvalue_inst(True) - raise Signal.make(T.TypeError, op, f"Operator '{op}' not implemented for {a.type} and {b.type}") + raise Signal.make( + T.TypeError, op, f"Operator '{op}' not implemented for {a.type} and {b.type}" + ) + @dp_inst def _prepare_to_compare(op, a: T.number | T.bool, b: T.number | T.bool): pass + + @dp_inst def _prepare_to_compare(op, a: T.string, b: T.string): pass + # XXX id/relation can be either int or string, so we can't tell if comparison is necessary or not -# So we always allow the comparison +# So we always allow the comparison # TODO use generics/phantoms to disambiguate the situation id_or_relation = T.t_relation | T.t_id + + @dp_inst def _prepare_to_compare(op, a: id_or_relation, b): pass + + @dp_inst def _prepare_to_compare(op, a, b: id_or_relation): pass + + @dp_inst def _prepare_to_compare(op, a: id_or_relation, b: id_or_relation): pass + @dp_inst def _compare(op, a: T.primitive, b: T.primitive): - if settings.optimize and isinstance(a, objects.ValueInstance) and isinstance(b, objects.ValueInstance): + if ( + settings.optimize + and isinstance(a, objects.ValueInstance) + and isinstance(b, objects.ValueInstance) + ): f = { '=': operator.eq, '!=': operator.ne, @@ -104,7 +136,11 @@ def _compare(op, a: T.primitive, b: T.primitive): try: return pyvalue_inst(f(a.local_value, b.local_value)) except TypeError as e: - raise Signal.make(T.TypeError, op, f"Operator '{op}' not implemented for {a.type} and {b.type}") + raise Signal.make( + T.TypeError, + op, + f"Operator '{op}' not implemented for {a.type} and {b.type}", + ) # TODO regular equality for primitives? (not 'is') res = _prepare_to_compare(op, a, b) @@ -120,23 +156,27 @@ def _compare(op, a: T.type, b: T.type): if op == '<=': return call_builtin_func("issubclass", [a, b]) if op != '=': - raise Signal.make(T.NotImplementedError, op, f"Cannot compare types using: {op}") + raise Signal.make( + T.NotImplementedError, op, f"Cannot compare types using: {op}" + ) return pyvalue_inst(a == b) + @dp_inst def _compare(op, a: T.primitive, b: T.row): return _compare(op, a, b.primary_key()) + @dp_inst def _compare(op, a: T.row, b: T.primitive): return _compare(op, b, a) + @dp_inst def _compare(op, a: T.row, b: T.row): return _compare(op, a.primary_key(), b.primary_key()) - ## Contains def contains(op, a, b): res = _contains(op, remove_phantom_type(a), remove_phantom_type(b)) @@ -151,6 +191,7 @@ def _contains(op, a: T.string, b: T.string): }[op] return call_builtin_func(f, [a, b]) + @dp_inst def _contains(op, a: T.primitive, b: T.table): b_list = cast(b, T.list) @@ -163,22 +204,29 @@ def _contains(op, a: T.primitive, b: T.table): code = sql.Contains(op, [a.code, b_list.code]) return objects.Instance.make(code, T.bool, [a, b_list]) + @dp_inst def _contains(op, a: T.any, b: T.any): - raise Signal.make(T.TypeError, op, f"Contains not implemented for {a.type} and {b.type}") - + raise Signal.make( + T.TypeError, op, f"Contains not implemented for {a.type} and {b.type}" + ) ## Arith + def compile_arith(op, a, b): res = _compile_arith(op, remove_phantom_type(a), remove_phantom_type(b)) return objects.inherit_phantom_type(res, [a, b]) + @dp_inst def _compile_arith(arith, a: T.any, b: T.any): - raise Signal.make(T.TypeError, arith.op, f"Operator '{arith.op}' not implemented for {a.type} and {b.type}") - + raise Signal.make( + T.TypeError, + arith.op, + f"Operator '{arith.op}' not implemented for {a.type} and {b.type}", + ) @dp_inst @@ -194,16 +242,23 @@ def _compile_arith(arith, a: T.table, b: T.table): try: op = ops[arith.op] except KeyError: - raise Signal.make(T.TypeError, arith.op, f"Operation '{arith.op}' not supported for tables ({a.type}, {b.type})") + raise Signal.make( + T.TypeError, + arith.op, + f"Operation '{arith.op}' not supported for tables ({a.type}, {b.type})", + ) return get_var(op).func(a, b) - @dp_inst def _compile_arith(arith, a: T.string, b: T.int): if arith.op != '*': - raise Signal.make(T.TypeError, arith.op, f"Operator '{arith.op}' not supported between string and integer.") + raise Signal.make( + T.TypeError, + arith.op, + f"Operator '{arith.op}' not supported between string and integer.", + ) return call_builtin_func("repeat", [a, b]) @@ -225,9 +280,17 @@ def _compile_arith(arith, a: T.number, b: T.number): '**': operator.pow, }[arith.op] except KeyError: - raise Signal.make(T.TypeError, arith, f"Operator {arith.op} not supported between types '{a.type}' and '{b.type}'") - - if settings.optimize and isinstance(a, objects.ValueInstance) and isinstance(b, objects.ValueInstance): + raise Signal.make( + T.TypeError, + arith, + f"Operator {arith.op} not supported between types '{a.type}' and '{b.type}'", + ) + + if ( + settings.optimize + and isinstance(a, objects.ValueInstance) + and isinstance(b, objects.ValueInstance) + ): # Local folding for better performance. # However, acts a little different than SQL. For example, in this branch 1/0 raises ValueError, # while SQL returns NULL @@ -242,6 +305,7 @@ def _compile_arith(arith, a: T.number, b: T.number): code = sql.arith(res_type, arith.op, [a.code, b.code]) return make_instance(code, res_type, [a, b]) + @dp_inst def _compile_arith(arith, a: T.string, b: T.string): if arith.op == 'like': @@ -249,12 +313,17 @@ def _compile_arith(arith, a: T.string, b: T.string): return objects.Instance.make(code, T.bool, [a, b]) if arith.op != '+': - raise Signal.make(T.TypeError, arith.op, f"Operator '{arith.op}' not supported for strings.") - - if settings.optimize and isinstance(a, objects.ValueInstance) and isinstance(b, objects.ValueInstance): + raise Signal.make( + T.TypeError, arith.op, f"Operator '{arith.op}' not supported for strings." + ) + + if ( + settings.optimize + and isinstance(a, objects.ValueInstance) + and isinstance(b, objects.ValueInstance) + ): # Local folding for better performance (optional, for better performance) return pyvalue_inst(a.local_value + b.local_value, T.string) code = sql.arith(T.string, arith.op, [a.code, b.code]) return make_instance(code, T.string, [a, b]) - diff --git a/preql/core/compiler.py b/preql/core/compiler.py index 09854cf..39482af 100644 --- a/preql/core/compiler.py +++ b/preql/core/compiler.py @@ -1,26 +1,48 @@ - -from preql.utils import safezip, listgen, find_duplicate, SafeDict, re_split, method from preql.context import context +from preql.utils import SafeDict, find_duplicate, listgen, method, re_split, safezip -from .exceptions import Signal, InsufficientAccessLevel, ReturnSignal, pql_AttributeError -from . import pql_objects as objects from . import pql_ast as ast +from . import pql_objects as objects from . import sql -from .interp_common import dsp, assert_type, pyvalue_inst, evaluate, cast_to_python_string, cast_to_python_int -from .state import use_scope, get_var, get_db, unique_name, require_access, AccessLevels, get_access_level -from .pql_types import T, Type, Id, ITEM_NAME -from .types_impl import flatten_type, pql_repr, kernel_type from .casts import cast -from .pql_objects import AbsInstance, projected, make_instance -from .compile_binops import compile_arith, contains, compare +from .compile_binops import compare, compile_arith, contains +from .exceptions import ( + InsufficientAccessLevel, + ReturnSignal, + Signal, + pql_AttributeError, +) +from .interp_common import ( + assert_type, + cast_to_python_int, + cast_to_python_string, + dsp, + evaluate, + pyvalue_inst, +) +from .pql_objects import AbsInstance, make_instance, projected +from .pql_types import ITEM_NAME, Id, T, Type +from .state import ( + AccessLevels, + get_access_level, + get_db, + get_var, + require_access, + unique_name, + use_scope, +) +from .types_impl import flatten_type, kernel_type, pql_repr + class AutocompleteSuggestions(Exception): pass + @dsp def cast_to_instance(x: list): return [cast_to_instance(i) for i in x] + @dsp def cast_to_instance(x): try: @@ -36,12 +58,13 @@ def cast_to_instance(x): if not isinstance(inst, AbsInstance): # TODO compile error? cast error? # TODO need to be able to catch this above, and provide better errors - raise Signal.make(T.TypeError, None, f"Could not compile {pql_repr(inst.type, inst)}") + raise Signal.make( + T.TypeError, None, f"Could not compile {pql_repr(inst.type, inst)}" + ) return inst - @listgen def _process_fields(fields): for f in fields: @@ -52,7 +75,9 @@ def _process_fields(fields): v = cast_to_instance(f.value) except Signal as e: if e.type <= T.TypeError: - raise e.replace(message=f"Cannot use object of type '{evaluate( f.value).type}' in projection.") + raise e.replace( + message=f"Cannot use object of type '{evaluate( f.value).type}' in projection." + ) raise # In Preql, {=>v} creates an array. In SQL, it selects the first element. @@ -63,7 +88,7 @@ def _process_fields(fields): v = make_instance(sql.MakeArray(t, v.code), t, [v]) suggested_name = str(f.name) if f.name else guess_field_name(f.value) - name = suggested_name.rsplit('.', 1)[-1] # Use the last attribute as name + name = suggested_name.rsplit('.', 1)[-1] # Use the last attribute as name yield [(f.user_defined and bool(f.name), name), v] @@ -90,23 +115,35 @@ def _expand_ellipsis(obj, fields): raise Signal.make(T.SyntaxError, f, msg) t = obj.type - assert t <= T.table or t <= T.struct # some_table{ ... } or some_table{ some_struct_item {...} } + assert ( + t <= T.table or t <= T.struct + ) # some_table{ ... } or some_table{ some_struct_item {...} } for n in f.value.exclude: if isinstance(n, ast.Marker): - raise AutocompleteSuggestions({k:(0, v) for k, v in t.elems.items() - if k not in direct_names - and k not in f.value.exclude}) + raise AutocompleteSuggestions( + { + k: (0, v) + for k, v in t.elems.items() + if k not in direct_names and k not in f.value.exclude + } + ) if n in direct_names: - raise Signal.make(T.NameError, n, f"Field to exclude '{n}' is explicitely included in projection") + raise Signal.make( + T.NameError, + n, + f"Field to exclude '{n}' is explicitely included in projection", + ) if f.value.from_struct: # Inline struct with use_scope(obj.all_attrs()): - s = evaluate( f.value.from_struct) + s = evaluate(f.value.from_struct) if not s.type <= T.struct: - raise Signal.make(T.TypeError, s, f"Cannot inline objects of type {s.type}") + raise Signal.make( + T.TypeError, s, f"Cannot inline objects of type {s.type}" + ) items = s.attrs else: @@ -114,16 +151,22 @@ def _expand_ellipsis(obj, fields): items = obj.all_attrs() try: - remaining_items = list(_exclude_items(items, set(f.value.exclude), direct_names)) + remaining_items = list( + _exclude_items(items, set(f.value.exclude), direct_names) + ) except ValueError as e: fte = set(e.args[0]) - raise Signal.make(T.NameError, obj, f"Fields to exclude '{fte}' not found") + raise Signal.make( + T.NameError, obj, f"Fields to exclude '{fte}' not found" + ) exclude = direct_names | set(f.value.exclude) for name, value in remaining_items: assert isinstance(name, str) assert name not in exclude - yield ast.NamedField(name, value, user_defined=False).set_text_ref(f.text_ref) + yield ast.NamedField(name, value, user_defined=False).set_text_ref( + f.text_ref + ) else: yield f @@ -131,24 +174,31 @@ def _expand_ellipsis(obj, fields): @dsp def guess_field_name(_f): return '_' + + @dsp def guess_field_name(f: ast.Attr): name = f.name if isinstance(name, ast.Marker): name = '' return guess_field_name(f.expr) + "." + name + + @dsp def guess_field_name(f: ast.Name): return str(f.name) + + @dsp def guess_field_name(f: ast.Projection): return guess_field_name(f.table) + + @dsp def guess_field_name(f: ast.FuncCall): return guess_field_name(f.func) - # # Compilation Code # @@ -157,6 +207,8 @@ def guess_field_name(f: ast.FuncCall): @method def compile_to_inst(x: objects.Object): return x + + @method def compile_to_inst(node: ast.Ast): return node @@ -169,6 +221,8 @@ def compile_to_inst(cb: ast.CodeBlock): # TODO some statements can be evaluated at compile time raise Signal.make(T.CompileError, cb, "Cannot compile this code block") + + @method def compile_to_inst(i: ast.If): cond = cast(cast_to_instance(i.cond), T.bool) @@ -181,42 +235,52 @@ def compile_to_inst(i: ast.If): return objects.inherit_phantom_type(inst, [cond, then, else_]) - @method def compile_to_inst(proj: ast.Projection): table = cast_to_instance(proj.table) if table is objects.EmptyList: - return table # Empty list projection is always an empty list. + return table # Empty list projection is always an empty list. t = T.union[T.table, T.struct] if not table.type <= t: - raise Signal.make(T.TypeError, proj, f"Cannot project objects of type {table.type}") + raise Signal.make( + T.TypeError, proj, f"Cannot project objects of type {table.type}" + ) fields = _expand_ellipsis(table, proj.fields) # Test duplicates in field names. If an automatic name is used, collision should be impossible - dup = find_duplicate([f for f in list(proj.fields) + list(proj.agg_fields) if f.name], key=lambda f: f.name) + dup = find_duplicate( + [f for f in list(proj.fields) + list(proj.agg_fields) if f.name], + key=lambda f: f.name, + ) if dup: - raise Signal.make(T.TypeError, dup, f"Field '{dup.name}' was already used in this projection") + raise Signal.make( + T.TypeError, dup, f"Field '{dup.name}' was already used in this projection" + ) - attrs = table.all_attrs() # TODO separate here between columns and methods? (not it's done in _process_fields) + attrs = ( + table.all_attrs() + ) # TODO separate here between columns and methods? (not it's done in _process_fields) with use_scope({n: projected(c) for n, c in attrs.items()}): fields = _process_fields(fields) for name, f in fields: if not f.type <= T.union[T.primitive, T.struct, T.json, T.nulltype, T.unknown]: - raise Signal.make(T.TypeError, proj, f"Cannot project values of type: {f.type}") + raise Signal.make( + T.TypeError, proj, f"Cannot project values of type: {f.type}" + ) if isinstance(table, objects.StructInstance): - d = {n[1]:c for n, c in fields} # Remove used_defined bool - t = T.struct({n:f.type for n, f in d.items()}) + d = {n[1]: c for n, c in fields} # Remove used_defined bool + t = T.struct({n: f.type for n, f in d.items()}) return objects.StructInstance(t, d) agg_fields = [] if proj.agg_fields: - with use_scope({n:objects.aggregate(c) for n, c in attrs.items()}): + with use_scope({n: objects.aggregate(c) for n, c in attrs.items()}): agg_fields = _process_fields(proj.agg_fields) all_fields = fields + agg_fields @@ -245,12 +309,10 @@ def compile_to_inst(proj: ast.Projection): # TODO inherit primary key? indexes? # codename = state.unique_name('proj') - new_table_type = T.table(elems, temporary=False) # XXX abstract=True + new_table_type = T.table(elems, temporary=False) # XXX abstract=True # Make code - flat_codes = [code - for _, inst in all_fields - for code in inst.flatten_code()] + flat_codes = [code for _, inst in all_fields for code in inst.flatten_code()] sql_fields = [ sql.ColumnAlias.make(code, nn) @@ -258,10 +320,14 @@ def compile_to_inst(proj: ast.Projection): ] if not sql_fields: - raise Signal.make(T.TypeError, proj, "No column provided for projection (empty projection)") + raise Signal.make( + T.TypeError, proj, "No column provided for projection (empty projection)" + ) # Make Instance - new_table = objects.TableInstance.make(sql.null, new_table_type, [table] + [inst for _, inst in all_fields]) + new_table = objects.TableInstance.make( + sql.null, new_table_type, [table] + [inst for _, inst in all_fields] + ) groupby = [] limit = None @@ -270,15 +336,18 @@ def compile_to_inst(proj: ast.Projection): # Alternatively we could # groupby = [sql.null] # But postgres doesn't support it - groupby = [sql.Primitive(T.int, str(i+1)) for i in range(len(fields))] + groupby = [sql.Primitive(T.int, str(i + 1)) for i in range(len(fields))] else: limit = 1 - code = sql.Select(new_table_type, table.code, sql_fields, group_by=groupby, limit=limit) + code = sql.Select( + new_table_type, table.code, sql_fields, group_by=groupby, limit=limit + ) # Make Instance return new_table.replace(code=code) + @method def compile_to_inst(order: ast.Order): table = cast_to_instance(order.table) @@ -290,20 +359,21 @@ def compile_to_inst(order: ast.Order): for f in fields: if not f.type <= T.primitive: # TODO Support 'ordering' trait? - raise Signal.make(T.TypeError, order, f"Arguments to 'order' must be primitive") - + raise Signal.make( + T.TypeError, order, f"Arguments to 'order' must be primitive" + ) code = sql.table_order(table, [c.code for c in fields]) return objects.TableInstance.make(code, table.type, [table] + fields) + @method def compile_to_inst(expr: ast.DescOrder): obj = cast_to_instance(expr.value) return obj.replace(code=sql.Desc(obj.code)) - # @method # def compile_to_inst(lst: list): # return [evaluate( e) for e in lst] @@ -320,6 +390,7 @@ def compile_to_inst(o: ast.Or): code = sql.Case(cond.code, a.code, b.code) return objects.make_instance(code, a.type, args) + @method def compile_to_inst(o: ast.And): args = cast_to_instance(o.args) @@ -331,6 +402,7 @@ def compile_to_inst(o: ast.And): code = sql.Case(cond.code, b.code, a.code) return objects.make_instance(code, a.type, args) + @method def compile_to_inst(o: ast.Not): expr = cast_to_instance(o.expr) @@ -339,12 +411,9 @@ def compile_to_inst(o: ast.Not): return objects.make_instance(code, T.bool, [expr]) - - - @method def compile_to_inst(cmp: ast.Compare): - insts = evaluate( cmp.args) + insts = evaluate(cmp.args) if cmp.op == 'in' or cmp.op == '!in': return contains(cmp.op, insts[0], insts[1]) @@ -355,6 +424,7 @@ def compile_to_inst(cmp: ast.Compare): }.get(cmp.op, cmp.op) return compare(op, insts[0], insts[1]) + @method def compile_to_inst(neg: ast.Neg): expr = cast_to_instance(neg.expr) @@ -369,7 +439,6 @@ def compile_to_inst(arith: ast.BinOp): return compile_arith(arith, *args) - @method def compile_to_inst(x: ast.Ellipsis): raise Signal.make(T.SyntaxError, x, "Ellipsis not allowed here") @@ -382,10 +451,11 @@ def compile_to_inst(c: ast.Const): return objects.null return pyvalue_inst(c.value, c.type) + @method def compile_to_inst(d: ast.Dict_): # TODO handle duplicate key names - elems = {k or guess_field_name(v): evaluate( v) for k, v in d.elems.items()} + elems = {k or guess_field_name(v): evaluate(v) for k, v in d.elems.items()} t = T.struct({k: v.type for k, v in elems.items()}) return objects.StructInstance(t, elems) @@ -393,7 +463,7 @@ def compile_to_inst(d: ast.Dict_): @method def compile_to_inst(lst: objects.PythonList): t = lst.type.elem - x = [sql.Primitive(t, sql._repr(t,i)) for i in lst.items] + x = [sql.Primitive(t, sql._repr(t, i)) for i in lst.items] name = unique_name("list_") table_code, subq, list_type = sql.create_list(name, x) inst = objects.TableInstance.make(table_code, list_type, []) @@ -407,14 +477,16 @@ def compile_to_inst(lst: ast.List_): # XXX a little awkward return objects.EmptyList - elems = evaluate( lst.elems ) + elems = evaluate(lst.elems) types = {e.type for e in elems} if len(types) > 1: - raise Signal.make(T.TypeError, lst, f"List members must be of the same type. Got {types}") + raise Signal.make( + T.TypeError, lst, f"List members must be of the same type. Got {types}" + ) - elem_type ,= types + (elem_type,) = types if elem_type <= T.struct: rows = [sql.ValuesTuple(obj.type, obj.flatten_code()) for obj in elems] @@ -423,9 +495,15 @@ def compile_to_inst(lst: ast.List_): table_code, subq = sql.create_table(list_type, name, rows) else: if elem_type <= (T.projected | T.aggregated): - raise Signal.make(T.TypeError, lst, "Cannot create lists of projections or aggregations (%s)" % elem_type) + raise Signal.make( + T.TypeError, + lst, + "Cannot create lists of projections or aggregations (%s)" % elem_type, + ) if not (elem_type <= T.union[T.primitive, T.nulltype]): - raise Signal.make(T.TypeError, lst, "Cannot create lists of type %s" % elem_type) + raise Signal.make( + T.TypeError, lst, "Cannot create lists of type %s" % elem_type + ) assert elem_type <= lst.type.elems[ITEM_NAME], (elem_type, lst.type) @@ -458,7 +536,9 @@ def compile_to_inst(res: ast.ResolveParameters): sq2 = SafeDict() code = _resolve_sql_parameters(obj.code, subqueries=sq2) - subqueries = {k: _resolve_sql_parameters(v, subqueries=sq2) for k, v in obj.subqueries.items()} + subqueries = { + k: _resolve_sql_parameters(v, subqueries=sq2) for k, v in obj.subqueries.items() + } return obj.replace(code=code, subqueries=SafeDict(subqueries).update(sq2)) @@ -472,7 +552,7 @@ def _resolve_sql_parameters(compiled_sql, wrap=False, subqueries=None): new_code = [] for c in compiled_sql.code: if isinstance(c, sql.Parameter): - inst = evaluate( get_var(c.name)) + inst = evaluate(get_var(c.name)) if inst.type != c.type: msg = f"Internal error: Parameter is of wrong type ({c.type} != {inst.type})" raise Signal.make(T.CastError, None, msg) @@ -487,15 +567,13 @@ def _resolve_sql_parameters(compiled_sql, wrap=False, subqueries=None): return res - - @method def compile_to_inst(rps: ast.ParameterizedSqlCode): sql_code = cast_to_python_string(rps.string) if not isinstance(sql_code, str): raise Signal.make(T.TypeError, rps, f"Expected string, got '{rps.string}'") - type_ = evaluate( rps.type) + type_ = evaluate(rps.type) if isinstance(type_, objects.Instance): type_ = type_.type assert isinstance(type_, Type), type_ @@ -514,7 +592,11 @@ def compile_to_inst(rps: ast.ParameterizedSqlCode): assert t[0] == '$' if t == '$self': if self_table is None: - raise Signal.make(T.TypeError, rps, f"$self is only available for queries that return a table") + raise Signal.make( + T.TypeError, + rps, + f"$self is only available for queries that return a table", + ) inst = self_table else: obj = get_var(t[1:]) @@ -526,7 +608,9 @@ def compile_to_inst(rps: ast.ParameterizedSqlCode): inst = cast_to_instance(obj) instances.append(inst) - new_code += _resolve_sql_parameters(inst.code, wrap=bool(new_code), subqueries=subqueries).code + new_code += _resolve_sql_parameters( + inst.code, wrap=bool(new_code), subqueries=subqueries + ).code assert not subqueries else: new_code.append(t) @@ -548,9 +632,10 @@ def compile_to_inst(rps: ast.ParameterizedSqlCode): # XXX So why is it okay for projected? # Maybe all results should be projected? type_ = objects.inherit_vectorized_type(type_, instances) - code = sql.CompiledSQL(type_, new_code, None, False, False) # XXX is False correct? + code = sql.CompiledSQL(type_, new_code, None, False, False) # XXX is False correct? return make_instance(code, type_, instances) + @method def compile_to_inst(s: ast.Slice): obj = cast_to_instance(s.obj) @@ -571,7 +656,9 @@ def compile_to_inst(s: ast.Slice): stop = None if obj.type <= T.string: - code = sql.StringSlice(obj.code, sql.add_one(start.code), stop and sql.add_one(stop.code)) + code = sql.StringSlice( + obj.code, sql.add_one(start.code), stop and sql.add_one(stop.code) + ) else: start_n = cast_to_python_int(start) stop_n = stop and cast_to_python_int(stop) @@ -579,6 +666,7 @@ def compile_to_inst(s: ast.Slice): return make_instance(code, obj.type, instances) + @method def compile_to_inst(sel: ast.Selection): obj = sel.table.simplify() @@ -588,17 +676,19 @@ def compile_to_inst(sel: ast.Selection): table = cast_to_instance(obj) if table.type <= T.string: - index ,= cast_to_instance(sel.conds) + (index,) = cast_to_instance(sel.conds) assert index.type <= T.int, index.type - table = table.replace(type=T.string) # XXX why get rid of projected here? because it's a table operation node? - slice = ast.Slice(table, - ast.Range(index, ast.BinOp('+', [index, ast.Const(T.int, 1)])) - ).set_text_ref(sel.text_ref) + table = table.replace( + type=T.string + ) # XXX why get rid of projected here? because it's a table operation node? + slice = ast.Slice( + table, ast.Range(index, ast.BinOp('+', [index, ast.Const(T.int, 1)])) + ).set_text_ref(sel.text_ref) return slice.compile_to_inst() assert_type(table.type, T.table, sel, "Selection") - with use_scope({n:projected(c) for n, c in table.all_attrs().items()}): + with use_scope({n: projected(c) for n, c in table.all_attrs().items()}): conds = cast_to_instance(sel.conds) if any(t <= T.unknown for t in table.type.elem_types): @@ -606,12 +696,17 @@ def compile_to_inst(sel: ast.Selection): else: for i, c in enumerate(conds): if not c.type <= T.bool: - raise Signal.make(T.TypeError, sel.conds[i], f"Selection expected boolean, got {c.type}") + raise Signal.make( + T.TypeError, + sel.conds[i], + f"Selection expected boolean, got {c.type}", + ) code = sql.table_selection(table, [c.code for c in conds]) return type(table).make(code, table.type, [table] + conds) + @method def compile_to_inst(param: ast.Parameter): if get_access_level() == AccessLevels.COMPILE: @@ -622,37 +717,43 @@ def compile_to_inst(param: ast.Parameter): return get_var(param.name) + @method def compile_to_inst(attr: ast.Attr): if isinstance(attr.name, ast.Marker): if attr.expr: - inst = evaluate( attr.expr) - attrs = {k:(0,v) for k, v in inst.all_attrs().items()} + inst = evaluate(attr.expr) + attrs = {k: (0, v) for k, v in inst.all_attrs().items()} else: attrs = {} raise AutocompleteSuggestions(attrs) if not attr.expr: - raise Signal.make(T.NotImplementedError, attr, "Implicit attribute syntax not supported") + raise Signal.make( + T.NotImplementedError, attr, "Implicit attribute syntax not supported" + ) - inst = evaluate( attr.expr) + inst = evaluate(attr.expr) try: - return evaluate( inst.get_attr(attr.name)) + return evaluate(inst.get_attr(attr.name)) except pql_AttributeError as e: raise Signal.make(T.AttributeError, attr, e.message) - def _apply_type_generics(gen_type, type_names): - type_objs = evaluate( type_names) + type_objs = evaluate(type_names) if not type_objs: - raise Signal.make(T.TypeError, None, f"Generics expression expected a type, got nothing.") + raise Signal.make( + T.TypeError, None, f"Generics expression expected a type, got nothing." + ) for o in type_objs: if not isinstance(o, Type): if isinstance(o.code, sql.Parameter): # XXX hacky test, hacky solution raise InsufficientAccessLevel() - raise Signal.make(T.TypeError, None, f"Generics expression expected a type, got '{o}'.") + raise Signal.make( + T.TypeError, None, f"Generics expression expected a type, got '{o}'." + ) if len(type_objs) > 1: if gen_type in (T.union,): @@ -660,19 +761,21 @@ def _apply_type_generics(gen_type, type_names): raise Signal.make(T.TypeError, None, "Union types not yet supported!") - t ,= type_objs + (t,) = type_objs try: return gen_type[t] except TypeError: raise Signal.make(T.TypeError, None, f"Type {t} isn't a container!") - @method def compile_to_inst(marker: ast.Marker): - all_vars = context.state.get_all_vars_with_rank() # Uses overridden version of AcState + all_vars = ( + context.state.get_all_vars_with_rank() + ) # Uses overridden version of AcState raise AutocompleteSuggestions(all_vars) + @method def compile_to_inst(range: ast.Range): target = get_db().target @@ -689,13 +792,17 @@ def compile_to_inst(range: ast.Range): if not isinstance(stop, int): raise Signal.make(T.TypeError, range, "Range must be between integers") elif target in (sql.mysql, sql.bigquery): - raise Signal.make(T.NotImplementedError, range, f"{target} doesn't support infinite series!") + raise Signal.make( + T.NotImplementedError, range, f"{target} doesn't support infinite series!" + ) type_ = T.list[T.int] if target == sql.bigquery: # Ensure SELECT *, since UNNEST at the root level is a syntax error - code = sql.RawSql(type_, f'SELECT * FROM UNNEST(GENERATE_ARRAY({start}, {stop-1})) as item') + code = sql.RawSql( + type_, f'SELECT * FROM UNNEST(GENERATE_ARRAY({start}, {stop-1})) as item' + ) return objects.TableInstance.make(code, type_, []) if stop is None: @@ -703,12 +810,9 @@ def compile_to_inst(range: ast.Range): else: stop_str = f" WHERE item+1<{stop}" - name = unique_name("range") skip = 1 code = f"SELECT {start} AS item UNION ALL SELECT item+{skip} FROM {name}{stop_str}" subq = sql.Subquery(name, [], sql.RawSql(type_, code)) code = sql.TableName(type_, Id(name)) return objects.TableInstance(code, type_, SafeDict({name: subq})) - - diff --git a/preql/core/display.py b/preql/core/display.py index f297799..149c098 100644 --- a/preql/core/display.py +++ b/preql/core/display.py @@ -1,22 +1,20 @@ import html -import rich.table -import rich.text import rich.console import rich.markup +import rich.table +import rich.text + +from preql.settings import Display as DisplaySettings +from preql.settings import color_theme -from .exceptions import Signal -from .pql_types import T, ITEM_NAME from . import pql_objects as objects +from .exceptions import Signal +from .interp_common import call_builtin_func, cast_to_python, cast_to_python_int from .pql_ast import pyvalue -from .types_impl import dp_type, pql_repr -from .interp_common import call_builtin_func, cast_to_python_int, cast_to_python +from .pql_types import ITEM_NAME, T from .state import get_display - -from preql.settings import color_theme, Display as DisplaySettings - - - +from .types_impl import dp_type, pql_repr @dp_type @@ -30,14 +28,20 @@ def pql_repr(t: T.function, value): return f'{{func {value.name}({", ".join(params)})}}' + @dp_type def pql_repr(t: T.decimal, value): raise Signal.make(T.NotImplementedError, None, "Decimal not implemented") + @dp_type def pql_repr(t: T.string, value): if not isinstance(value, str): - raise Signal.make(T.TypeError, None, f"Expected value of type 'string', instead got {type(value)}") + raise Signal.make( + T.TypeError, + None, + f"Expected value of type 'string', instead got {type(value)}", + ) value = value.replace('"', r'\"') res = f'"{value}"' @@ -50,11 +54,13 @@ def pql_repr(t: T.string, value): return res + @dp_type def pql_repr(t: T.text, value): assert isinstance(value, str), value return str(value) + @dp_type def pql_repr(t: T._rich, value): r = rich.text.Text.from_markup(str(value)) @@ -62,10 +68,12 @@ def pql_repr(t: T._rich, value): return _rich_to_html(r) return r + @dp_type def pql_repr(t: T.bool, value): return 'true' if value else 'false' + @dp_type def pql_repr(t: T.nulltype, value): return 'null' @@ -74,7 +82,9 @@ def pql_repr(t: T.nulltype, value): def _rich_to_html(r): console = rich.console.Console(record=True) console.print(r) - return console.export_html(code_format='
{code}
').replace('━', '-') + return console.export_html( + code_format='
{code}
' + ).replace('━', '-') def table_limit(table, limit, offset=0): @@ -112,10 +122,15 @@ def _html_table(name, count_str, rows, offset, has_more, colors): } """ - return '%s%s%s
' % (header, ths, '\n'.join(trs)) + style + return ( + '%s%s%s
' % (header, ths, '\n'.join(trs)) + + style + ) -def _rich_table(name, count_str, rows, offset, has_more, colors=True, show_footer=False): +def _rich_table( + name, count_str, rows, offset, has_more, colors=True, show_footer=False +): header = 'table ' if name: header += name @@ -126,32 +141,37 @@ def _rich_table(name, count_str, rows, offset, has_more, colors=True, show_foote if not rows: return header - table = rich.table.Table(title=rich.text.Text(header), show_footer=show_footer, min_width=len(header)) + table = rich.table.Table( + title=rich.text.Text(header), show_footer=show_footer, min_width=len(header) + ) # TODO enable/disable styling for k, v in rows[0].items(): kw = {} if isinstance(v, (int, float)): - kw['justify']='right' + kw['justify'] = 'right' if colors: if isinstance(v, int): kw['style'] = color_theme['number'] elif isinstance(v, float): - kw['style'] ='yellow' + kw['style'] = 'yellow' elif isinstance(v, str): kw['style'] = color_theme['string'] table.add_column(k, footer=k, **kw) for r in rows: - table.add_row(*[rich.markup.escape(str(x) if x is not None else '-') for x in r.values()]) + table.add_row( + *[rich.markup.escape(str(x) if x is not None else '-') for x in r.values()] + ) if has_more: table.add_row(*['...' for x in rows[0]]) return table + _g_last_table = None _g_last_offset = 0 @@ -162,6 +182,7 @@ def _table_name(table): except KeyError: return '' + def _preview_table(table, size, offset): if size == 0: return [] @@ -185,11 +206,12 @@ def table_inline_repr(self): return '[%s]' % ', '.join(repr(r) for r in rows) - def table_repr(self, offset=0): max_count = DisplaySettings.MAX_AUTO_COUNT - count = cast_to_python_int(call_builtin_func('count', [table_limit(self, max_count)])) + count = cast_to_python_int( + call_builtin_func('count', [table_limit(self, max_count)]) + ) if count == max_count: count_str = f'>={count}' else: @@ -234,6 +256,7 @@ def module_repr(module): res = html.escape(res) return res + def function_repr(func): res = '<%s>' % func.help_str() if get_display().format == 'html': @@ -245,14 +268,16 @@ class Display: def print(self, repr_): print(repr_) + def _print_rich_exception(console, e): console.print('[bold]Exception traceback:[/bold]') for ref in e.text_refs: - for line in (ref.get_pinpoint_text(rich=True) if ref else ['???']): + for line in ref.get_pinpoint_text(rich=True) if ref else ['???']: console.print(line) console.print() console.print(rich.text.Text('%s: %s' % (e.type, e.message))) + class RichDisplay(Display): format = "rich" @@ -274,7 +299,7 @@ def print_exception(self, e): def print_to_string(x, format): console = rich.console.Console(color_system=None) - with console.capture() as capture: + with console.capture() as capture: console.print(x) return capture.get() @@ -293,7 +318,9 @@ def print(self, repr_, end="
"): def print_exception(self, e): console = rich.console.Console(record=True) _print_rich_exception(console, e) - res = console.export_html(code_format='
{code}
').replace('━', '-') + res = console.export_html( + code_format='
{code}
' + ).replace('━', '-') self.buffer.append(res) def as_html(self): @@ -304,7 +331,6 @@ def as_html(self): return res - def install_reprs(): objects.CollectionInstance.repr = table_repr objects.CollectionInstance.inline_repr = table_inline_repr diff --git a/preql/core/evaluate.py b/preql/core/evaluate.py index e4a1d0c..a55a55d 100644 --- a/preql/core/evaluate.py +++ b/preql/core/evaluate.py @@ -1,31 +1,56 @@ -from typing import List, Optional import logging from pathlib import Path +from typing import List, Optional -from preql.utils import safezip, dataclass, SafeDict, listgen, method from preql import settings from preql.context import context +from preql.utils import SafeDict, dataclass, listgen, method, safezip -from .interp_common import assert_type, exclude_fields, call_builtin_func, is_global_scope, cast_to_python_string, cast_to_python_int -from .state import set_var, use_scope, get_var, unique_name, get_db, catch_access, AccessLevels, get_access_level, reduce_access, has_var from . import exceptions as exc -from . import pql_objects as objects from . import pql_ast as ast +from . import pql_objects as objects from . import sql -from .parser import Str -from .interp_common import dsp, pyvalue_inst, cast_to_python from .compiler import cast_to_instance -from .pql_types import T, Type, Object, Id, dp_inst -from .types_impl import table_params, table_flat_for_insert, flatten_type, pql_repr, kernel_type from .exceptions import InsufficientAccessLevel, ReturnSignal, Signal - +from .interp_common import ( + assert_type, + call_builtin_func, + cast_to_python, + cast_to_python_int, + cast_to_python_string, + dsp, + exclude_fields, + is_global_scope, + pyvalue_inst, +) +from .parser import Str +from .pql_types import Id, Object, T, Type, dp_inst +from .state import ( + AccessLevels, + catch_access, + get_access_level, + get_db, + get_var, + has_var, + reduce_access, + set_var, + unique_name, + use_scope, +) +from .types_impl import ( + flatten_type, + kernel_type, + pql_repr, + table_flat_for_insert, + table_params, +) MODULES_PATH = Path(__file__).parent.parent / 'modules' @dsp def resolve(struct_def: ast.StructDef): - members = {str(k):resolve(v) for k, v in struct_def.members} + members = {str(k): resolve(v) for k, v in struct_def.members} struct = T.struct(members) set_var(struct_def.name, struct) return struct @@ -37,13 +62,18 @@ def _resolve_name_and_scope(name, ast_node): else: temporary = True if len(name.parts) > 1: - raise Signal(T.NameError, ast_node, "Local tables cannot include a schema (namespace)") - name ,= name.parts + raise Signal( + T.NameError, + ast_node, + "Local tables cannot include a schema (namespace)", + ) + (name,) = name.parts name = Id('__local_' + unique_name(name)) name = get_db().qualified_name(name) return name, temporary + @dsp def resolve(table_def: ast.TableDef): name, temporary = _resolve_name_and_scope(table_def.name, table_def) @@ -56,10 +86,11 @@ def resolve(table_def: ast.TableDef): if table_def.methods: methods = evaluate(table_def.methods) - t.proto_attrs.update({m.userfunc.name:m.userfunc for m in methods}) + t.proto_attrs.update({m.userfunc.name: m.userfunc for m in methods}) return t + @dsp def resolve(col_def: ast.ColumnDef): coltype = resolve(col_def.type) @@ -71,29 +102,48 @@ def resolve(col_def: ast.ColumnDef): table = coltype.parent.type if 'name' not in table.options: # XXX better test for persistence - raise Signal.make(T.TypeError, col_def.type, "Tables provided as relations must be persistent.") - - x = T.t_relation[coltype.type](rel={'table': table, 'column': coltype.name, 'key': False}) - return x.replace(_nullable=coltype.type._nullable) # inherit is_nullable (TODO: use sumtypes?) + raise Signal.make( + T.TypeError, + col_def.type, + "Tables provided as relations must be persistent.", + ) + + x = T.t_relation[coltype.type]( + rel={'table': table, 'column': coltype.name, 'key': False} + ) + return x.replace( + _nullable=coltype.type._nullable + ) # inherit is_nullable (TODO: use sumtypes?) elif coltype <= T.table: if 'name' not in coltype.options: # XXX better test for persistence - raise Signal.make(T.TypeError, col_def.type, "Tables provided as relations must be persistent.") - - x = T.t_relation[T.t_id.as_nullable()](rel={'table': coltype, 'column': 'id', 'key': True}) - return x.replace(_nullable=coltype._nullable) # inherit is_nullable (TODO: use sumtypes?) + raise Signal.make( + T.TypeError, + col_def.type, + "Tables provided as relations must be persistent.", + ) + + x = T.t_relation[T.t_id.as_nullable()]( + rel={'table': coltype, 'column': 'id', 'key': True} + ) + return x.replace( + _nullable=coltype._nullable + ) # inherit is_nullable (TODO: use sumtypes?) return coltype(default=col_def.default) + @dsp def resolve(type_: ast.Type): - t = evaluate( type_.type_obj) + t = evaluate(type_.type_obj) if isinstance(t, objects.TableInstance): t = t.type if not isinstance(t, (Type, objects.SelectedColumnInstance)): - raise Signal.make(T.TypeError, type_, f"Expected type in column definition. Instead got '{t}'") + raise Signal.make( + T.TypeError, type_, f"Expected type in column definition. Instead got '{t}'" + ) if type_.nullable: t = t.as_nullable() @@ -101,31 +151,34 @@ def resolve(type_: ast.Type): return t - def db_query(sql_code, subqueries=None): try: return get_db().query(sql_code, subqueries) except exc.DatabaseQueryError as e: raise Signal.make(T.DbQueryError, None, e.args[0]) from e + def drop_table(table_type): name = table_type.options['name'] code = sql.compile_drop_table(name) return db_query(code, {}) - @dsp def _set_value(name: ast.Name, value): set_var(name.name, value) + @dsp def _set_value(attr: ast.Attr, value): - raise Signal.make(T.NotImplementedError, attr, f"Cannot set attribute for {attr.expr.repr()}") + raise Signal.make( + T.NotImplementedError, attr, f"Cannot set attribute for {attr.expr.repr()}" + ) + def _copy_rows(target_name: ast.Name, source: objects.TableInstance): - if source is objects.EmptyList: # Nothing to add + if source is objects.EmptyList: # Nothing to add return objects.null target = evaluate(target_name) @@ -133,17 +186,27 @@ def _copy_rows(target_name: ast.Name, source: objects.TableInstance): params = dict(table_params(target.type)) for p in params: if p not in source.type.elems: - raise Signal.make(T.TypeError, source, f"Missing column '{p}' in {source.type}") + raise Signal.make( + T.TypeError, source, f"Missing column '{p}' in {source.type}" + ) read_only, columns = table_flat_for_insert(target.type) if get_db().target == sql.bigquery and 'id' in read_only: # XXX very hacky! to_exclude = ['id'] if 'id' in source.type.elems else [] - proj = ast.Projection(source, [ - ast.NamedField('id', objects.Instance.make(sql.RawSql(T.string, 'GENERATE_UUID()'), T.string, [])), - ast.NamedField(None, ast.Ellipsis(None, to_exclude)) - ]) + proj = ast.Projection( + source, + [ + ast.NamedField( + 'id', + objects.Instance.make( + sql.RawSql(T.string, 'GENERATE_UUID()'), T.string, [] + ), + ), + ast.NamedField(None, ast.Ellipsis(None, to_exclude)), + ], + ) source = cast_to_instance(proj) read_only.remove('id') columns.insert(0, 'id') @@ -154,7 +217,9 @@ def _copy_rows(target_name: ast.Name, source: objects.TableInstance): try: table_name = table.options['name'] except KeyError: - raise Signal.make(T.ValueError, target_name, "Cannot add a new row to an unnamed table") + raise Signal.make( + T.ValueError, target_name, "Cannot add a new row to an unnamed table" + ) code = sql.Insert(table_name, columns, source.code) db_query(code, source.subqueries) @@ -165,6 +230,7 @@ def _copy_rows(target_name: ast.Name, source: objects.TableInstance): def _execute(struct_def: ast.StructDef): resolve(struct_def) + @method def _execute(table_def: ast.TableDefFromExpr): expr = cast_to_instance(table_def.expr) @@ -174,14 +240,15 @@ def _execute(table_def: ast.TableDefFromExpr): # name = get_db().qualified_name(name) t = new_table_from_expr(name, expr, table_def.const, temporary) set_var(table_def.name, t) - + + @method def _execute(var_def: ast.SetValue): res = evaluate(var_def.value) if res.type <= T.primitive and not res.type <= (T.union[T.aggregated, T.projected]): res = objects.pyvalue_inst(res.localize(), res.type) - + _set_value(var_def.name, res) return res @@ -192,15 +259,16 @@ def ensure_namespace(name: Id): return if len(path) > 1: - raise Signal(T.NotImplementedError, name, "Nested namespaces not supported yet!") + raise Signal( + T.NotImplementedError, name, "Nested namespaces not supported yet!" + ) - name ,= path + (name,) = path if not has_var(name): module = objects.Module(name, {}) set_var(name, module) - @method def _execute(table_def: ast.TableDef): if table_def.columns and isinstance(table_def.columns[-1], ast.Ellipsis): @@ -219,20 +287,26 @@ def _execute(table_def: ast.TableDef): if t.options['temporary']: # register name for later removal - get_var('__unwind__').append( lambda: drop_table(t) ) + get_var('__unwind__').append(lambda: drop_table(t)) exists = get_db().table_exists(db_name) if exists: assert not t.options['temporary'] - cur_type = get_db().import_table_type(db_name, None if ellipsis else set(t.elems) | {'id'}) + cur_type = get_db().import_table_type( + db_name, None if ellipsis else set(t.elems) | {'id'} + ) if ellipsis: - elems_to_add = {Str(n, ellipsis.text_ref): v for n, v in cur_type.elems.items() if n not in t.elems} + elems_to_add = { + Str(n, ellipsis.text_ref): v + for n, v in cur_type.elems.items() + if n not in t.elems + } # TODO what is primary key isn't included? t = t({**t.elems, **elems_to_add}, **cur_type.options) # Auto-add id only if it exists already and not defined by user - if 'id' in cur_type.elems: #and 'id' not in t.elems: + if 'id' in cur_type.elems: # and 'id' not in t.elems: # t = t(dict(id=T.t_id, **t.elems), pk=[['id']]) assert cur_type.elems['id'] <= T.primitive, cur_type.elems['id'] t.elems['id'] = T.t_id @@ -240,7 +314,11 @@ def _execute(table_def: ast.TableDef): for e_name, e1_type in t.elems.items(): if e_name not in cur_type.elems: - raise Signal.make(T.TypeError, table_def, f"Column '{e_name}' defined, but doesn't exist in database.") + raise Signal.make( + T.TypeError, + table_def, + f"Column '{e_name}' defined, but doesn't exist in database.", + ) # e2_type = cur_type.elems[e_name] # XXX use can_cast() instead of hardcoding it @@ -263,6 +341,7 @@ def _execute(table_def: ast.TableDef): sql_code = sql.compile_type_def(db_name, t) db_query(sql_code) + @method def _execute(insert_rows: ast.InsertRows): if not isinstance(insert_rows.name, ast.Name): @@ -275,6 +354,7 @@ def _execute(insert_rows: ast.InsertRows): return _copy_rows(insert_rows.name, rval) + @method def _execute(func_def: ast.FuncDef): func = func_def.userfunc @@ -289,6 +369,7 @@ def _execute(func_def: ast.FuncDef): set_var(func.name, func.replace(params=new_params)) + @method def _execute(p: ast.Print): display = context.state.display @@ -305,17 +386,19 @@ def _execute(p: ast.Print): display.print(repr_, end=" ") display.print("") + @method def _execute(p: ast.Assert): res = cast_to_python(p.cond) if not res: # TODO pretty print values if isinstance(p.cond, ast.Compare): - s = (' %s '%p.cond.op).join(str(evaluate(a).repr()) for a in p.cond.args) + s = (' %s ' % p.cond.op).join(str(evaluate(a).repr()) for a in p.cond.args) else: s = p.cond.repr() raise Signal.make(T.AssertError, p.cond, f"Assertion failed: {s}") + @method def _execute(cb: ast.CodeBlock): for stmt in cb.statements: @@ -332,11 +415,13 @@ def _execute(i: ast.If): elif i.else_: execute(i.else_) + @method def _execute(w: ast.While): while cast_to_python(w.cond): execute(w.do) + @method def _execute(f: ast.For): expr = cast_to_python(f.iterable) @@ -344,6 +429,7 @@ def _execute(f: ast.For): with use_scope({f.var: objects.from_python(i)}): execute(f.do) + @method def _execute(t: ast.Try): try: @@ -351,7 +437,11 @@ def _execute(t: ast.Try): except Signal as e: catch_type = evaluate(t.catch_expr).localize() if not isinstance(catch_type, Type): - raise Signal.make(T.TypeError, t.catch_expr, f"Catch expected type, got {t.catch_expr.type}") + raise Signal.make( + T.TypeError, + t.catch_expr, + f"Catch expected type, got {t.catch_expr.type}", + ) if e.type <= catch_type: scope = {t.catch_name: e} if t.catch_name else {} with use_scope(scope): @@ -363,7 +453,7 @@ def _execute(t: ast.Try): def find_module(module_name): paths = [MODULES_PATH, Path.cwd()] for path in paths: - module_path = (path / module_name).with_suffix(".pql") + module_path = (path / module_name).with_suffix(".pql") if module_path.exists(): return module_path @@ -398,11 +488,13 @@ def _execute(r: ast.Import): set_var(r.as_name or r.module_path, module) return module + @method def _execute(r: ast.Return): value = evaluate(r.value) raise ReturnSignal(value) + @method def _execute(t: ast.Throw): e = evaluate(t.value) @@ -411,17 +503,17 @@ def _execute(t: ast.Throw): assert isinstance(e, Exception), e raise e + def execute(stmt): if isinstance(stmt, ast.Statement): return stmt._execute() or objects.null return evaluate(stmt) - - # Simplify performs local operations before any db-specific compilation occurs # Technically not super useful at the moment, but makes conceptual sense. + @method def simplify(cb: ast.CodeBlock): # if len(cb.statements) == 1: @@ -441,15 +533,18 @@ def simplify(cb: ast.CodeBlock): except InsufficientAccessLevel: return cb + @method def simplify(n: ast.Name): # XXX what happens to caching if this is a global variable? return get_var(n.name) + @method def simplify(x: Object): return x + # @dsp # def simplify(ls: list): # return [simplify(i) for i in ls] @@ -483,11 +578,15 @@ def simplify(x: Object): # TODO Optimize these, right now failure to evaluate will lose all work @method def simplify(obj: ast.Or): - a, b = evaluate( obj.args) + a, b = evaluate(obj.args) ta = kernel_type(a.type) tb = kernel_type(b.type) if ta != tb: - raise Signal.make(T.TypeError, obj, f"'or' operator requires both arguments to be of the same type, but got '{ta}' and '{tb}'.") + raise Signal.make( + T.TypeError, + obj, + f"'or' operator requires both arguments to be of the same type, but got '{ta}' and '{tb}'.", + ) try: if a.test_nonzero(): return a @@ -498,11 +597,15 @@ def simplify(obj: ast.Or): @method def simplify(obj: ast.And): - a, b = evaluate( obj.args) + a, b = evaluate(obj.args) ta = kernel_type(a.type) tb = kernel_type(b.type) if ta != tb: - raise Signal.make(T.TypeError, obj, f"'and' operator requires both arguments to be of the same type, but got '{ta}' and '{tb}'.") + raise Signal.make( + T.TypeError, + obj, + f"'and' operator requires both arguments to be of the same type, but got '{ta}' and '{tb}'.", + ) try: if not a.test_nonzero(): return a @@ -513,7 +616,7 @@ def simplify(obj: ast.And): @method def simplify(obj: ast.Not): - inst = evaluate( obj.expr) + inst = evaluate(obj.expr) try: nz = inst.test_nonzero() except InsufficientAccessLevel: @@ -521,7 +624,6 @@ def simplify(obj: ast.Not): return objects.pyvalue_inst(not nz) - @method def simplify(funccall: ast.FuncCall): state = context.state @@ -529,7 +631,11 @@ def simplify(funccall: ast.FuncCall): if isinstance(func, objects.UnknownInstance): # evaluate( [a.value for a in funccall.args]) - raise Signal.make(T.TypeError, funccall.func, f"Error: Object of type '{func.type}' is not callable") + raise Signal.make( + T.TypeError, + funccall.func, + f"Error: Object of type '{func.type}' is not callable", + ) args = funccall.args if isinstance(func, Type): @@ -538,7 +644,11 @@ def simplify(funccall: ast.FuncCall): func = get_var('cast') if not isinstance(func, objects.Function): - raise Signal.make(T.TypeError, funccall.func, f"Error: Object of type '{func.type}' is not callable") + raise Signal.make( + T.TypeError, + funccall.func, + f"Error: Object of type '{func.type}' is not callable", + ) state.stacktrace.append(funccall.text_ref) try: @@ -569,14 +679,19 @@ def eval_func_call(func, args): # Don't I need an instance to ensure I have type? for i, (p, a) in enumerate(matched_args): - if not p.name.startswith('$'): # $param means don't evaluate expression, leave it to the function - a = evaluate( a) + if not p.name.startswith( + '$' + ): # $param means don't evaluate expression, leave it to the function + a = evaluate(a) # TODO cast? if p.type and not a.type <= p.type: - raise Signal.make(T.TypeError, func, f"Argument #{i} of '{func.name}' is of type '{a.type}', expected '{p.type}'") + raise Signal.make( + T.TypeError, + func, + f"Argument #{i} of '{func.name}' is of type '{a.type}', expected '{p.type}'", + ) ordered_args[p.name] = a - if isinstance(func, objects.InternalFunction): # TODO ensure pure function? # TODO Ensure correct types @@ -586,7 +701,10 @@ def eval_func_call(func, args): # TODO make tests to ensure caching was successful expr = func.expr if settings.cache: - params = {name: ast.Parameter(name, value.type) for name, value in ordered_args.items()} + params = { + name: ast.Parameter(name, value.type) + for name, value in ordered_args.items() + } sig = (func.name,) + tuple(a.type for a in ordered_args.values()) try: @@ -629,50 +747,57 @@ def _call_expr(expr): except ReturnSignal as r: return r.value + # TODO fix these once we have proper types @method def test_nonzero(table: objects.TableInstance): count = call_builtin_func("count", [table]) return bool(cast_to_python_int(count)) + @method def test_nonzero(inst: objects.Instance): return bool(cast_to_python(inst)) + @method def test_nonzero(inst: Type): return True - - - - - - @method def apply_database_rw(o: ast.One): # TODO move these to the core/base module - obj = evaluate( o.expr) + obj = evaluate(o.expr) if obj.type <= T.struct: if len(obj.attrs) != 1: - raise Signal.make(T.ValueError, o, f"'one' expected a struct with a single attribute, got {len(obj.attrs)}") - x ,= obj.attrs.values() + raise Signal.make( + T.ValueError, + o, + f"'one' expected a struct with a single attribute, got {len(obj.attrs)}", + ) + (x,) = obj.attrs.values() return x - slice_ast = ast.Slice(obj, ast.Range(None, ast.Const(T.int, 2))).set_text_ref(o.text_ref) - table = evaluate( slice_ast) + slice_ast = ast.Slice(obj, ast.Range(None, ast.Const(T.int, 2))).set_text_ref( + o.text_ref + ) + table = evaluate(slice_ast) - assert (table.type <= T.table), table - rows = table.localize() # Must be 1 row + assert table.type <= T.table, table + rows = table.localize() # Must be 1 row if len(rows) == 0: if not o.nullable: - raise Signal.make(T.ValueError, o, "'one' expected a single result, got an empty expression") + raise Signal.make( + T.ValueError, + o, + "'one' expected a single result, got an empty expression", + ) return objects.null elif len(rows) > 1: raise Signal.make(T.ValueError, o, "'one' expected a single result, got more") - row ,= rows + (row,) = rows rowtype = T.row[table.type] if table.type <= T.list: @@ -690,13 +815,15 @@ def apply_database_rw(d: ast.Delete): # TODO Optimize: Delete on condition, not id, when possible cond_table = ast.Selection(d.table, d.conds).set_text_ref(d.text_ref) - table = evaluate( cond_table) + table = evaluate(cond_table) if not table.type <= T.table: raise Signal.make(T.TypeError, d.table, f"Expected a table. Got: {table.type}") if not 'name' in table.type.options: - raise Signal.make(T.ValueError, d.table, "Cannot delete. Table is not persistent") + raise Signal.make( + T.ValueError, d.table, "Cannot delete. Table is not persistent" + ) rows = list(table.localize()) if rows: @@ -708,37 +835,44 @@ def apply_database_rw(d: ast.Delete): for code in sql.deletes_by_ids(table, ids): db_query(code, table.subqueries) - return evaluate( d.table) + return evaluate(d.table) + @method def apply_database_rw(u: ast.Update): catch_access(AccessLevels.WRITE_DB) # TODO Optimize: Update on condition, not id, when possible - table = evaluate( u.table) + table = evaluate(u.table) if not table.type <= T.table: raise Signal.make(T.TypeError, u.table, f"Expected a table. Got: {table.type}") if not 'name' in table.type.options: - raise Signal.make(T.ValueError, u.table, "Cannot update: Table is not persistent") + raise Signal.make( + T.ValueError, u.table, "Cannot update: Table is not persistent" + ) for f in u.fields: if not f.name: - raise Signal.make(T.SyntaxError, f, f"Update requires that all fields have a name") + raise Signal.make( + T.SyntaxError, f, f"Update requires that all fields have a name" + ) # TODO verify table is concrete (i.e. lvalue, not a transitory expression) - update_scope = {n:c for n, c in table.all_attrs().items()} + update_scope = {n: c for n, c in table.all_attrs().items()} with use_scope(update_scope): - proj = {f.name:evaluate( f.value) for f in u.fields} + proj = {f.name: evaluate(f.value) for f in u.fields} rows = list(table.localize()) if rows: if 'id' not in rows[0]: raise Signal.make(T.TypeError, u, "Update error: Table does not contain id") if not set(proj) < set(rows[0]): - raise Signal.make(T.TypeError, u, "Update error: Not all keys exist in table") + raise Signal.make( + T.TypeError, u, "Update error: Not all keys exist in table" + ) ids = [row['id'] for row in rows] @@ -756,11 +890,13 @@ def apply_database_rw(new: ast.NewRows): obj = get_var(new.type) if len(new.args) > 1: - raise Signal.make(T.NotImplementedError, new, "Not yet implemented") #. Requires column-wise table concat (use join and enum)") + raise Signal.make( + T.NotImplementedError, new, "Not yet implemented" + ) # . Requires column-wise table concat (use join and enum)") if isinstance(obj, objects.UnknownInstance): - arg ,= new.args - table = evaluate( arg.value) + (arg,) = new.args + table = evaluate(arg.value) fakerows = [objects.RowInstance(T.row[table], {'id': T.t_id})] return ast.List_(T.list[T.int], fakerows).set_text_ref(new.text_ref) @@ -768,12 +904,14 @@ def apply_database_rw(new: ast.NewRows): # XXX Is it always TableInstance? Just sometimes? What's the transition here? obj = obj.type - assert_type(obj, T.table, new, "'new' expected an object of type '%s', instead got '%s'") + assert_type( + obj, T.table, new, "'new' expected an object of type '%s', instead got '%s'" + ) - arg ,= new.args + (arg,) = new.args # TODO postgres can do it better! - table = evaluate( arg.value) + table = evaluate(arg.value) rows = table.localize() # TODO ensure rows are the right type @@ -784,7 +922,9 @@ def apply_database_rw(new: ast.NewRows): ids = [] for row in rows: matched = cons.match_params([objects.from_python(v) for v in row.values()]) - ids += [_new_row(new, obj, matched).primary_key()] # XXX return everything, not just pk? + ids += [ + _new_row(new, obj, matched).primary_key() + ] # XXX return everything, not just pk? # XXX find a nicer way - requires a better typesystem, where id(t) < int return ast.List_(T.list[T.int], ids).set_text_ref(new.text_ref) @@ -813,27 +953,34 @@ def _destructure_param_match(ast_node, param_match): def _new_value(v, type_): if isinstance(v, list): - return evaluate( objects.PythonList(v)) + return evaluate(objects.PythonList(v)) return objects.pyvalue_inst(v, type_=type_) + @dsp def freeze(i: objects.Instance): - return _new_value(cast_to_python(i), type_=i.type ) + return _new_value(cast_to_python(i), type_=i.type) + @dsp def freeze(i: objects.RowInstance): return i.replace(attrs={k: freeze(v) for k, v in i.attrs.items()}) + def _new_row(new_ast, table, matched): - matched = [(k, freeze(evaluate( v))) for k, v in matched] + matched = [(k, freeze(evaluate(v))) for k, v in matched] destructured_pairs = _destructure_param_match(new_ast, matched) keys = [name for (name, _) in destructured_pairs] - values = [sql.make_value(v) for (_,v) in destructured_pairs] + values = [sql.make_value(v) for (_, v) in destructured_pairs] # XXX use regular insert? if 'name' not in table.options: - raise Signal.make(T.TypeError, new_ast, f"'new' expects a persistent table. Instead got a table expression.") + raise Signal.make( + T.TypeError, + new_ast, + f"'new' expects a persistent table. Instead got a table expression.", + ) if get_db().target == sql.bigquery: rowid = db_query(sql.FuncCall(T.string, 'GENERATE_UUID', [])) @@ -843,7 +990,9 @@ def _new_row(new_ast, table, matched): try: table_name = table.options['name'] except KeyError: - raise Signal.make(T.ValueError, new_ast, "Cannot add a new row to an unnamed table") + raise Signal.make( + T.ValueError, new_ast, "Cannot add a new row to an unnamed table" + ) q = sql.InsertConsts(table_name, keys, [values]) db_query(q) @@ -852,11 +1001,10 @@ def _new_row(new_ast, table, matched): rowid = db_query(sql.LastRowId()) d = SafeDict({'id': objects.pyvalue_inst(rowid)}) - d.update({p.name:v for p, v in matched}) + d.update({p.name: v for p, v in matched}) return objects.RowInstance(T.row[table], d) - @method def apply_database_rw(new: ast.New): catch_access(AccessLevels.WRITE_DB) @@ -865,27 +1013,43 @@ def apply_database_rw(new: ast.New): # XXX Assimilate this special case if isinstance(obj, Type) and obj <= T.Exception: + def create_exception(msg): state = context.state msg = cast_to_python(msg) assert new.text_ref is state.stacktrace[-1] - return Signal(obj, list(state.stacktrace), msg) # TODO move this to `throw`? - f = objects.InternalFunction(obj.typename, [objects.Param('message')], create_exception) - res = evaluate( ast.FuncCall(f, new.args).set_text_ref(new.text_ref)) + return Signal( + obj, list(state.stacktrace), msg + ) # TODO move this to `throw`? + + f = objects.InternalFunction( + obj.typename, [objects.Param('message')], create_exception + ) + res = evaluate(ast.FuncCall(f, new.args).set_text_ref(new.text_ref)) return res if not isinstance(obj, objects.TableInstance): - raise Signal.make(T.TypeError, new, f"'new' expects a table or exception, instead got {obj.repr()}") + raise Signal.make( + T.TypeError, + new, + f"'new' expects a table or exception, instead got {obj.repr()}", + ) table = obj # TODO assert tabletype is a real table and not a query (not transient), otherwise new is meaningless - assert_type(table.type, T.table, new, "'new' expected an object of type '%s', instead got '%s'") + assert_type( + table.type, + T.table, + new, + "'new' expected an object of type '%s', instead got '%s'", + ) cons = TableConstructor.make(table.type) matched = cons.match_params(new.args) return _new_row(new, table.type, matched) + @method def apply_database_rw(x: Object): return x @@ -901,22 +1065,29 @@ class TableConstructor(objects.Function): @classmethod def make(cls, table): - return cls([ - objects.Param(name, p, p.options.get('default'), orig=p).set_text_ref(getattr(name, 'text_ref', None)) - for name, p in table_params(table) - ]) + return cls( + [ + objects.Param(name, p, p.options.get('default'), orig=p).set_text_ref( + getattr(name, 'text_ref', None) + ) + for name, p in table_params(table) + ] + ) def add_as_subquery(inst: objects.Instance): code_cls = sql.TableName if (inst.type <= T.table) else sql.Name name = unique_name(inst) - return inst.replace(code=code_cls(inst.code.type, name), subqueries=inst.subqueries.update({name: inst.code})) - + return inst.replace( + code=code_cls(inst.code.type, name), + subqueries=inst.subqueries.update({name: inst.code}), + ) @dsp def evaluate(obj: list): - return [evaluate( item) for item in obj] + return [evaluate(item) for item in obj] + @dsp def evaluate(obj_): @@ -954,7 +1125,6 @@ def evaluate(obj_): return obj - # # localize() # ------------- @@ -962,14 +1132,17 @@ def evaluate(obj_): # Return the local value of the expression. Only requires computation if the value is an instance. # + @method def localize(inst: objects.AbsInstance): raise NotImplementedError(inst) + @method def localize(inst: objects.AbsStructInstance): return {k: evaluate(v).localize() for k, v in inst.attrs.items()} + @method def localize(inst: objects.Instance): # TODO This protection doesn't work for unoptimized code @@ -981,16 +1154,19 @@ def localize(inst: objects.Instance): return db_query(inst.code, inst.subqueries) + @method def localize(inst: objects.ValueInstance): return inst.local_value + @method def localize(inst: objects.SelectedColumnInstance): # XXX is this right? p = evaluate(inst.parent) return p.get_attr(inst.name) + @method def localize(x: Object): return x @@ -1000,13 +1176,10 @@ def new_table_from_rows(name, columns, rows, temporary): # TODO check table doesn't exist name = Id(name) - tuples = [ - [sql.make_value(i) for i in row] - for row in rows - ] + tuples = [[sql.make_value(i) for i in row] for row in rows] # TODO refactor into function? - elems = {c:v.type.as_nullable() for c,v in zip(columns, tuples[0])} + elems = {c: v.type.as_nullable() for c, v in zip(columns, tuples[0])} elems = {'id': T.t_id, **elems} table = T.table(elems, temporary=temporary, pk=[['id']], name=name) @@ -1027,7 +1200,9 @@ def new_table_from_expr(name, expr, const, temporary): msg = "Field 'id' already exists. Rename it, or use 'const table' to copy it as-is." raise Signal.make(T.NameError, None, msg) - table = T.table(dict(elems), name=name, pk=[] if const else [['id']], temporary=temporary) + table = T.table( + dict(elems), name=name, pk=[] if const else [['id']], temporary=temporary + ) if not const: table.elems['id'] = T.t_id @@ -1035,17 +1210,25 @@ def new_table_from_expr(name, expr, const, temporary): db_query(sql.compile_type_def(name, table)) if temporary: - get_var('__unwind__').append( lambda: drop_table(table) ) + get_var('__unwind__').append(lambda: drop_table(table)) read_only, flat_columns = table_flat_for_insert(table) if get_db().target == sql.bigquery and 'id' in read_only: # XXX very hacky! to_exclude = ['id'] if 'id' in expr.type.elems else [] - proj = ast.Projection(expr, [ - ast.NamedField('id', objects.Instance.make(sql.RawSql(T.string, 'GENERATE_UUID()'), T.string, [])), - ast.NamedField(None, ast.Ellipsis(None, to_exclude)) - ]) + proj = ast.Projection( + expr, + [ + ast.NamedField( + 'id', + objects.Instance.make( + sql.RawSql(T.string, 'GENERATE_UUID()'), T.string, [] + ), + ), + ast.NamedField(None, ast.Ellipsis(None, to_exclude)), + ], + ) expr = cast_to_instance(proj) read_only.remove('id') flat_columns.insert(0, 'id') @@ -1058,15 +1241,18 @@ def new_table_from_expr(name, expr, const, temporary): # cast_to_python - make sure the value is a native python object, not a preql instance + @dsp def cast_to_python(obj): raise Signal.make(T.TypeError, None, f"Unexpected value: {pql_repr(obj.type, obj)}") + @dsp def cast_to_python(obj: ast.Ast): inst = cast_to_instance(obj) return cast_to_python(inst) + @dsp def cast_to_python(obj: objects.AbsInstance): # if state.access_level <= state.AccessLevels.QUERY: @@ -1084,18 +1270,20 @@ def cast_to_python(obj: objects.AbsInstance): return res - ### Added functions + def function_localize_keys(self, struct): return cast_to_python(struct) + objects.Function._localize_keys = function_localize_keys def instance_repr(self): return pql_repr(self.type, self.localize()) + objects.Instance.repr = instance_repr diff --git a/preql/core/exceptions.py b/preql/core/exceptions.py index f1af3de..e6da065 100644 --- a/preql/core/exceptions.py +++ b/preql/core/exceptions.py @@ -1,17 +1,17 @@ -from typing import Optional, List +from typing import List, Optional from lark.exceptions import GrammarError -from preql.utils import dataclass, TextReference from preql.context import context +from preql.utils import TextReference, dataclass from .base import Object @dataclass class Signal(Object, Exception): - type: object # Type - text_refs: List[Optional[TextReference]] # XXX must it be optional? + type: object # Type + text_refs: List[Optional[TextReference]] # XXX must it be optional? message: Optional[str] orig_exc: Optional[Exception] = None @@ -19,7 +19,7 @@ class Signal(Object, Exception): def make(cls, type, ast, message): ast_ref = getattr(ast, 'text_ref', None) try: - refs = context.state.stacktrace+([ast_ref] if ast_ref else []) + refs = context.state.stacktrace + ([ast_ref] if ast_ref else []) except AttributeError: refs = [] return cls(type, refs, message) @@ -50,10 +50,12 @@ class pql_SyntaxError(GrammarError): text_ref: TextReference message: str + @dataclass class ExitInterp(Exception): value: object + @dataclass class ReturnSignal(Exception): value: object @@ -63,7 +65,10 @@ class ReturnSignal(Exception): class pql_AttributeError(Exception): message: str + class InsufficientAccessLevel(Exception): pass + + class DatabaseQueryError(Exception): pass diff --git a/preql/core/interp_common.py b/preql/core/interp_common.py index 34dfa83..8135762 100644 --- a/preql/core/interp_common.py +++ b/preql/core/interp_common.py @@ -1,29 +1,31 @@ from logging import getLogger -from preql.utils import dsp from preql.context import context +from preql.utils import dsp from . import pql_ast as ast from . import pql_objects as objects from .exceptions import Signal -from .pql_types import Type, T - +from .pql_types import T, Type logger = getLogger('interp') # Define common dispatch functions + @dsp -def evaluate( obj: type(NotImplemented)) -> object: +def evaluate(obj: type(NotImplemented)) -> object: raise NotImplementedError() + @dsp def cast_to_python(obj: type(NotImplemented)) -> object: raise NotImplementedError(obj) - -def assert_type(t, type_, ast_node, op, msg="%s expected an object of type %s, instead got '%s'"): +def assert_type( + t, type_, ast_node, op, msg="%s expected an object of type %s, instead got '%s'" +): assert isinstance(t, Type), t assert isinstance(type_, Type) if not t <= type_: @@ -33,18 +35,26 @@ def assert_type(t, type_, ast_node, op, msg="%s expected an object of type %s, i type_str = "'%s'" % type_ raise Signal.make(T.TypeError, ast_node, msg % (op, type_str, t)) + def exclude_fields(table, fields): - proj = ast.Projection(table, [ast.NamedField(None, ast.Ellipsis(None, exclude=list(fields) ), user_defined=False)]) + proj = ast.Projection( + table, + [ + ast.NamedField( + None, ast.Ellipsis(None, exclude=list(fields)), user_defined=False + ) + ], + ) return evaluate(proj) + def call_builtin_func(name, args): "Call a builtin pql function" builtins = context.state.ns.get_var('__builtins__') assert isinstance(builtins, objects.Module) expr = ast.FuncCall(builtins.namespace[name], args) - return evaluate( expr) - + return evaluate(expr) def is_global_scope(state): @@ -57,17 +67,19 @@ def is_global_scope(state): # assert isinstance(res, (int, str, float, dict, list, type(None), datetime)), (res, type(res)) # return res + def cast_to_python_string(obj: objects.AbsInstance): res = cast_to_python(obj) if not isinstance(res, str): raise Signal.make(T.TypeError, obj, f"Expected string, got '{res}'") return res + def cast_to_python_int(obj: objects.AbsInstance): res = cast_to_python(obj) if not isinstance(res, int): raise Signal.make(T.TypeError, obj, f"Expected int, got '{res}'") return res - + pyvalue_inst = objects.pyvalue_inst diff --git a/preql/core/interpreter.py b/preql/core/interpreter.py index f16d30e..85f89d3 100644 --- a/preql/core/interpreter.py +++ b/preql/core/interpreter.py @@ -1,26 +1,25 @@ -from copy import copy import threading -from pathlib import Path +from copy import copy from functools import wraps +from pathlib import Path -from preql.utils import classify from preql.context import context +from preql.utils import classify -from .exceptions import Signal, pql_SyntaxError, ReturnSignal -from .evaluate import execute, eval_func_call, import_module, evaluate, cast_to_python -from .parser import parse_stmts from . import pql_ast as ast from . import pql_objects as objects -from .interp_common import pyvalue_inst, call_builtin_func +from .evaluate import cast_to_python, eval_func_call, evaluate, execute, import_module +from .exceptions import ReturnSignal, Signal, pql_SyntaxError +from .interp_common import call_builtin_func, pyvalue_inst +from .parser import parse_stmts +from .pql_functions import import_pandas, internal_funcs, joins +from .pql_types import Object, T from .state import ThreadState -from .pql_types import T, Object -from .pql_functions import import_pandas -from .pql_functions import internal_funcs, joins def initial_namespace(): # TODO localinstance / metainstance - ns = {k:v for k, v in T.items()} + ns = {k: v for k, v in T.items()} ns.update(internal_funcs) ns.update(joins) # TODO all exceptions @@ -34,6 +33,7 @@ def entrypoint(f): def inner(interp, *args, **kwargs): with interp.setup_context(): return f(interp, *args, **kwargs) + return inner @@ -50,9 +50,13 @@ def __getattr__(self, attr): class Interpreter: def __init__(self, sqlengine, display, use_core=True): - self.state = ThreadState.from_components(self, sqlengine, display, initial_namespace()) + self.state = ThreadState.from_components( + self, sqlengine, display, initial_namespace() + ) if use_core: - mns = import_module(self.state, ast.Import('__builtins__', use_core=False)).namespace + mns = import_module( + self.state, ast.Import('__builtins__', use_core=False) + ).namespace bns = self.state.get_var('__builtins__').namespace # safe-update for k, v in mns.items(): @@ -72,7 +76,6 @@ def _execute_code(self, code, source_file, args=None): except pql_SyntaxError as e: raise Signal(T.SyntaxError, [e.text_ref], e.message) - if stmts: if isinstance(stmts[0], ast.Const) and stmts[0].type == T.string: self.set_var('__doc__', stmts[0].value) @@ -80,7 +83,7 @@ def _execute_code(self, code, source_file, args=None): last = None # with self.state.ns.use_parameters(args or {}): - with context(parameters=args or {}): # Set parameters for Namespace.get_var() + with context(parameters=args or {}): # Set parameters for Namespace.get_var() for stmt in stmts: try: last = execute(stmt) @@ -111,14 +114,11 @@ def has_var(self, name): return False return True - - ##################### execute_code = entrypoint(_execute_code) include = entrypoint(_include) - @entrypoint def evaluate_obj(self, obj): return evaluate(obj) @@ -130,7 +130,7 @@ def localize_obj(self, obj): @entrypoint def call_func(self, fname, args, kw=None): if kw: - args = args + [ast.NamedField(k, v, False) for k,v in kw.items()] + args = args + [ast.NamedField(k, v, False) for k, v in kw.items()] res = eval_func_call(context.state.get_var(fname), args) return evaluate(res) @@ -150,7 +150,6 @@ def import_pandas(self, dfs): def list_tables(self): return self.state.db.list_tables() - @entrypoint def load_all_tables(self): modules = {} @@ -168,7 +167,6 @@ def get_module(name): modules[name] = module self.set_var(name, module) - table_types = self.state.db.import_table_types() table_types_by_schema = classify(table_types, lambda x: x[0], lambda x: x[1:]) @@ -183,9 +181,8 @@ def get_module(name): if not self.has_var(table_name): self.set_var(table_name, inst) - def clone(self, use_core): state = self.state i = Interpreter(state.db, state.display, use_core=use_core) - i.state.stacktrace = state.stacktrace # XXX proper interface - return i \ No newline at end of file + i.state.stacktrace = state.stacktrace # XXX proper interface + return i diff --git a/preql/core/parser.py b/preql/core/parser.py index b188858..a9b0716 100644 --- a/preql/core/parser.py +++ b/preql/core/parser.py @@ -1,15 +1,15 @@ from ast import literal_eval from pathlib import Path -from lark import Lark, Transformer, v_args, UnexpectedInput, UnexpectedToken +from lark import Lark, Transformer, UnexpectedInput, UnexpectedToken, v_args from preql.utils import TextPos, TextRange, TextReference -from .exceptions import pql_SyntaxError from . import pql_ast as ast from . import pql_objects as objects from .compiler import guess_field_name -from .pql_types import T, Id +from .exceptions import pql_SyntaxError +from .pql_types import Id, T class Str(str): @@ -34,7 +34,7 @@ def make_text_reference(text, source_file, meta, children=()): meta.end_pos or meta.start_pos, meta.end_line or meta.line, meta.end_column or meta.column, - ) + ), ) for c in children: @@ -87,11 +87,13 @@ def _wrap_result(res, f, meta, children): res.set_text_ref(ref) return res + def _args_wrapper(f, _data, children, meta): "Create meta with 'code' from transformer" res = f(*children) return _wrap_result(res, f, meta, children) + def _args_wrapper_meta(f, _data, children, meta): ref = make_text_reference(*f.__self__.code_ref, meta, children) res = f(ref, *children) @@ -99,13 +101,16 @@ def _args_wrapper_meta(f, _data, children, meta): res.set_text_ref(ref) return res + def _args_wrapper_list(f, _data, children, meta): res = f(children) return _wrap_result(res, f, meta, children) + with_meta = v_args(wrapper=_args_wrapper_meta) no_inline = v_args(wrapper=_args_wrapper_list) + def token_value(self, t): return Str(str(t)) @@ -124,6 +129,7 @@ def QUOTED_NAME(self, name): def STRING(self, s): return _fix_escaping(s[1:-1]) + def LONG_STRING(self, s): return _fix_escaping(s[3:-3]) @@ -136,7 +142,9 @@ def pql_dict(self, meta, items): for item in items: name = item.name or guess_field_name(item.value) if name in d: - raise pql_SyntaxError(meta, f"Dict key appearing more than once: {name}") + raise pql_SyntaxError( + meta, f"Dict key appearing more than once: {name}" + ) d[name] = item.value return ast.Dict_(d) @@ -149,8 +157,10 @@ def float(self, f): def null(self): return ast.Const(T.nulltype, None) + def false(self): return ast.Const(T.bool, False) + def true(self): return ast.Const(T.bool, True) @@ -165,6 +175,7 @@ def as_list(self, args): # types def typemod(self, *args): return [t.value for t in args] + def type(self, name, mods): # TODO pk return ast.Type(name, '?' in (mods or '')) @@ -174,14 +185,14 @@ def type(self, name, mods): comp_op = token_value def compare(self, a, op, b): - return ast.Compare(op, [a,b]) + return ast.Compare(op, [a, b]) def _arith_expr(self, a, op, b): - return ast.BinOp(op, [a,b]) + return ast.BinOp(op, [a, b]) add_expr = _arith_expr term = _arith_expr - power = lambda self, a,b: self._arith_expr(a, '**', b) + power = lambda self, a, b: self._arith_expr(a, '**', b) def like(self, string, pattern): return ast.BinOp('like', [string, pattern]) @@ -233,25 +244,28 @@ def func_def(self, meta, name, params, expr): collector = None for i, p in enumerate(params): if isinstance(p, objects.ParamVariadic): - if i != len(params)-1: + if i != len(params) - 1: msg = f"A variadic parameter may only appear at the end of the function ({p.name})" raise pql_SyntaxError(meta, msg) collector = p params = params[:-1] - docstring = get_docstring(expr) - return ast.FuncDef(objects.UserFunction(name, params, expr, collector, docstring=docstring)) + return ast.FuncDef( + objects.UserFunction(name, params, expr, collector, docstring=docstring) + ) @with_meta def func_call(self, meta, func, args): for i, a in enumerate(args): if isinstance(a, ast.Ellipsis): - if i != len(args)-1: - raise pql_SyntaxError(meta, f"An inlined struct must appear at the end of the function call ({a})") - + if i != len(args) - 1: + raise pql_SyntaxError( + meta, + f"An inlined struct must appear at the end of the function call ({a})", + ) return ast.FuncCall(func, args) @@ -291,7 +305,6 @@ def table_def_from_expr(self, const, name, table_expr): def ellipsis(self, from_struct, *exclude): return ast.Ellipsis(from_struct, list(exclude)) - def __default__(self, data, children, meta): raise Exception("Unknown rule:", data) @@ -320,11 +333,15 @@ def always_accept(self): # Changing these terminals in the grammar will prevent collision detection # Waiting on interregular! from lark.lexer import PatternRE + _operators = ['IN', 'NOT_IN', 'AND', 'OR'] + + def _edit_terminals(t): if t.name in _operators: t.pattern = PatternRE('%s(?!\w)' % t.pattern.value) + parser = Lark.open( 'preql.lark', rel_to=__file__, @@ -346,24 +363,28 @@ def terminal_desc(name): return p.value return '<%s>' % name + def terminal_list_desc(term_list): return [terminal_desc(x) for x in term_list if x != 'MARKER'] + def parse_stmts(s, source_file, wrap_syntax_error=True): try: - tree = parser.parse(s+"\n", start="module") + tree = parser.parse(s + "\n", start="module") except UnexpectedInput as e: if not wrap_syntax_error: raise assert isinstance(source_file, (str, Path)), source_file - pos = TextPos(e.pos_in_stream, e.line, e.column) + pos = TextPos(e.pos_in_stream, e.line, e.column) ref = TextReference(s, str(source_file), TextRange(pos, pos)) if isinstance(e, UnexpectedToken): if e.token.type == '$END': msg = "Code ended unexpectedly" - ref = TextReference(s, str(source_file), TextRange(pos, TextPos(len(s), -1 ,-1))) + ref = TextReference( + s, str(source_file), TextRange(pos, TextPos(len(s), -1, -1)) + ) else: msg = "Unexpected token: %r" % e.token.value diff --git a/preql/core/pql_ast.py b/preql/core/pql_ast.py index 294322e..35c29ab 100644 --- a/preql/core/pql_ast.py +++ b/preql/core/pql_ast.py @@ -1,17 +1,17 @@ -from typing import List, Any, Optional, Dict, Union from dataclasses import field +from typing import Any, Dict, List, Optional, Union -from preql.utils import dataclass, TextReference +from preql.utils import TextReference, dataclass from . import pql_types as types -from .pql_types import T, Object, Id +from .pql_types import Id, Object, T from .types_impl import pql_repr - # TODO We want Ast to typecheck, but sometimes types are still unknown (i.e. at parse time). # * Use incremental type checks? # * Use two tiers of Ast? + @dataclass class Ast(Object): text_ref: Optional[TextReference] = field(init=False, default=None) @@ -20,14 +20,18 @@ def set_text_ref(self, text_ref): object.__setattr__(self, 'text_ref', text_ref) return self + class Expr(Ast): _args = () + @dataclass class Marker(Expr): pass -class Statement(Ast): pass + +class Statement(Ast): + pass @dataclass @@ -38,12 +42,14 @@ class Name(Expr): def __repr__(self): return f'Name({self.name})' + @dataclass class Parameter(Expr): "A typed object without a value" name: str type: types.Type + @dataclass class ResolveParameters(Expr): obj: Object @@ -61,10 +67,11 @@ class ParameterizedSqlCode(Expr): @dataclass class Attr(Expr): "Reference to an attribute (usually a column)" - expr: Optional[Object] #Expr + expr: Optional[Object] # Expr name: Union[str, Marker] - _args = 'expr', + _args = ('expr',) + @dataclass class Const(Expr): @@ -74,16 +81,19 @@ class Const(Expr): def repr(self): return pql_repr(self.type, self.value) + @dataclass class Ellipsis(Expr): from_struct: Optional[Union[Expr, Marker]] exclude: List[Union[str, Marker]] + class BinOpExpr(Expr): - _args = 'args', + _args = ('args',) + class UnaryOpExpr(Expr): - _args = 'expr', + _args = ('expr',) @dataclass @@ -97,32 +107,39 @@ class BinOp(BinOpExpr): op: str args: List[Object] + @dataclass class Or(BinOpExpr): args: List[Object] + @dataclass class And(BinOpExpr): args: List[Object] + @dataclass class Not(UnaryOpExpr): expr: Object + @dataclass class Neg(UnaryOpExpr): expr: Object + @dataclass class Contains(BinOpExpr): op: str args: List[Object] + @dataclass class DescOrder(Expr): value: Object - _args = 'value', + _args = ('value',) + @dataclass class Range(Expr): @@ -131,23 +148,26 @@ class Range(Expr): _args = 'start', 'stop' + @dataclass class NamedField(Expr): name: Optional[str] - value: Object #(Expr, types.PqlType) + value: Object # (Expr, types.PqlType) user_defined: bool = True - _args = 'value', + _args = ('value',) +class TableOperation(Expr): + pass -class TableOperation(Expr): pass @dataclass class Selection(TableOperation): table: Object conds: List[Expr] + @dataclass class Projection(TableOperation): table: Object @@ -161,43 +181,49 @@ def __post_init__(self): else: assert self.fields and not self.agg_fields + @dataclass class Order(TableOperation): table: Object fields: List[Expr] + @dataclass class Update(TableOperation): table: Object fields: List[NamedField] + @dataclass class Delete(TableOperation): table: Object conds: List[Expr] + @dataclass class Slice(Expr): obj: Object range: Range - _args = 'obj', + _args = ('obj',) + @dataclass class New(Expr): type: str - args: list # Func args + args: list # Func args + @dataclass class NewRows(Expr): type: str - args: list # Func args + args: list # Func args + @dataclass class FuncCall(Expr): - func: Any # objects.Function ? - args: list # Func args - + func: Any # objects.Function ? + args: list # Func args @dataclass @@ -211,9 +237,11 @@ class Type(Ast): type_obj: Object nullable: bool = False + class Definition: pass + @dataclass class ColumnDef(Ast, Definition): name: str @@ -221,9 +249,10 @@ class ColumnDef(Ast, Definition): query: Optional[Expr] = None default: Optional[Expr] = None + @dataclass class FuncDef(Statement, Definition): - userfunc: Object # XXX Why not use UserFunction? + userfunc: Object # XXX Why not use UserFunction? @dataclass @@ -245,42 +274,51 @@ class StructDef(Statement, Definition): name: str members: list + @dataclass class SetValue(Statement): name: (Name, Attr) value: Expr + @dataclass class InsertRows(Statement): name: (Name, Attr) value: Expr + @dataclass class Print(Statement): value: List[Object] + @dataclass class Assert(Statement): cond: Object + @dataclass class Return(Statement): value: Object + @dataclass class Throw(Statement): value: Object + @dataclass class Import(Statement): module_path: str as_name: Optional[str] = None use_core: bool = True + @dataclass class CodeBlock(Statement): statements: List[Ast] + @dataclass class Try(Statement): try_: CodeBlock @@ -288,43 +326,47 @@ class Try(Statement): catch_expr: Expr catch_block: CodeBlock + @dataclass class If(Statement): cond: Object then: Statement else_: Optional[Statement] = None + @dataclass class For(Statement): var: str iterable: Object do: CodeBlock + @dataclass class While(Statement): cond: Object do: Statement - # Collections + @dataclass class List_(Expr): type: Object elems: list + @dataclass class Table_Columns(Expr): type: Object cols: Dict[str, list] + @dataclass class Dict_(Expr): elems: dict - def pyvalue(value): "Create an AST node from a Python primitive. For internal use." t = types.from_python(type(value)) @@ -332,4 +374,4 @@ def pyvalue(value): false = Const(T.bool, False) -true = Const(T.bool, True) \ No newline at end of file +true = Const(T.bool, True) diff --git a/preql/core/pql_functions.py b/preql/core/pql_functions.py index ca17264..350e591 100644 --- a/preql/core/pql_functions.py +++ b/preql/core/pql_functions.py @@ -1,38 +1,60 @@ +import csv import inspect -import re +import itertools import os +import re from datetime import datetime -import csv -import itertools import rich.progress import runtype -from preql.utils import safezip, listgen, re_split -from preql.docstring.autodoc import autodoc, AutoDocError from preql.context import context +from preql.docstring.autodoc import AutoDocError, autodoc +from preql.utils import listgen, re_split, safezip -from .exceptions import Signal, ExitInterp -from . import pql_objects as objects from . import pql_ast as ast +from . import pql_objects as objects from . import sql -from .interp_common import pyvalue_inst, assert_type, cast_to_python_string, cast_to_python_int, cast_to_python -from .state import get_var, get_db, use_scope, unique_name, get_db, require_access, AccessLevels, set_var -from .evaluate import evaluate, db_query, TableConstructor, new_table_from_expr, new_table_from_rows -from .pql_types import T, Type, Id -from .types_impl import join_names from .casts import cast from .compiler import cast_to_instance +from .evaluate import ( + TableConstructor, + db_query, + evaluate, + new_table_from_expr, + new_table_from_rows, +) +from .exceptions import ExitInterp, Signal +from .interp_common import ( + assert_type, + cast_to_python, + cast_to_python_int, + cast_to_python_string, + pyvalue_inst, +) +from .pql_types import Id, T, Type +from .state import ( + AccessLevels, + get_db, + get_var, + require_access, + set_var, + unique_name, + use_scope, +) +from .types_impl import join_names + def new_str(x): return pyvalue_inst(str(x), T.string) + def _pql_PY_callback(var: str): var = var.group() assert var[0] == '$' var_name = var[1:] obj = get_var(var_name) - inst = evaluate( obj) + inst = evaluate(obj) if not isinstance(inst, objects.ValueInstance): raise Signal.make(T.TypeError, None, f"Cannot convert {inst} to a Python value") @@ -63,22 +85,29 @@ def pql_PY(code_expr: T.string, code_setup: T.string.as_nullable() = objects.nul try: res = exec(py_setup) except Exception as e: - raise Signal.make(T.EvalError, code_expr, f"Python code provided returned an error: {e}") + raise Signal.make( + T.EvalError, code_expr, f"Python code provided returned an error: {e}" + ) try: res = eval(py_code) except Exception as e: - raise Signal.make(T.EvalError, code_expr, f"Python code provided returned an error: {e}") + raise Signal.make( + T.EvalError, code_expr, f"Python code provided returned an error: {e}" + ) return objects.from_python(res) # return pyvalue_inst(res) -def pql_inspect_sql(obj: T.object): - """Returns the SQL code that would be executed to evaluate the given object - """ +def pql_inspect_sql(obj: T.object): + """Returns the SQL code that would be executed to evaluate the given object""" if not isinstance(obj, objects.Instance): - raise Signal.make(T.TypeError, None, f"inspect_sql() expects a concrete object. Instead got: {obj.type}") + raise Signal.make( + T.TypeError, + None, + f"inspect_sql() expects a concrete object. Instead got: {obj.type}", + ) s = get_db().compile_sql(obj.code, obj.subqueries) return objects.ValueInstance.make(sql.make_value(s), T.text, [], s) @@ -122,12 +151,14 @@ def pql_SQL(result_type: T.union[T.table, T.type], sql_code: T.string): # .. why not just compile with parameters? the types are already known return ast.ParameterizedSqlCode(result_type, sql_code) + def pql_force_eval(expr: T.object): """Forces the evaluation of the given expression. Executes any db queries necessary. """ - return objects.pyvalue_inst( cast_to_python(expr) ) + return objects.pyvalue_inst(cast_to_python(expr)) + def pql_fmt(s: T.string): """Format the given string using interpolation on variables marked as `$var` @@ -164,22 +195,30 @@ def pql_fmt(s: T.string): a = string_parts[0] for b in string_parts[1:]: - a = ast.BinOp("+", [a,b]) + a = ast.BinOp("+", [a, b]) return cast_to_instance(a) - def _canonize_default(d): return None if d is inspect._empty else d + def create_internal_func(fname, f): sig = inspect.signature(f) - return objects.InternalFunction(fname, [ - objects.Param(pname, type_ if isinstance(type_, Type) else T.any, - _canonize_default(sig.parameters[pname].default)) - for pname, type_ in list(f.__annotations__.items()) - ], f) + return objects.InternalFunction( + fname, + [ + objects.Param( + pname, + type_ if isinstance(type_, Type) else T.any, + _canonize_default(sig.parameters[pname].default), + ) + for pname, type_ in list(f.__annotations__.items()) + ], + f, + ) + def create_internal_funcs(d): new_d = {} @@ -190,6 +229,7 @@ def create_internal_funcs(d): new_d[name] = create_internal_func(name, f) return new_d + def create_internal_properties(d): return {k: objects.Property(v) for k, v in create_internal_funcs(d).items()} @@ -207,11 +247,13 @@ def pql_breakpoint(): breakpoint() return objects.null + def pql_set_setting(name: T.string, value: T.any): name = cast_to_python_string(name) value = cast_to_python(value) from preql.settings import Display + setattr(Display, name, value) return objects.null @@ -230,7 +272,6 @@ def pql_debug(): return objects.null - def pql_issubclass(a: T.type, b: T.type): """Checks if type 'a' is a subclass of type 'b' @@ -248,6 +289,7 @@ def pql_issubclass(a: T.type, b: T.type): assert isinstance(b, Type) return pyvalue_inst(a <= b, T.bool) + def pql_isa(obj: T.any, type: T.type): """Checks if the give object is an instance of the given type @@ -265,6 +307,7 @@ def pql_isa(obj: T.any, type: T.type): res = obj.isa(type) return pyvalue_inst(res, T.bool) + def _count(obj, table_func, name='count'): if obj is objects.null: code = sql.FieldFunc(name, sql.AllFields(T.any)) @@ -286,6 +329,7 @@ def _count(obj, table_func, name='count'): return objects.Instance.make(code, T.int, [obj]) + def pql_count(obj: T.container.as_nullable() = objects.null): """Count how many rows are in the given table, or in the projected column. @@ -330,9 +374,11 @@ def pql_temptable(expr: T.table, const: T.bool.as_nullable() = objects.null): const = cast_to_python(const) assert_type(expr.type, T.table, expr, 'temptable') - name = get_db().qualified_name(Id(unique_name("temp"))) # TODO get name from table options + name = get_db().qualified_name( + Id(unique_name("temp")) + ) # TODO get name from table options - with use_scope({'__unwind__': []}): # Disable unwinding + with use_scope({'__unwind__': []}): # Disable unwinding return new_table_from_expr(name, expr, const, True) @@ -347,13 +393,16 @@ def pql_get_db_type(): return pyvalue_inst(get_db().target, T.string) - def sql_bin_op(op, t1, t2, name): if not isinstance(t1, objects.CollectionInstance): - raise Signal.make(T.TypeError, t1, f"First argument isn't a table, it's a {t1.type}") + raise Signal.make( + T.TypeError, t1, f"First argument isn't a table, it's a {t1.type}" + ) if not isinstance(t2, objects.CollectionInstance): - raise Signal.make(T.TypeError, t2, f"Second argument isn't a table, it's a {t2.type}") + raise Signal.make( + T.TypeError, t2, f"Second argument isn't a table, it's a {t2.type}" + ) # TODO Smarter matching? l1 = len(t1.type.elems) @@ -364,12 +413,17 @@ def sql_bin_op(op, t1, t2, name): for e1, e2 in zip(t1.type.elems.values(), t2.type.elems.values()): if not e2 <= e1: - raise Signal.make(T.TypeError, None, f"Cannot {name}. Column types don't match: '{e1}' and '{e2}'") + raise Signal.make( + T.TypeError, + None, + f"Cannot {name}. Column types don't match: '{e1}' and '{e2}'", + ) code = sql.TableArith(op, [t1.code, t2.code]) return type(t1).make(code, t1.type, [t1, t2]) + def pql_table_intersect(t1: T.table, t2: T.table): "Intersect two tables. Used for `t1 & t2`" if get_db().target == sql.bigquery: @@ -378,16 +432,20 @@ def pql_table_intersect(t1: T.table, t2: T.table): op = 'INTERSECT' return sql_bin_op(op, t1, t2, "intersect") + def pql_table_substract(t1: T.table, t2: T.table): "Substract two tables (except). Used for `t1 - t2`" if get_db().target is sql.mysql: - raise Signal.make(T.NotImplementedError, t1, "MySQL doesn't support EXCEPT (yeah, really!)") + raise Signal.make( + T.NotImplementedError, t1, "MySQL doesn't support EXCEPT (yeah, really!)" + ) if get_db().target == sql.bigquery: op = 'EXCEPT DISTINCT' else: op = 'EXCEPT' return sql_bin_op(op, t1, t2, "subtract") + def pql_table_union(t1: T.table, t2: T.table): "Union two tables. Used for `t1 | t2`" if get_db().target == sql.bigquery: @@ -396,6 +454,7 @@ def pql_table_union(t1: T.table, t2: T.table): op = 'UNION' return sql_bin_op(op, t1, t2, "union") + def pql_table_concat(t1: T.table, t2: T.table): "Concatenate two tables (union all). Used for `t1 + t2`" if isinstance(t1, objects.EmptyListInstance): @@ -413,25 +472,35 @@ def _get_table(t): raise Signal.make(T.TypeError, None, f"join() arguments must be tables") return t + def _join2(a, b): - if isinstance(a, objects.SelectedColumnInstance) and isinstance(b, objects.SelectedColumnInstance): + if isinstance(a, objects.SelectedColumnInstance) and isinstance( + b, objects.SelectedColumnInstance + ): return [a, b] if not ((a.type <= T.table) and (b.type <= T.table)): a = a.type.repr() b = b.type.repr() - raise Signal.make(T.TypeError, None, f"join() arguments must be of same type. Instead got:\n * {a}\n * {b}") + raise Signal.make( + T.TypeError, + None, + f"join() arguments must be of same type. Instead got:\n * {a}\n * {b}", + ) return _auto_join(a, b) + def _auto_join(ta, tb): refs1 = _find_table_reference(ta, tb) refs2 = _find_table_reference(tb, ta) auto_join_count = len(refs1) + len(refs2) if auto_join_count < 1: raise NoAutoJoinFound(ta, tb) - elif auto_join_count > 1: # Ambiguity in auto join resolution - raise Signal.make(T.JoinError, None, "Cannot auto-join: Several plausible relations found") + elif auto_join_count > 1: # Ambiguity in auto join resolution + raise Signal.make( + T.JoinError, None, "Cannot auto-join: Several plausible relations found" + ) if len(refs1) == 1: dst, src = refs1[0] @@ -445,11 +514,14 @@ def _auto_join(ta, tb): def _join(join: str, on, exprs_dict: dict, joinall=False, nullable=None): names = list(exprs_dict) - exprs = [evaluate( value) for value in exprs_dict.values()] + exprs = [evaluate(value) for value in exprs_dict.values()] if len(exprs_dict.attrs) < 2: - raise Signal.make(T.ValueError, None, f"Need at least two tables to join. Got {len(exprs_dict.attrs)}") - + raise Signal.make( + T.ValueError, + None, + f"Need at least two tables to join. Got {len(exprs_dict.attrs)}", + ) # Validation and edge cases for x in exprs: @@ -468,28 +540,39 @@ def _join(join: str, on, exprs_dict: dict, joinall=False, nullable=None): tables = [_get_table(x) for x in exprs] assert all((t.type <= T.table) for t in tables) - structs = {name: T.struct(table.type.elems) for name, table in safezip(names, tables)} + structs = { + name: T.struct(table.type.elems) for name, table in safezip(names, tables) + } if nullable: # Update nullable for left/right/outer joins - structs = {name: t.as_nullable() if n else t - for (name, t), n in safezip(structs.items(), nullable)} + structs = { + name: t.as_nullable() if n else t + for (name, t), n in safezip(structs.items(), nullable) + } tables = [objects.alias_table_columns(t, n) for n, t in safezip(names, tables)] - primary_keys = [[name] + pk - for name, t in safezip(names, tables) - for pk in t.type.options.get('pk', []) - ] - table_type = T.table(structs, name=Id(unique_name("joinall" if joinall else "join")), pk=primary_keys) + primary_keys = [ + [name] + pk + for name, t in safezip(names, tables) + for pk in t.type.options.get('pk', []) + ] + table_type = T.table( + structs, name=Id(unique_name("joinall" if joinall else "join")), pk=primary_keys + ) conds = [] if joinall: for e in exprs: if not isinstance(e, objects.CollectionInstance): - raise Signal.make(T.TypeError, None, f"joinall() expected tables. Got {e}") + raise Signal.make( + T.TypeError, None, f"joinall() expected tables. Got {e}" + ) elif on is not objects.null: - attrs = {n: objects.make_instance_from_name(table_type.elems[n], n) for n in names} + attrs = { + n: objects.make_instance_from_name(table_type.elems[n], n) for n in names + } with use_scope({n: objects.projected(c) for n, c in attrs.items()}): on = cast_to_instance(on) conds.append(on.code) @@ -501,7 +584,13 @@ def _join(join: str, on, exprs_dict: dict, joinall=False, nullable=None): for (na, ta), (nb, tb) in itertools.combinations(safezip(names, exprs), 2): try: cols = _join2(ta, tb) - cond = sql.Compare('=', [sql.Name(c.type, join_names((n, c.name))) for n, c in safezip([na, nb], cols)]) + cond = sql.Compare( + '=', + [ + sql.Name(c.type, join_names((n, c.name))) + for n, c in safezip([na, nb], cols) + ], + ) conds.append(cond) joined_exprs |= {id(ta), id(tb)} except NoAutoJoinFound as e: @@ -510,8 +599,11 @@ def _join(join: str, on, exprs_dict: dict, joinall=False, nullable=None): if {id(e) for e in exprs} != set(joined_exprs): # TODO better error!!! table name?? specific failed auto-join? s = ', '.join(repr(t.type) for t in exprs) - raise Signal.make(T.JoinError, None, f"Cannot auto-join: No plausible relations found between {s}") - + raise Signal.make( + T.JoinError, + None, + f"Cannot auto-join: No plausible relations found between {s}", + ) code = sql.Join(table_type, join, [t.code for t in tables], conds) return objects.TableInstance.make(code, table_type, exprs) @@ -567,6 +659,7 @@ def pql_join(on, tables): """ return _join("JOIN", on, tables) + def pql_leftjoin(on, tables): """Left-join any number of tables @@ -574,6 +667,7 @@ def pql_leftjoin(on, tables): """ return _join("LEFT JOIN", on, tables, nullable=[False, True]) + def pql_outerjoin(on, tables): """Outer-join any number of tables @@ -581,6 +675,7 @@ def pql_outerjoin(on, tables): """ return _join("FULL OUTER JOIN", on, tables, nullable=[False, True]) + def pql_joinall(on, tables): """Cartesian product of any number of tables @@ -600,6 +695,7 @@ def pql_joinall(on, tables): """ return _join("JOIN", on, tables, True) + class NoAutoJoinFound(Exception): pass @@ -609,8 +705,11 @@ def _find_table_reference(t1, t2): for name, c in t1.type.elems.items(): if c <= T.t_relation: rel = c.options['rel'] - if rel['table'] == t2.type: # if same table - yield t2.get_attr(rel['column']), objects.SelectedColumnInstance(t1, c, name) + if rel['table'] == t2.type: # if same table + yield t2.get_attr(rel['column']), objects.SelectedColumnInstance( + t1, c, name + ) + def pql_type(obj: T.any): """Returns the type of the given object @@ -627,10 +726,11 @@ def pql_type(obj: T.any): def pql_repr(obj: T.any): - """Returns the representation text of the given object - """ + """Returns the representation text of the given object""" if obj.type <= T.projected | T.aggregated: - raise Signal.make(T.CompileError, obj, "repr() cannot run in projected/aggregated mode") + raise Signal.make( + T.CompileError, obj, "repr() cannot run in projected/aggregated mode" + ) try: return pyvalue_inst(obj.repr()) @@ -638,6 +738,7 @@ def pql_repr(obj: T.any): value = repr(cast_to_python(obj)) return pyvalue_inst(value) + def pql_columns(obj: T.container): """Returns a dictionary `{column_name: column_type}` for the given table @@ -647,8 +748,8 @@ def pql_columns(obj: T.container): """ elems = obj.type.elems - if isinstance(elems, tuple): # Create a tuple/list instead of dict? - elems = {f't{i}':e for i, e in enumerate(elems)} + if isinstance(elems, tuple): # Create a tuple/list instead of dict? + elems = {f't{i}': e for i, e in enumerate(elems)} return ast.Dict_(elems) @@ -666,7 +767,9 @@ def pql_cast(obj: T.any, target_type: T.type): """ type_ = target_type if not isinstance(type_, Type): - raise Signal.make(T.TypeError, type_, f"Cast expected a type, got {type_} instead.") + raise Signal.make( + T.TypeError, type_, f"Cast expected a type, got {type_} instead." + ) if obj.type is type_: return obj @@ -674,7 +777,9 @@ def pql_cast(obj: T.any, target_type: T.type): return cast(obj, type_) -def pql_import_table(name: T.string, columns: T.list[T.string].as_nullable() = objects.null): +def pql_import_table( + name: T.string, columns: T.list[T.string].as_nullable() = objects.null +): """Import an existing table from the database, and fill in the types automatically. Parameters: @@ -699,8 +804,9 @@ def pql_import_table(name: T.string, columns: T.list[T.string].as_nullable() = o return objects.new_table(t, select_fields=bool(columns_whitelist)) - -def pql_connect(uri: T.string, load_all_tables: T.bool = ast.false, auto_create: T.bool = ast.false): +def pql_connect( + uri: T.string, load_all_tables: T.bool = ast.false, auto_create: T.bool = ast.false +): """Connect to a new database, specified by the uri Parameters: @@ -717,12 +823,12 @@ def pql_connect(uri: T.string, load_all_tables: T.bool = ast.false, auto_create: auto_create = cast_to_python(auto_create) state.connect(uri, auto_create=auto_create) if load_all_tables: - state.interp.load_all_tables() # XXX + state.interp.load_all_tables() # XXX return objects.null + def pql_help(inst: T.any = objects.null): - """Provides a brief summary for the given object - """ + """Provides a brief summary for the given object""" if inst is objects.null: text = ( "To see the list of functions and objects available in the namespace, type '[b]names()[/b]'\n" @@ -734,7 +840,6 @@ def pql_help(inst: T.any = objects.null): ) return pyvalue_inst(text, T.string).replace(type=T._rich) - lines = [] table_type = None @@ -747,7 +852,7 @@ def pql_help(inst: T.any = objects.null): inst = T.table try: - doc = autodoc(inst).print_text() # TODO maybe html + doc = autodoc(inst).print_text() # TODO maybe html if doc: lines += [doc] except NotImplementedError: @@ -760,17 +865,17 @@ def pql_help(inst: T.any = objects.null): inst_repr = inst.rich_repr() lines += [f""] - - text = '\n'.join(lines) + '\n' return pyvalue_inst(text).replace(type=T._rich) + def _get_doc(v): s = '' if v.type <= T.function and v.docstring: s = v.docstring.splitlines()[0] return new_str(s).code + def pql_names(obj: T.any = objects.null): """List all names in the namespace of the given object. @@ -778,14 +883,19 @@ def pql_names(obj: T.any = objects.null): """ state = context.state if obj is objects.null: - all_vars = (state.ns.get_all_vars()) + all_vars = state.ns.get_all_vars() else: all_vars = obj.all_attrs() assert all(isinstance(s, str) for s in all_vars) all_vars = list(all_vars.items()) all_vars.sort() - tuples = [sql.Tuple(T.list[T.string], [new_str(n).code,new_str(v.type).code, _get_doc(v)]) for n,v in all_vars] + tuples = [ + sql.Tuple( + T.list[T.string], [new_str(n).code, new_str(v.type).code, _get_doc(v)] + ) + for n, v in all_vars + ] table_type = T.table(dict(name=T.string, type=T.string, doc=T.string)) return objects.new_const_table(table_type, tuples) @@ -799,7 +909,10 @@ def pql_tables(): db = get_db() names = db.list_tables() values = [(name, db.import_table_type(name, None)) for name in names] - tuples = [sql.Tuple(T.list[T.string], [new_str('.'.join(n.parts)).code,new_str(t).code]) for n,t in values] + tuples = [ + sql.Tuple(T.list[T.string], [new_str('.'.join(n.parts)).code, new_str(t).code]) + for n, t in values + ] table_type = T.table(dict(name=T.string, type=T.string)) return objects.new_const_table(table_type, tuples) @@ -810,15 +923,16 @@ def pql_env_vars(): The resulting table has two columns: name, and value. """ - tuples = [sql.Tuple(T.list[T.string], [new_str(n).code,new_str(t).code]) for n,t in os.environ.items()] + tuples = [ + sql.Tuple(T.list[T.string], [new_str(n).code, new_str(t).code]) + for n, t in os.environ.items() + ] - table_type = T.table({'name':T.string, 'value': T.string}) + table_type = T.table({'name': T.string, 'value': T.string}) return objects.new_const_table(table_type, tuples) -breakpoint_funcs = create_internal_funcs({ - ('c', 'continue'): pql_brk_continue -}) +breakpoint_funcs = create_internal_funcs({('c', 'continue'): pql_brk_continue}) def pql_exit(value: T.any.as_nullable() = None): @@ -833,11 +947,10 @@ def pql_exit(value: T.any.as_nullable() = None): raise ExitInterp(value) - def import_pandas(dfs): - """Import pandas.DataFrame instances into SQL tables - """ + """Import pandas.DataFrame instances into SQL tables""" import pandas as pd + def normalize_item(i): if pd.isna(i): return None @@ -847,18 +960,21 @@ def normalize_item(i): for name, df in dfs.items(): if isinstance(df, pd.Series): cols = ['key', 'value'] - rows = [(dt.to_pydatetime() if isinstance(dt, datetime) else dt,v) for dt, v in df.items()] + rows = [ + (dt.to_pydatetime() if isinstance(dt, datetime) else dt, v) + for dt, v in df.items() + ] else: assert isinstance(df, pd.DataFrame) cols = list(df) - rows = [[normalize_item(i) for i in rec] - for rec in df.to_records()] - rows = [ row[1:] for row in rows ] # drop index + rows = [[normalize_item(i) for i in rec] for rec in df.to_records()] + rows = [row[1:] for row in rows] # drop index tbl = new_table_from_rows(name, cols, rows, temporary=True) set_var(name, tbl) yield tbl + def pql_import_json(table_name: T.string, uri: T.string): """Imports a json file into a new table. @@ -876,17 +992,19 @@ def pql_import_json(table_name: T.string, uri: T.string): print(f"Importing JSON file: '{uri}'") import pandas + try: df = pandas.read_json(uri) except ValueError as e: raise Signal.make(T.ValueError, uri, f'Pandas error: {e}') - tbl ,= import_pandas({table_name: df}) + (tbl,) = import_pandas({table_name: df}) return tbl - -def pql_import_csv(table: T.table, filename: T.string, header: T.bool = ast.Const(T.bool, False)): +def pql_import_csv( + table: T.table, filename: T.string, header: T.bool = ast.Const(T.bool, False) +): """Import a csv file into an existing table Parameters: @@ -910,14 +1028,15 @@ def insert_values(): q = sql.InsertConsts2(table.type.options['name'], keys, rows) db_query(q) - try: with open(filename, 'r', encoding='utf8') as f: line_count = len(list(f)) f.seek(0) reader = csv.reader(f) - for i, row in enumerate(rich.progress.track(reader, total=line_count, description=msg)): + for i, row in enumerate( + rich.progress.track(reader, total=line_count, description=msg) + ): if i == 0: matched = cons.match_params(row) keys = [p.name for (p, _) in matched] @@ -929,7 +1048,7 @@ def insert_values(): values = ["'%s'" % (v.replace("'", "''")) for v in row] rows.append(values) - if (i+1) % ROWS_PER_QUERY == 0: + if (i + 1) % ROWS_PER_QUERY == 0: insert_values() rows = [] @@ -943,26 +1062,33 @@ def insert_values(): def _rest_func_endpoint(func): from starlette.responses import JSONResponse + async def callback(request): params = [objects.pyvalue_inst(v) for k, v in request.path_params.items()] expr = ast.FuncCall(func, params) - res = evaluate( expr) + res = evaluate(expr) res = cast_to_python(res) return JSONResponse(res) + return callback + def _rest_table_endpoint(table): from starlette.responses import JSONResponse + async def callback(request): tbl = table params = dict(request.query_params) if params: - conds = [ast.Compare('=', [ast.Name(k), objects.pyvalue_inst(v)]) - for k, v in params.items()] + conds = [ + ast.Compare('=', [ast.Name(k), objects.pyvalue_inst(v)]) + for k, v in params.items() + ] expr = ast.Selection(tbl, conds) - tbl = evaluate( expr) + tbl = evaluate(expr) res = cast_to_python(tbl) return JSONResponse(res) + return callback @@ -990,21 +1116,23 @@ def pql_serve_rest(endpoints: T.struct, port: T.int = pyvalue_inst(8080)): from starlette.responses import JSONResponse from starlette.routing import Route except ImportError: - raise Signal.make(T.ImportError, None, "starlette not installed! Run 'pip install starlette'") + raise Signal.make( + T.ImportError, None, "starlette not installed! Run 'pip install starlette'" + ) try: import uvicorn except ImportError: - raise Signal.make(T.ImportError, None, "uvicorn not installed! Run 'pip install uvicorn'") + raise Signal.make( + T.ImportError, None, "uvicorn not installed! Run 'pip install uvicorn'" + ) port_ = cast_to_python_int(port) async def root(_request): return JSONResponse(list(endpoints.attrs)) - routes = [ - Route("/", endpoint=root) - ] + routes = [Route("/", endpoint=root)] for func_name, func in endpoints.attrs.items(): path = "/" + func_name @@ -1016,7 +1144,9 @@ async def root(_request): elif func.type <= T.table: routes.append(Route(path, endpoint=_rest_table_endpoint(func))) else: - raise Signal.make(T.TypeError, func, f"Expected a function or a table, got {func.type}") + raise Signal.make( + T.TypeError, func, f"Expected a function or a table, got {func.type}" + ) app = Starlette(debug=True, routes=routes) @@ -1031,7 +1161,7 @@ def pql_table_add_index(table, column_name: T.string, unique: T.bool = ast.false Parameters: column_name: The name of the column to add index - unique: If true, every value in the column is expected to be unique + unique: If true, every value in the column is expected to be unique Note: Future versions of this function will accept several columns. @@ -1043,11 +1173,13 @@ def pql_table_add_index(table, column_name: T.string, unique: T.bool = ast.false try: table_name = table.type.options['name'] except KeyError: - raise Signal.make(T.TypeError, None, f"Can only add indexes to persistent tables") + raise Signal.make( + T.TypeError, None, f"Can only add indexes to persistent tables" + ) unique = cast_to_python(unique) column_name = cast_to_python(column_name) - + index_name = unique_name(f'index_{table_name.repr_name}_{column_name}') code = sql.AddIndex(Id(index_name), table_name, column_name, unique=unique) @@ -1055,57 +1187,68 @@ def pql_table_add_index(table, column_name: T.string, unique: T.bool = ast.false return objects.null -T.table.proto_attrs.update(create_internal_funcs({ - 'add_index': pql_table_add_index -})) + +T.table.proto_attrs.update(create_internal_funcs({'add_index': pql_table_add_index})) def _make_datetime_method(func_name): def f(datetime): return context.state.interp.call_func(func_name, [datetime]) + return f -_datetime_methods = 'hour', 'minute', 'day', 'month', 'year', 'day_of_week', 'week_of_year' - -T.timestamp.proto_attrs.update(create_internal_properties({ - n: _make_datetime_method(n) for n in _datetime_methods -})) - - -internal_funcs = create_internal_funcs({ - 'exit': pql_exit, - 'help': pql_help, - 'names': pql_names, - 'tables': pql_tables, - 'env_vars': pql_env_vars, - 'dir': pql_names, - 'connect': pql_connect, - 'import_table': pql_import_table, - 'count': pql_count, - 'temptable': pql_temptable, - 'table_concat': pql_table_concat, - 'table_intersect': pql_table_intersect, - 'table_union': pql_table_union, - 'table_subtract': pql_table_substract, - 'SQL': pql_SQL, - 'inspect_sql': pql_inspect_sql, - 'PY': pql_PY, - 'isa': pql_isa, - 'issubclass': pql_issubclass, - 'type': pql_type, - 'repr': pql_repr, - 'debug': pql_debug, - '_breakpoint': pql_breakpoint, - 'get_db_type': pql_get_db_type, - 'cast': pql_cast, - 'columns': pql_columns, - 'import_csv': pql_import_csv, - 'import_json': pql_import_json, - 'serve_rest': pql_serve_rest, - 'force_eval': pql_force_eval, - 'fmt': pql_fmt, - 'set_setting': pql_set_setting, -}) + +_datetime_methods = ( + 'hour', + 'minute', + 'day', + 'month', + 'year', + 'day_of_week', + 'week_of_year', +) + +T.timestamp.proto_attrs.update( + create_internal_properties({n: _make_datetime_method(n) for n in _datetime_methods}) +) + + +internal_funcs = create_internal_funcs( + { + 'exit': pql_exit, + 'help': pql_help, + 'names': pql_names, + 'tables': pql_tables, + 'env_vars': pql_env_vars, + 'dir': pql_names, + 'connect': pql_connect, + 'import_table': pql_import_table, + 'count': pql_count, + 'temptable': pql_temptable, + 'table_concat': pql_table_concat, + 'table_intersect': pql_table_intersect, + 'table_union': pql_table_union, + 'table_subtract': pql_table_substract, + 'SQL': pql_SQL, + 'inspect_sql': pql_inspect_sql, + 'PY': pql_PY, + 'isa': pql_isa, + 'issubclass': pql_issubclass, + 'type': pql_type, + 'repr': pql_repr, + 'debug': pql_debug, + '_breakpoint': pql_breakpoint, + 'get_db_type': pql_get_db_type, + 'cast': pql_cast, + 'columns': pql_columns, + 'import_csv': pql_import_csv, + 'import_json': pql_import_json, + 'serve_rest': pql_serve_rest, + 'force_eval': pql_force_eval, + 'fmt': pql_fmt, + 'set_setting': pql_set_setting, + } +) _joins = { 'join': pql_join, @@ -1115,6 +1258,8 @@ def f(datetime): } joins = { - k: objects.InternalFunction(k, [objects.Param('$on', default=objects.null)], v, objects.Param('tables')) + k: objects.InternalFunction( + k, [objects.Param('$on', default=objects.null)], v, objects.Param('tables') + ) for k, v in _joins.items() } diff --git a/preql/core/pql_objects.py b/preql/core/pql_objects.py index a011240..d2ab0e3 100644 --- a/preql/core/pql_objects.py +++ b/preql/core/pql_objects.py @@ -2,18 +2,16 @@ A collection of objects that may come to interaction with the user. """ -from typing import List, Optional, Callable, Any, Dict +from typing import Any, Callable, Dict, List, Optional -from preql.utils import dataclass, SafeDict, X, listgen from preql import settings +from preql.utils import SafeDict, X, dataclass, listgen -from .exceptions import pql_AttributeError, Signal from . import pql_ast as ast -from . import sql -from . import pql_types +from . import pql_types, sql +from .exceptions import Signal, pql_AttributeError +from .pql_types import Object, T, Type, dp_inst from .state import unique_name - -from .pql_types import T, Type, Object, dp_inst from .types_impl import flatten_type, join_names, pql_repr @@ -23,7 +21,8 @@ class Param(ast.Ast): name: str type: Optional[Object] = None default: Optional[Object] = None - orig: Any = None # XXX temporary and lazy, for TableConstructor + orig: Any = None # XXX temporary and lazy, for TableConstructor + class ParamVariadic(Param): pass @@ -41,7 +40,8 @@ def items(self): @property def type(self): - return tuple((n,p.type) for n,p in self.params.items()) + return tuple((n, p.type) for n, p in self.params.items()) + @dataclass class Module(Object): @@ -65,8 +65,11 @@ def __repr__(self): return f'' def public_functions(self): - funcs = [v for v in self.namespace.values() - if v.type <= T.function and v.docstring and not v.name.startswith('_')] + funcs = [ + v + for v in self.namespace.values() + if v.type <= T.function and v.docstring and not v.name.startswith('_') + ] funcs.sort(key=lambda f: f.name) return funcs @@ -84,7 +87,9 @@ class Function(Object): @property def type(self): - return T.function[tuple(p.type or T.any for p in self.params)](param_collector=self.param_collector is not None) + return T.function[tuple(p.type or T.any for p in self.params)]( + param_collector=self.param_collector is not None + ) def help_str(self): # XXX probably belongs in display.py @@ -116,13 +121,11 @@ def match_params_fast(self, args): msg = f"Function '{self.name}' is missing a value for parameter '{p.name}'" raise Signal.make(T.TypeError, None, msg) - yield p, v if self.param_collector: yield self.param_collector, ast.Dict_({}) - def _localize_keys(self, struct): raise NotImplementedError() @@ -140,15 +143,19 @@ def match_params(self, args): if isinstance(a, ast.NamedField): inline_args.append(a) elif isinstance(a, ast.Ellipsis): - assert i == len(args)-1 + assert i == len(args) - 1 if a.exclude: - raise NotImplementedError("Cannot exclude keys when inlining struct") + raise NotImplementedError( + "Cannot exclude keys when inlining struct" + ) # XXX we only want to localize the keys, not the values # TODO remove this? d = self._localize_keys(a.from_struct) if not isinstance(d, dict): - raise Signal.make(T.TypeError, None, f"Expression to inline is not a map: {d}") + raise Signal.make( + T.TypeError, None, f"Expression to inline is not a map: {d}" + ) for k, v in d.items(): inline_args.append(ast.NamedField(k, pyvalue_inst(v))) else: @@ -185,7 +192,11 @@ def match_params(self, args): arg_name = named_arg.name if arg_name in values: if arg_name in names_set: - raise Signal.make(T.SyntaxError, None, f"Function '{self.name}' recieved argument '{arg_name}' both as keyword and as positional.") + raise Signal.make( + T.SyntaxError, + None, + f"Function '{self.name}' recieved argument '{arg_name}' both as keyword and as positional.", + ) names_set.add(arg_name) values[arg_name] = named_arg.value @@ -194,8 +205,11 @@ def match_params(self, args): collected[arg_name] = named_arg.value else: # TODO meta - raise Signal.make(T.TypeError, None, f"Function '{self.name}' has no parameter named '{arg_name}'") - + raise Signal.make( + T.TypeError, + None, + f"Function '{self.name}' has no parameter named '{arg_name}'", + ) for name, value in values.items(): if value is None: @@ -210,7 +224,6 @@ def match_params(self, args): return matched - @dataclass class UserFunction(Function): name: str @@ -227,12 +240,13 @@ class InternalFunction(Function): func: Callable param_collector: Optional[Param] = None - meta = None # Not defined in PQL code + meta = None # Not defined in PQL code @property def docstring(self): return self.func.__doc__ + @dataclass class Property(Object): func: Function @@ -243,23 +257,26 @@ class Property(Object): # post_instance_getattr. Property handling is specified in evaluate + @dp_inst def post_instance_getattr(inst, obj): return obj + @dp_inst def post_instance_getattr(inst, f: T.function): return MethodInstance(inst, f) - # Instances + class AbsInstance(Object): def get_attr(self, name): v = self.type.get_attr(name) return post_instance_getattr(self, v) + @dataclass class MethodInstance(AbsInstance, Function): parent: AbsInstance @@ -271,6 +288,7 @@ class MethodInstance(AbsInstance, Function): name = property(X.func.name) + @dataclass class PropertyInstance(AbsInstance): parent: AbsInstance @@ -279,12 +297,12 @@ class PropertyInstance(AbsInstance): name = property(X.func.name) type = T.any + @dataclass class ExceptionInstance(AbsInstance): exc: Exception - @dataclass class Instance(AbsInstance): code: sql.Sql @@ -312,7 +330,6 @@ def primary_key(self): return self - def pyvalue_inst(value, type_=None, force_type=False): r = sql.make_value(value) @@ -325,7 +342,7 @@ def pyvalue_inst(value, type_=None, force_type=False): else: type_ = r.type - if settings.optimize: # XXX a little silly? But maybe good for tests + if settings.optimize: # XXX a little silly? But maybe good for tests return ValueInstance.make(r, type_, [], value) return Instance.make(r, type_, []) @@ -345,19 +362,20 @@ def value(self): class CollectionInstance(Instance): pass + @dataclass class TableInstance(CollectionInstance): def __post_init__(self): - assert self.type <= T.table, self.type #and not self.type <= T.list, self.type + assert self.type <= T.table, self.type # and not self.type <= T.list, self.type @property def __columns(self): - return {n:self.get_column(n) for n in self.type.elems.keys()} + return {n: self.get_column(n) for n in self.type.elems.keys()} def get_column(self, name): # TODO memoize? columns shouldn't change t = self.type.elems - return make_instance_from_name(t[name], name) #t.column_codename(name)) + return make_instance_from_name(t[name], name) # t.column_codename(name)) def all_attrs(self): attrs = SafeDict(self.type.proto_attrs) @@ -371,13 +389,18 @@ def get_attr(self, name): return super().get_attr(name) - - def make_instance_from_name(t, cn): if t <= T.struct: - return StructInstance(t, {n: make_instance_from_name(mt, join_names((cn, n))) for n,mt in t.elems.items()}) + return StructInstance( + t, + { + n: make_instance_from_name(mt, join_names((cn, n))) + for n, mt in t.elems.items() + }, + ) return make_instance(sql.Name(t, cn), t, []) + def make_instance(code, t, insts): if t.issubtype(T.struct): raise Signal.make(T.TypeError, t, "Cannot instanciate structs directly") @@ -391,7 +414,6 @@ def make_instance(code, t, insts): return Instance.make(code, t, insts) - class AbsStructInstance(AbsInstance): type: Type attrs: Dict[str, Object] @@ -495,7 +517,7 @@ class UnknownInstance(AbsInstance): code = sql.unknown def get_attr(self, name): - return self # XXX use name? + return self # XXX use name? def all_attrs(self): return {} @@ -510,6 +532,7 @@ def replace(self, **_kw): unknown = UnknownInstance() + @dataclass class SelectedColumnInstance(AbsInstance): parent: CollectionInstance @@ -523,6 +546,7 @@ def subqueries(self): @property def code(self): raise Signal.make(T.TypeError, [], f"Operation not supported for {self}") + # return self._resolve_attr().code def flatten_code(self): @@ -542,7 +566,6 @@ def repr(self): return f'{p}.{self.name}' - def merge_subqueries(instances): return SafeDict().update(*[i.subqueries for i in instances]) @@ -554,9 +577,11 @@ def ensure_phantom_type(inst, ptype): return inst return inst.replace(type=ptype[inst.type]) + def aggregate(inst): return ensure_phantom_type(inst, T.aggregated) + def projected(inst): return ensure_phantom_type(inst, T.projected) @@ -566,6 +591,7 @@ def remove_phantom_type(inst): return inst.replace(type=inst.type.elem) return inst + def inherit_vectorized_type(t, objs): # XXX reevaluate this function for src in objs: @@ -573,6 +599,7 @@ def inherit_vectorized_type(t, objs): return T.projected[t] return t + def inherit_phantom_type(o, objs): for src in objs: if src.type <= T.projected | T.aggregated: @@ -580,16 +607,18 @@ def inherit_phantom_type(o, objs): return o - null = ValueInstance.make(sql.null, T.nulltype, [], None) + @dataclass class EmptyListInstance(TableInstance): - """Special case, because it is untyped - """ + """Special case, because it is untyped""" + _empty_list_type = T.list[T.nulltype] -EmptyList = EmptyListInstance.make(sql.EmptyList(_empty_list_type), _empty_list_type, []) +EmptyList = EmptyListInstance.make( + sql.EmptyList(_empty_list_type), _empty_list_type, [] +) def alias_table_columns(t, prefix): @@ -612,11 +641,14 @@ def new_table(type_, name=None, instances=None, select_fields=False): inst = TableInstance.make(sql.TableName(type_, name), type_, instances or []) if select_fields: - code = sql.Select(type_, inst.code, [sql.Name(t, n) for n, t in type_.elems.items()]) + code = sql.Select( + type_, inst.code, [sql.Name(t, n) for n, t in type_.elems.items()] + ) inst = inst.replace(code=code) return inst + def new_const_table(table_type, tuples): name = unique_name("table_") table_code, subq = sql.create_table(table_type, name, tuples) @@ -626,7 +658,6 @@ def new_const_table(table_type, tuples): return inst - class PythonList(ast.Ast): # TODO just a regular const? def __init__(self, items): @@ -641,7 +672,7 @@ def __init__(self, items): if len(types) > 1: raise ValueError("Expecting all items of the list to be of the same type") # TODO if not one type, raise typeerror - type_ ,= types + (type_,) = types self.type = T.list[pql_types.from_python(type_)] # allow to compile it straight to SQL, no AST in the middle @@ -656,43 +687,57 @@ def from_python(value: type(None)): assert value is None return null + @dsp def from_python(value: str): return ast.Const(T.string, value) + @dsp def from_python(value: bytes): return ast.Const(T.string, value.decode()) + @dsp def from_python(value: bool): return ast.Const(T.bool, value) + @dsp def from_python(value: int): return ast.Const(T.int, value) + @dsp def from_python(value: float): return ast.Const(T.float, value) + @dsp def from_python(value: list): return PythonList(value) + @dsp def from_python(value: dict): - elems = {k:from_python(v) for k,v in value.items()} + elems = {k: from_python(v) for k, v in value.items()} return ast.Dict_(elems) + @dsp def from_python(value: type): return pql_types.from_python(value) + @dsp def from_python(value: Object): - return value + return value + @dsp def from_python(value): - raise Signal.make(T.TypeError, None, f"Cannot import into Preql a Python object of type {type(value)}") + raise Signal.make( + T.TypeError, + None, + f"Cannot import into Preql a Python object of type {type(value)}", + ) diff --git a/preql/core/pql_types.py b/preql/core/pql_types.py index 214cb4a..4eff140 100644 --- a/preql/core/pql_types.py +++ b/preql/core/pql_types.py @@ -1,19 +1,21 @@ +from collections import defaultdict, deque from contextlib import suppress -from typing import Union -from datetime import datetime from dataclasses import field +from datetime import datetime from decimal import Decimal -from collections import defaultdict, deque +from typing import Union import arrow import runtype from runtype.typesystem import TypeSystem from preql.utils import dataclass + from .base import Object global_methods = {} + class Id: def __init__(self, *parts): assert all(isinstance(p, str) for p in parts), parts @@ -28,7 +30,7 @@ def __str__(self): def __hash__(self): return hash(tuple(self.parts)) - + def __eq__(self, other): if not isinstance(other, Id): return NotImplemented @@ -48,13 +50,16 @@ def lower(self): def _repr_type_elem(t, depth): - return _repr_type(t, depth-1) if isinstance(t, Type) else repr(t) + return _repr_type(t, depth - 1) if isinstance(t, Type) else repr(t) + def _repr_type(t, depth=2): if t.elems: if depth > 0: if isinstance(t.elems, dict): - elems = '[%s]' % ', '.join(f'{k}: {_repr_type_elem(v, depth)}' for k,v in t.elems.items()) + elems = '[%s]' % ', '.join( + f'{k}: {_repr_type_elem(v, depth)}' for k, v in t.elems.items() + ) else: elems = '[%s]' % ', '.join(_repr_type_elem(e, depth) for e in t.elems) else: @@ -63,15 +68,19 @@ def _repr_type(t, depth=2): elems = '' return f'{t._typename_with_q}{elems}' + ITEM_NAME = 'item' + @dataclass class Type(Object): typename: str supertypes: frozenset elems: Union[tuple, dict] = field(hash=False, default_factory=dict) options: dict = field(hash=False, compare=False, default_factory=dict) - proto_attrs: dict = field(hash=False, compare=False, default_factory=lambda: dict(global_methods)) + proto_attrs: dict = field( + hash=False, compare=False, default_factory=lambda: dict(global_methods) + ) _nullable: bool = field(default_factory=bool) @property @@ -82,9 +91,9 @@ def _typename_with_q(self): @property def elem(self): if isinstance(self.elems, dict): - elem ,= self.elems.values() + (elem,) = self.elems.values() else: - elem ,= self.elems + (elem,) = self.elems return elem def as_nullable(self): @@ -95,11 +104,7 @@ def maybe_null(self): return self._nullable or self is T.nulltype def supertype_chain(self): - res = { - t2 - for t1 in self.supertypes - for t2 in t1.supertype_chain() - } + res = {t2 for t1 in self.supertypes for t2 in t1.supertype_chain()} assert self not in res return res | {self} @@ -107,7 +112,6 @@ def supertype_chain(self): def __eq__(self, other, memo=None): "Repetitive nested equalities are assumed to be true" - if not isinstance(other, Type): return False @@ -115,23 +119,26 @@ def __eq__(self, other, memo=None): memo = set() a, b = id(self), id(other) - if (a,b) in memo or (b,a) in memo: + if (a, b) in memo or (b, a) in memo: return True memo.add((a, b)) l1 = self.elems if isinstance(self.elems, dict) else dict(enumerate(self.elems)) - l2 = other.elems if isinstance(other.elems, dict) else dict(enumerate(other.elems)) + l2 = ( + other.elems + if isinstance(other.elems, dict) + else dict(enumerate(other.elems)) + ) if len(l1) != len(l2): return False res = self.typename == other.typename and all( - k1==k2 and v1.__eq__(v2, memo) - for (k1,v1), (k2,v2) in zip(l1.items(), l2.items()) + k1 == k2 and v1.__eq__(v2, memo) + for (k1, v1), (k2, v2) in zip(l1.items(), l2.items()) ) return res - @property def elem_types(self): if isinstance(self.elems, dict): @@ -140,7 +147,7 @@ def elem_types(self): def issubtype(self, t): assert isinstance(t, Type), t - if t.typename == 'union': # XXX a little hacky. Change to issupertype? + if t.typename == 'union': # XXX a little hacky. Change to issupertype? return any(self.issubtype(t2) for t2 in t.elem_types) if self is T.nulltype: @@ -149,7 +156,9 @@ def issubtype(self, t): # TODO zip should be aware of lengths if t.typename in (s.typename for s in self.supertype_chain()): - return all(e1.issubtype(e2) for e1, e2 in zip(self.elem_types, t.elem_types)) + return all( + e1.issubtype(e2) for e1, e2 in zip(self.elem_types, t.elem_types) + ) return False def __le__(self, other): @@ -162,7 +171,11 @@ def __getitem__(self, elems): return self.replace(elems=elems) def __call__(self, elems=None, **options): - return self.replace(elems=elems or self.elems, proto_attrs=dict(self.proto_attrs), options={**self.options, **options}) + return self.replace( + elems=elems or self.elems, + proto_attrs=dict(self.proto_attrs), + options={**self.options, **options}, + ) def __repr__(self): # TODO Move to dp_inst? @@ -193,6 +206,7 @@ def repr(self): def __or__(self, other): return T.union[self, other] + class TupleType(Type): def __getitem__(self, elems): assert not self.elems @@ -201,21 +215,23 @@ def __getitem__(self, elems): def __or__(self, other): return self.replace(elems=self.elems + (other,)) + class SumType(TupleType): def issubtype(self, other): return all(t.issubtype(other) for t in self.elem_types) + class ProductType(TupleType): def issubtype(self, other): return all(a.issubtype(b) for a, b in zip(self.elem_types, other.elem_types)) + class PhantomType(Type): def issubtype(self, other): return super().issubtype(other) or self.elem.issubtype(other) class TypeDict(dict): - def _register(self, name, supertypes=(), elems=(), type_class=Type): t = type_class(name, frozenset(supertypes), elems) assert name not in self @@ -229,7 +245,6 @@ def __setattr__(self, name, args): self._register(name, args) - T = TypeDict() T.any = () @@ -252,14 +267,14 @@ def __setattr__(self, name, args): T.number = [T.primitive] T.int = [T.number] T.float = [T.number] -T.bool = [T.primitive] # number? +T.bool = [T.primitive] # number? T.decimal = [T.number] -# TODO datetime vs timestamp ! -T.timestamp = [T.primitive] # struct? -T.datetime = [T.primitive] # struct? -T.date = [T.primitive] # struct? -T.time = [T.primitive] # struct? +# TODO datetime vs timestamp ! +T.timestamp = [T.primitive] # struct? +T.datetime = [T.primitive] # struct? +T.date = [T.primitive] # struct? +T.time = [T.primitive] # struct? T.container = [T.object] @@ -273,7 +288,7 @@ def __setattr__(self, name, args): T.list = [T.table], {ITEM_NAME: T.any} T.set = [T.table], {ITEM_NAME: T.any} T.t_id = [T.primitive], (T.table,) -T.t_relation = [T.primitive], (T.any,) # t_id? +T.t_relation = [T.primitive], (T.any,) # t_id? # XXX sequence instead of container? T._register('aggregated', [T.container], (T.any,), type_class=PhantomType) @@ -290,7 +305,7 @@ def __setattr__(self, name, args): T.module = [T.object] T.signal = [T.object] -#----------- +# ----------- T.Exception = [T.signal] @@ -330,9 +345,10 @@ def _get_subtypes(): d[st].append(t) return dict(d) + subtypes = _get_subtypes() -#------------- +# ------------- _python_type_to_sql_type = { @@ -342,8 +358,10 @@ def _get_subtypes(): str: T.string, datetime: T.timestamp, Decimal: T.decimal, - arrow.Arrow: T.timestamp, # datetime? + arrow.Arrow: T.timestamp, # datetime? } + + def from_python(t): # TODO throw proper exception if this fails return _python_type_to_sql_type[t] @@ -378,7 +396,7 @@ def union_types(types): if len(ts) > 1: elem_type = T.union(elems=tuple(ts)) else: - elem_type ,= ts + (elem_type,) = ts return elem_type diff --git a/preql/core/sql.py b/preql/core/sql.py index 8a3918f..f562c75 100644 --- a/preql/core/sql.py +++ b/preql/core/sql.py @@ -1,12 +1,12 @@ -from typing import List, Optional, Dict +from typing import Dict, List, Optional -from preql.utils import dataclass, X, listgen, safezip +from preql.utils import X, dataclass, listgen, safezip from . import pql_types -from .pql_types import T, Type, dp_type, Id -from .types_impl import join_names, flatten_type -from .state import get_db from .exceptions import Signal +from .pql_types import Id, T, Type, dp_type +from .state import get_db +from .types_impl import flatten_type, join_names duck = 'duck' sqlite = 'sqlite' @@ -14,6 +14,7 @@ bigquery = 'bigquery' mysql = 'mysql' + class QueryBuilder: def __init__(self, is_root=True, start_count=0): self.target = get_db().target @@ -29,20 +30,21 @@ def unique_name(self): def replace(self, is_root): if is_root == self.is_root: - return self # Optimize + return self # Optimize return QueryBuilder(is_root, self.counter) def push_table(self, t): self.table_name.append(t) + def pop_table(self, t): t2 = self.table_name.pop() assert t2 == t - class Sql: pass + @dataclass class SqlTree(Sql): _is_select = False @@ -59,12 +61,18 @@ def compile(self, qb): assert isinstance(sql_code, list), self assert all(isinstance(c, (str, Parameter)) for c in sql_code), self - return CompiledSQL(self.type, sql_code, self, self._is_select, self._needs_select) + return CompiledSQL( + self.type, sql_code, self, self._is_select, self._needs_select + ) def finalize_with_subqueries(self, qb, subqueries): if subqueries: - subqs = [q.compile_wrap(qb).finalize(qb) for (name, q) in subqueries.items()] - sql_code = ['WITH RECURSIVE '] if qb.target in (postgres, mysql) else ['WITH '] + subqs = [ + q.compile_wrap(qb).finalize(qb) for (name, q) in subqueries.items() + ] + sql_code = ( + ['WITH RECURSIVE '] if qb.target in (postgres, mysql) else ['WITH '] + ) sql_code += join_comma([q, '\n '] for q in subqs) else: sql_code = [] @@ -77,13 +85,15 @@ class CompiledSQL(Sql): type: Type code: list source_tree: Optional[Sql] - _is_select: bool # Needed for embedding in SqlTree + _is_select: bool # Needed for embedding in SqlTree _needs_select: bool def finalize(self, qb): wrapped = self.wrap(qb) assert qb.is_root - if wrapped.type <= T.primitive and not wrapped.code[0].lower().startswith('select '): + if wrapped.type <= T.primitive and not wrapped.code[0].lower().startswith( + 'select ' + ): code = ['SELECT '] + wrapped.code else: code = wrapped.code @@ -109,8 +119,10 @@ def wrap(self, qb): def compile_wrap(self, qb): return self.wrap(qb) + def compile(self, qb): return self + def finalize_with_subqueries(self, qb, subqueries): # Why not inherit from Sql? return SqlTree.finalize_with_subqueries(self, qb, subqueries) @@ -142,7 +154,10 @@ def _compile(self, qb): @property def _is_select(self): - return self.text.lstrip().lower().startswith('select') # XXX Hacky! Is there a cleaner solution? + return ( + self.text.lstrip().lower().startswith('select') + ) # XXX Hacky! Is there a cleaner solution? + @dataclass class Null(SqlTree): @@ -151,14 +166,17 @@ class Null(SqlTree): def _compile(self, qb): return ['null'] + @dataclass class Unknown(SqlTree): def _compile(self, qb): raise NotImplementedError("Unknown") + null = Null() unknown = Unknown() + @dataclass class Parameter(SqlTree): type: Type @@ -172,10 +190,12 @@ def _compile(self, qb): class Scalar(SqlTree): pass + @dataclass class Atom(Scalar): pass + @dataclass class Primitive(Atom): type: Type @@ -189,6 +209,7 @@ def _compile(self, qb): class Table(SqlTree): pass + @dataclass class EmptyList(Table): type: Type @@ -198,6 +219,7 @@ class EmptyList(Table): def _compile(self, qb): return ['SELECT NULL AS ITEM LIMIT 0'] + @dataclass class TableName(Table): type: Type @@ -231,6 +253,7 @@ class CountTable(Scalar): def _compile(self, qb): return [f'(SELECT COUNT(*) FROM '] + self.table.compile_wrap(qb).code + [')'] + @dataclass class JsonLength(Scalar): expr: Sql @@ -239,7 +262,13 @@ class JsonLength(Scalar): def _compile(self, qb): code = self.expr.compile_wrap(qb).code if qb.target == sqlite: - return [f'(length('] + code + [') - length(replace('] + code + [f', "{_ARRAY_SEP}", ""))) / length("{_ARRAY_SEP}") + 1'] + return ( + [f'(length('] + + code + + [') - length(replace('] + + code + + [f', "{_ARRAY_SEP}", ""))) / length("{_ARRAY_SEP}") + 1'] + ) elif qb.target == postgres: return [f'array_length('] + code + [', 1)'] elif qb.target == bigquery: @@ -247,6 +276,7 @@ def _compile(self, qb): else: return [f'json_length('] + code + [')'] + @dataclass class FuncCall(SqlTree): type: Type @@ -257,6 +287,7 @@ def _compile(self, qb): s = join_comma(f.compile_wrap(qb).code for f in self.fields) return [f'{self.name}('] + s + [')'] + @dataclass class Cast(SqlTree): type: Type @@ -267,7 +298,9 @@ def _compile(self, qb): # XXX distinguish between these cases properly, not as a hack t = 'char' else: - t = _compile_type(qb.target, self.type.as_nullable()) # XXX as-nullable here is a hack + t = _compile_type( + qb.target, self.type.as_nullable() + ) # XXX as-nullable here is a hack return [f'CAST('] + self.value.compile_wrap(qb).code + [f' AS {t})'] @@ -283,13 +316,15 @@ class Case(SqlTree): def _compile(self, qb): cond = self.cond.compile_wrap(qb).code then = self.then.compile_wrap(qb).code - code = ["CASE WHEN "] + cond +[" THEN "] + then + code = ["CASE WHEN "] + cond + [" THEN "] + then if self.else_: - code += [ " ELSE " ] + self.else_.compile_wrap(qb).code + code += [" ELSE "] + self.else_.compile_wrap(qb).code return code + [" END "] + _ARRAY_SEP = '||' + @dataclass class MakeArray(SqlTree): type: Type @@ -326,6 +361,7 @@ def _compile(self, qb): def parens(x): return ['('] + x + [')'] + @dataclass class Compare(Scalar): op: str @@ -343,25 +379,19 @@ def _compile(self, qb): if any(e.type.maybe_null() for e in self.exprs): # Null values are possible, so we'll use identity operators if qb.target in (sqlite, duck): - op = { - '=': 'is', - '!=': 'is not' - }.get(op, op) + op = {'=': 'is', '!=': 'is not'}.get(op, op) elif qb.target is mysql: if op == '!=': # Special case, - return parens( ['not '] + join_sep(elems, f' <=> ') ) + return parens(['not '] + join_sep(elems, f' <=> ')) op = { '=': '<=>', }.get(op, op) else: - op = { - '=': 'is not distinct from', - '!=': 'is distinct from' - }.get(op, op) + op = {'=': 'is not distinct from', '!=': 'is distinct from'}.get(op, op) - return parens( join_sep(elems, f' {op} ') ) + return parens(join_sep(elems, f' {op} ')) @dataclass @@ -375,6 +405,7 @@ def _compile(self, qb): type = T.bool + @dataclass class LogicalNot(Scalar): expr: Sql @@ -397,7 +428,6 @@ def _compile(self, qb): return parens(x) - @dataclass class TableArith(TableOperation): op: str @@ -415,7 +445,7 @@ def _compile(self, qb): return code - type = property(X.exprs[0].type) # TODO ensure type correctness + type = property(X.exprs[0].type) # TODO ensure type correctness @dataclass @@ -428,6 +458,7 @@ def _compile(self, qb): type = property(X.expr.type) + @dataclass class Desc(SqlTree): expr: Sql @@ -438,7 +469,20 @@ def _compile(self, qb): type = property(X.expr.type) -_reserved = {'index', 'create', 'unique', 'table', 'select', 'where', 'group', 'by', 'over', 'user'} + +_reserved = { + 'index', + 'create', + 'unique', + 'table', + 'select', + 'where', + 'group', + 'by', + 'over', + 'user', +} + @dataclass class Name(SqlTree): @@ -454,16 +498,16 @@ def _compile(self, qb): name = qb.table_name[-1] + '.' + name return [name] + @dataclass class Attr(SqlTree): type: Type obj: Sql name: str - - # return base + @dataclass class ColumnAlias(SqlTree): value: Sql @@ -488,6 +532,7 @@ def _compile(self, qb): class SqlStatement(SqlTree): type = T.nulltype + @dataclass class AddIndex(SqlStatement): index_name: Id @@ -496,8 +541,11 @@ class AddIndex(SqlStatement): unique: bool def _compile(self, qb): - return [f"CREATE {'UNIQUE' if self.unique else ''} INDEX IF NOT EXISTS {quote_id(self.index_name)}" - f" ON {quote_id(self.table_name)}({self.column})"] + return [ + f"CREATE {'UNIQUE' if self.unique else ''} INDEX IF NOT EXISTS {quote_id(self.index_name)}" + f" ON {quote_id(self.table_name)}({self.column})" + ] + @dataclass class Insert(SqlStatement): @@ -507,7 +555,9 @@ class Insert(SqlStatement): def _compile(self, qb): columns = [quote_name(c) for c in self.columns] - return [f'INSERT INTO {quote_id(self.table_name)}({", ".join(columns)}) '] + self.query.compile(qb).code + return [ + f'INSERT INTO {quote_id(self.table_name)}({", ".join(columns)}) ' + ] + self.query.compile(qb).code def finalize_with_subqueries(self, qb, subqueries): if qb.target in (mysql, bigquery): @@ -518,11 +568,12 @@ def finalize_with_subqueries(self, qb, subqueries): return super().finalize_with_subqueries(qb, subqueries) + @dataclass class InsertConsts(SqlStatement): table: Id cols: List[str] - tuples: list #List[List[Sql]] + tuples: list # List[List[Sql]] def _compile(self, qb): cols = self.cols @@ -533,36 +584,41 @@ def _compile(self, qb): return ['INSERT INTO', quote_id(self.table), 'DEFAULT VALUES'] values = join_comma( - parens(join_comma([e.compile_wrap(qb).code for e in tpl])) - for tpl in tuples + parens(join_comma([e.compile_wrap(qb).code for e in tpl])) for tpl in tuples ) cols = [quote_name(c) for c in cols] - q = ['INSERT INTO', quote_id(self.table), - "(", ', '.join(cols), ")", - "VALUES ", + q = [ + 'INSERT INTO', + quote_id(self.table), + "(", + ', '.join(cols), + ")", + "VALUES ", ] - return [' '.join(q)] + values #+ [';'] + return [' '.join(q)] + values # + [';'] + @dataclass class InsertConsts2(SqlStatement): table: Id cols: List[str] - tuples: list #List[List[Sql]] + tuples: list # List[List[Sql]] def _compile(self, qb): assert self.tuples, self - values = join_comma( - parens(join_comma([t] for t in tpl)) - for tpl in self.tuples - ) + values = join_comma(parens(join_comma([t] for t in tpl)) for tpl in self.tuples) - q = ['INSERT INTO', quote_id(self.table), - "(", ', '.join(self.cols), ")", - "VALUES ", + q = [ + 'INSERT INTO', + quote_id(self.table), + "(", + ', '.join(self.cols), + ")", + "VALUES ", ] - return [' '.join(q)] + values #+ ';' + return [' '.join(q)] + values # + ';' @dataclass @@ -571,11 +627,12 @@ class LastRowId(Atom): def _compile(self, qb): if qb.target in (sqlite, duck): - return ['last_insert_rowid()'] # Sqlite + return ['last_insert_rowid()'] # Sqlite elif qb.target == mysql: return ['last_insert_id()'] else: - return ['lastval()'] # Postgres + return ['lastval()'] # Postgres + @dataclass class SelectValue(Atom, TableOperation): @@ -601,13 +658,16 @@ def _compile(self, qb): return ['SELECT ' + nulls + ' LIMIT 0'] if qb.target == mysql: + def row_func(x): return ['ROW('] + x + [')'] + else: row_func = parens return ['VALUES '] + join_comma(row_func(v.code) for v in values) + @dataclass class Tuple(SqlTree): type: Type @@ -617,6 +677,7 @@ def _compile(self, qb): values = [v.compile_wrap(qb).code for v in self.values] return join_comma(values) + @dataclass class ValuesTuple(Tuple): type: Type @@ -626,6 +687,7 @@ def _compile(self, qb): values = [v.compile_wrap(qb) for v in self.values] return join_comma(v.code for v in values) + @dataclass class ValuesTuples(Table): type: Type @@ -633,12 +695,13 @@ class ValuesTuples(Table): def _compile(self, qb): if not self.values: # SQL doesn't support empty values - return ['SELECT '] + join_comma(['NULL']*len(self.type.elems)) + ['LIMIT 0'] + return ( + ['SELECT '] + join_comma(['NULL'] * len(self.type.elems)) + ['LIMIT 0'] + ) values = [v.compile_wrap(qb) for v in self.values] return ['VALUES '] + join_comma(v.code for v in values) - @dataclass class AllFields(SqlTree): type: Type @@ -646,6 +709,7 @@ class AllFields(SqlTree): def _compile(self, qb): return ['*'] + @dataclass class Update(SqlTree): table: TableName @@ -654,16 +718,22 @@ class Update(SqlTree): type = T.nulltype def _compile(self, qb): - fields_sql = [k.compile_wrap(qb).code + [' = '] + v.compile_wrap(qb).code for k, v in self.fields.items()] + fields_sql = [ + k.compile_wrap(qb).code + [' = '] + v.compile_wrap(qb).code + for k, v in self.fields.items() + ] fields_sql = join_comma(fields_sql) sql = ['UPDATE '] + self.table.compile_wrap(qb).code + [' SET '] + fields_sql if self.conds: - sql += [' WHERE '] + join_sep([c.compile_wrap(qb).code for c in self.conds], ' AND ') + sql += [' WHERE '] + join_sep( + [c.compile_wrap(qb).code for c in self.conds], ' AND ' + ) return sql + @dataclass class Delete(SqlTree): table: TableName @@ -674,10 +744,11 @@ def _compile(self, qb): conds = join_sep([c.compile_wrap(qb).code for c in self.conds], ' AND ') return ['DELETE FROM '] + self.table.compile_wrap(qb).code + [' WHERE '] + conds + @dataclass class Select(TableOperation): type: Type - table: Sql # XXX Table won't work with RawSQL + table: Sql # XXX Table won't work with RawSQL fields: List[Sql] conds: List[Sql] = [] group_by: List[Sql] = [] @@ -724,14 +795,19 @@ def _compile(self, qb): sql = ['SELECT '] + select_sql + [' FROM '] + self.table.compile_wrap(qb).code if self.conds: - sql += [' WHERE '] + join_sep([c.compile_wrap(qb).code for c in self.conds], ' AND ') - + sql += [' WHERE '] + join_sep( + [c.compile_wrap(qb).code for c in self.conds], ' AND ' + ) if self.group_by: - sql += [' GROUP BY '] + join_comma(e.compile_wrap(qb).code for e in self.group_by) + sql += [' GROUP BY '] + join_comma( + e.compile_wrap(qb).code for e in self.group_by + ) if self.order: - sql += [' ORDER BY '] + join_comma(o.compile_wrap(qb).code for o in self.order) + sql += [' ORDER BY '] + join_comma( + o.compile_wrap(qb).code for o in self.order + ) if self.limit is not None: sql += [' LIMIT ', str(self.limit)] @@ -746,7 +822,6 @@ def _compile(self, qb): # BigQuery requires a specific limit, always! sql += [' LIMIT 9223372036854775807'] - if self.offset is not None: sql += [' OFFSET ', str(self.offset)] @@ -763,9 +838,11 @@ def join_sep(code_list, sep): yield sep yield from c + def join_comma(code_list): return join_sep(code_list, ", ") + @dataclass class Subquery(SqlTree): table_name: str @@ -778,10 +855,13 @@ def _compile(self, qb): if qb.target == bigquery: fields_str = [] else: - fields = [f.compile_wrap(qb.replace(is_root=False)).code for f in self.fields] + fields = [ + f.compile_wrap(qb.replace(is_root=False)).code for f in self.fields + ] fields_str = ["("] + join_comma(fields) + [")"] if fields else [] return [f"{self.table_name}"] + fields_str + [" AS ("] + query + [")"] + def _enum_is_last(seq): last = len(seq) - 1 for i, item in enumerate(seq): @@ -807,12 +887,11 @@ def _compile(self, qb): if self.conds and is_last: code += join_sep([c.compile_wrap(qb).code for c in self.conds], ' AND ') else: - code += ['1=1'] # Postgres requires ON clause + code += ['1=1'] # Postgres requires ON clause return code - @dataclass class BigQueryValues(SqlTree): type: Type @@ -823,16 +902,21 @@ def _compile(self, qb): rows = [ ( - ['STRUCT('] + join_comma(v.compile(qb).code + [" as ", name] - for name, v in safezip(cols, row.values)) + [")"] - ) if isinstance(row, Tuple) else row.compile(qb).code - for row in self.values + ['STRUCT('] + + join_comma( + v.compile(qb).code + [" as ", name] + for name, v in safezip(cols, row.values) + ) + + [")"] + ) + if isinstance(row, Tuple) + else row.compile(qb).code + for row in self.values ] return ["SELECT * FROM UNNEST(["] + join_comma(rows) + ["]) as item"] - @dataclass class TableQueryValues(SqlTree): "A subquery which returns a table of given values" @@ -844,7 +928,10 @@ class TableQueryValues(SqlTree): def _compile(self, qb): if qb.target != 'bigquery': values_cls = Values - fields = [Name(col_type, col_name) for col_name, col_type in self.type.elems.items()] + fields = [ + Name(col_type, col_name) + for col_name, col_type in self.type.elems.items() + ] else: values_cls = BigQueryValues fields = None @@ -853,7 +940,6 @@ def _compile(self, qb): return subq._compile(qb) - @dataclass class StringSlice(SqlTree): string: Sql @@ -891,24 +977,29 @@ def _compile(self, qb): def _repr(_t: T.number, x): return str(x) + @dp_type def _repr(_t: T.bool, x): return ['false', 'true'][x] + @dp_type def _repr(_t: T.decimal, x): return repr(float(x)) # TODO SQL decimal? + @dp_type def _repr(_t: T.datetime, x): # TODO Better to pass the object instead of a string? return repr(str(x)) + @dp_type def _repr(_t: T.timestamp, x): # TODO Better to pass the object instead of a string? return repr(str(x)) + @dp_type def _repr(_t: T.union[T.string, T.text], x): target = get_db().target @@ -918,10 +1009,12 @@ def _repr(_t: T.union[T.string, T.text], x): res = res.replace('\n', '\\n') return res + def quote_name(name): assert isinstance(name, str), name return get_db().quote_name(name) + def quote_id(id_): assert isinstance(id_, Id) return '.'.join(quote_name(n) for n in id_.parts) @@ -930,7 +1023,7 @@ def quote_id(id_): @dp_type def _compile_type(target, type_: T.t_relation): # TODO might have a different type - #return 'INTEGER' # Foreign-key is integer + # return 'INTEGER' # Foreign-key is integer return _compile_type(target, type_.elems['item']) @@ -945,6 +1038,7 @@ class Types_PqlToSql: string = "VARCHAR(4000)" text = "TEXT" + class P2S_BigQuery(Types_PqlToSql): int = "INT64" string = "STRING" @@ -955,17 +1049,20 @@ class P2S_BigQuery(Types_PqlToSql): class P2S_MySql(Types_PqlToSql): int = "SIGNED" + class P2S_Sqlite(Types_PqlToSql): datetime = "DATETIME" timestamp = "TIMESTAMP" + class P2S_Postgres(Types_PqlToSql): datetime = "timestamp with time zone" + _pql_to_sql_by_target = { bigquery: P2S_BigQuery, mysql: P2S_MySql, - "mysql_def": Types_PqlToSql, # Standard (Different) types for declaration! + "mysql_def": Types_PqlToSql, # Standard (Different) types for declaration! sqlite: P2S_Sqlite, duck: P2S_Sqlite, postgres: P2S_Postgres, @@ -979,38 +1076,39 @@ def _compile_type(target, type_: T.primitive): s += " NOT NULL" return s + @dp_type def _compile_type(target, _type: T.nulltype): if target == bigquery: return 'INT64' - return 'INTEGER' # TODO is there a better value here? Maybe make it read-only somehow + return ( + 'INTEGER' # TODO is there a better value here? Maybe make it read-only somehow + ) + @dp_type def _compile_type(target, idtype: T.t_id): if target == bigquery: s = "STRING" else: - s = "INTEGER" # TODO non-int idtypes + s = "INTEGER" # TODO non-int idtypes if not idtype.maybe_null(): s += " NOT NULL" return s + @dp_type def _compile_type(target, _type: T.json): return 'JSON' - - - # API + def compile_drop_table(table_name) -> Sql: return RawSql(T.nulltype, f'DROP TABLE {quote_id(table_name)}') - - def compile_type_def(table_name, table) -> Sql: assert isinstance(table_name, Id) assert table <= T.table @@ -1026,9 +1124,11 @@ def compile_type_def(table_name, table) -> Sql: if name in pks and c <= T.t_id: type_decl = db.id_type_decl else: - type_decl = _compile_type(db.target if db.target != mysql else 'mysql_def', c) + type_decl = _compile_type( + db.target if db.target != mysql else 'mysql_def', c + ) - columns.append( f'{quote_name(name)} {type_decl}' ) + columns.append(f'{quote_name(name)} {type_decl}') if c <= T.t_relation: if db.supports_foreign_key: @@ -1036,24 +1136,33 @@ def compile_type_def(table_name, table) -> Sql: if not table.options.get('temporary', False): # In postgres, constraints on temporary tables may reference only temporary tables rel = c.options['rel'] - if rel['key']: # Requires a unique constraint + if rel['key']: # Requires a unique constraint tbl_name = rel['table'].options['name'] s = f"FOREIGN KEY({name}) REFERENCES {quote_id(tbl_name)}({rel['column']})" posts.append(s) - if pks and db.supports_foreign_key: + if pks and db.supports_foreign_key: names = ", ".join(pks) posts.append(f"PRIMARY KEY ({names})") # Consistent among SQL databases if db.target == 'bigquery': - command = ("CREATE TABLE" if table.options.get('temporary', False) else "CREATE TABLE IF NOT EXISTS") + command = ( + "CREATE TABLE" + if table.options.get('temporary', False) + else "CREATE TABLE IF NOT EXISTS" + ) else: - command = "CREATE TEMPORARY TABLE" if table.options.get('temporary', False) else "CREATE TABLE IF NOT EXISTS" - - return RawSql(T.nulltype, f'{command} {quote_id(table_name)} (' + ', '.join(columns + posts) + ')') - + command = ( + "CREATE TEMPORARY TABLE" + if table.options.get('temporary', False) + else "CREATE TABLE IF NOT EXISTS" + ) + return RawSql( + T.nulltype, + f'{command} {quote_id(table_name)} (' + ', '.join(columns + posts) + ')', + ) def deletes_by_ids(table, ids): @@ -1061,12 +1170,16 @@ def deletes_by_ids(table, ids): compare = Compare('=', [Name(T.t_id, 'id'), Primitive(T.t_id, repr(id_))]) yield Delete(TableName(table.type, table.type.options['name']), [compare]) + def updates_by_ids(table, proj, ids): # TODO this function is not safe & secure enough sql_proj = {Name(value.type, name): value.code for name, value in proj.items()} for id_ in ids: compare = Compare('=', [Name(T.t_id, 'id'), Primitive(T.t_id, repr(id_))]) - yield Update(TableName(table.type, table.type.options['name']), sql_proj, [compare]) + yield Update( + TableName(table.type, table.type.options['name']), sql_proj, [compare] + ) + def create_list(name, elems): # Assumes all elems have the same type! @@ -1076,22 +1189,27 @@ def create_list(name, elems): return table, subq, t - def create_table(t, name, rows): subq = TableQueryValues(t, name, rows) table = TableName(t, Id(name)) return table, subq + def table_slice(table, start, stop): limit = stop - start if stop else None - return Select(table.type, table.code, [AllFields(table.type)], offset=start, limit=limit) + return Select( + table.type, table.code, [AllFields(table.type)], offset=start, limit=limit + ) + def table_selection(table, conds): return Select(table.type, table.code, [AllFields(table.type)], conds=conds) + def table_order(table, fields): return Select(table.type, table.code, [AllFields(table.type)], order=fields) + def arith(res_type, op, args): target = get_db().target @@ -1099,7 +1217,7 @@ def arith(res_type, op, args): if res_type == T.string: assert op == '+' op = '||' - if target is mysql: # doesn't support a || b + if target is mysql: # doesn't support a || b return FuncCall(res_type, 'concat', arg_codes) elif op == '/': if target != mysql: @@ -1121,7 +1239,7 @@ def arith(res_type, op, args): return BinOp(res_type, op, arg_codes) - + def make_value(x): if x is None: return null @@ -1129,9 +1247,12 @@ def make_value(x): try: t = pql_types.from_python(type(x)) except KeyError as e: - raise Signal.make(T.ValueError, x, f"Cannot import value of Python type {type(x)}") from e + raise Signal.make( + T.ValueError, x, f"Cannot import value of Python type {type(x)}" + ) from e return Primitive(t, _repr(t, x)) + def add_one(x): return BinOp(x.type, '+', [x, make_value(1)]) diff --git a/preql/core/sql_import_result.py b/preql/core/sql_import_result.py index ae42dd7..24010bb 100644 --- a/preql/core/sql_import_result.py +++ b/preql/core/sql_import_result.py @@ -8,18 +8,19 @@ """ -from datetime import datetime import decimal import json +from datetime import datetime import arrow +from preql.context import context from preql.utils import listgen, safezip -from .pql_types import T, dp_type, dp_inst + from .exceptions import Signal +from .pql_types import T, dp_inst, dp_type +from .sql import _ARRAY_SEP, mysql, sqlite from .types_impl import flatten_type -from preql.context import context -from .sql import _ARRAY_SEP, sqlite, mysql def _from_datetime(s): @@ -45,14 +46,17 @@ def _from_datetime(s): def _restructure_result(t, i): raise Signal.make(T.TypeError, None, f"Unexpected type used: {t}") + @dp_type def _restructure_result(t: T.table, i): # return ({name: _restructure_result(state, col, i) for name, col in t.elem_dict.items()}) return next(i) + @dp_type def _restructure_result(t: T.struct, i): - return ({name: _restructure_result(col, i) for name, col in t.elems.items()}) + return {name: _restructure_result(col, i) for name, col in t.elems.items()} + @dp_type def _restructure_result(_t: T.union[T.primitive, T.nulltype], i): @@ -70,7 +74,11 @@ def _restructure_result(t: T.json_array[T.union[T.primitive, T.nulltype]], i): res = json.loads(res) elif target == sqlite: if not isinstance(res, str): - raise Signal.make(T.TypeError, None, f"json_array type expected a string separated by {_ARRAY_SEP}. Got: '{res}'") + raise Signal.make( + T.TypeError, + None, + f"json_array type expected a string separated by {_ARRAY_SEP}. Got: '{res}'", + ) res = res.split(_ARRAY_SEP) # XXX hack! TODO Use a generic form to cast types @@ -80,95 +88,121 @@ def _restructure_result(t: T.json_array[T.union[T.primitive, T.nulltype]], i): elif t.elem <= T.float: res = [float(x) for x in res] except ValueError: - raise Signal.make(T.TypeError, None, f"Error trying to convert values to type {t.elem}") + raise Signal.make( + T.TypeError, None, f"Error trying to convert values to type {t.elem}" + ) return res + @dp_type def _restructure_result(_t: T.datetime, i): s = next(i) return _from_datetime(s) + @dp_type def _restructure_result(_t: T.timestamp, i): s = next(i) return _from_datetime(s) - def _extract_primitive(res, expected): try: - row ,= res.value - item ,= row + (row,) = res.value + (item,) = row except ValueError: - raise Signal.make(T.TypeError, None, f"Expected a single {expected}. Got: '{res.value}'") + raise Signal.make( + T.TypeError, None, f"Expected a single {expected}. Got: '{res.value}'" + ) return item + @dp_inst def sql_result_to_python(res: T.bool): item = _extract_primitive(res, 'bool') if item not in (0, 1): - raise Signal.make(T.ValueError, None, f"Expected SQL to return a bool. Instead got '{item}'") + raise Signal.make( + T.ValueError, None, f"Expected SQL to return a bool. Instead got '{item}'" + ) return bool(item) + @dp_inst def sql_result_to_python(res: T.int): item = _extract_primitive(res, 'int') if not isinstance(item, int): breakpoint() - raise Signal.make(T.ValueError, None, f"Expected SQL to return an int. Instead got '{item}'") + raise Signal.make( + T.ValueError, None, f"Expected SQL to return an int. Instead got '{item}'" + ) return item + @dp_inst def sql_result_to_python(res: T.primitive): return _extract_primitive(res, res) + @dp_inst def sql_result_to_python(res): return res.value + @dp_inst def sql_result_to_python(res: T.datetime): # XXX doesn't belong here? item = _extract_primitive(res, 'datetime') return _from_datetime(item) + @dp_inst def sql_result_to_python(res: T.timestamp): # XXX doesn't belong here? item = _extract_primitive(res, 'datetime') return _from_datetime(item) + def _from_sql_primitive(p): if isinstance(p, decimal.Decimal): # TODO Needs different handling when we expect a decimal return float(p) return p + @dp_inst def sql_result_to_python(arr: T.list): fields = flatten_type(arr.type) - if not all(len(e)==len(fields) for e in arr.value): - raise Signal.make(T.TypeError, None, f"Expected 1 column. Got {len(arr.value[0])}") + if not all(len(e) == len(fields) for e in arr.value): + raise Signal.make( + T.TypeError, None, f"Expected 1 column. Got {len(arr.value[0])}" + ) if arr.type.elem <= T.struct: - return [{n: _from_sql_primitive(e) for (n, _t), e in safezip(fields, tpl)} for tpl in arr.value] + return [ + {n: _from_sql_primitive(e) for (n, _t), e in safezip(fields, tpl)} + for tpl in arr.value + ] else: return [_from_sql_primitive(e[0]) for e in arr.value] + @dp_inst @listgen def sql_result_to_python(arr: T.table): - expected_length = len(flatten_type(arr.type)) # TODO optimize? + expected_length = len(flatten_type(arr.type)) # TODO optimize? for row in arr.value: if len(row) != expected_length: - raise Signal.make(T.TypeError, None, f"Expected {expected_length} columns, but got {len(row)}") + raise Signal.make( + T.TypeError, + None, + f"Expected {expected_length} columns, but got {len(row)}", + ) i = iter(row) - yield {name: _restructure_result(col, i) for name, col in arr.type.elems.items()} - - - + yield { + name: _restructure_result(col, i) for name, col in arr.type.elems.items() + } def _bool_from_sql(n): @@ -179,12 +213,13 @@ def _bool_from_sql(n): assert isinstance(n, bool), n return n + def type_from_sql(type, nullable): type = type.lower() d = { 'integer': T.int, - 'int': T.int, # mysql - 'tinyint(1)': T.bool, # mysql + 'int': T.int, # mysql + 'tinyint(1)': T.bool, # mysql 'serial': T.t_id, 'bigserial': T.t_id, 'smallint': T.int, # TODO smallint / bigint? @@ -193,7 +228,7 @@ def type_from_sql(type, nullable): 'character': T.string, # TODO char? 'real': T.float, 'float': T.float, - 'double precision': T.float, # double on 32-bit? + 'double precision': T.float, # double on 32-bit? 'boolean': T.bool, 'timestamp': T.timestamp, 'timestamp without time zone': T.timestamp, @@ -206,11 +241,11 @@ def type_from_sql(type, nullable): try: v = d[type] except KeyError: - if type.startswith('int('): # TODO actually parse it + if type.startswith('int('): # TODO actually parse it return T.int - elif type.startswith('tinyint('): # TODO actually parse it + elif type.startswith('tinyint('): # TODO actually parse it return T.int - elif type.startswith('varchar('): # TODO actually parse it + elif type.startswith('varchar('): # TODO actually parse it return T.string return T.string.as_nullable() diff --git a/preql/core/state.py b/preql/core/state.py index a472c34..957cc4c 100644 --- a/preql/core/state.py +++ b/preql/core/state.py @@ -1,13 +1,13 @@ -from logging import getLogger -from copy import copy from contextlib import contextmanager +from copy import copy +from logging import getLogger from preql.context import context - -from .exceptions import InsufficientAccessLevel, Signal -from .pql_types import T, Id from . import pql_ast as ast +from .exceptions import InsufficientAccessLevel, Signal +from .pql_types import Id, T + class NameNotFound(Exception): pass @@ -15,13 +15,16 @@ class NameNotFound(Exception): logger = getLogger('state') + class Namespace: def __init__(self, ns=None): self._ns = ns or [{}] # self._parameters = None def __copy__(self): - return Namespace([self._ns[0]] + [dict(n) for n in self._ns[1:]]) # Shared global namespace + return Namespace( + [self._ns[0]] + [dict(n) for n in self._ns[1:]] + ) # Shared global namespace def get_var(self, name): # Uses context.parameters, set in Interpreter @@ -48,7 +51,6 @@ def set_var(self, name, value): assert not isinstance(value, ast.Name) self._ns[-1][name] = value - @contextmanager def use_scope(self, scope: dict): x = len(self._ns) @@ -59,7 +61,6 @@ def use_scope(self, scope: dict): _discarded_scope = self._ns.pop() assert x == len(self._ns) - # def push_scope(self): # self.ns.append({}) @@ -72,7 +73,7 @@ def __len__(self): def get_all_vars(self): d = {} for scope in reversed(self._ns): - d.update(scope) # Overwrite upper scopes + d.update(scope) # Overwrite upper scopes return d def get_all_vars_with_rank(self): @@ -94,7 +95,6 @@ class AccessLevels: class State: - def __init__(self, interp, db, display, ns=None): self.db = db self.interp = interp @@ -105,9 +105,9 @@ def __init__(self, interp, db, display, ns=None): self._cache = {} - def connect(self, uri, auto_create=False): from preql.sql_interface import ConnectError, create_engine + logger.info(f"[Preql] Connecting to {uri}") try: self.db = create_engine(uri, self.db._print_sql, auto_create) @@ -120,7 +120,6 @@ def connect(self, uri, auto_create=False): self._db_uri = uri - def unique_name(self, obj): self.tick[0] += 1 return obj + str(self.tick[0]) @@ -151,7 +150,6 @@ def db(self): @property def display(self): return self.state.display - @classmethod def clone(cls, inst): @@ -161,7 +159,6 @@ def clone(cls, inst): s.stacktrace = copy(inst.stacktrace) return s - def limit_access(self, new_level): return self.reduce_access(min(new_level, self.access_level)) @@ -174,11 +171,11 @@ def reduce_access(self, new_level): def require_access(self, level): if self.access_level < level: raise InsufficientAccessLevel(level) + def catch_access(self, level): if self.access_level < level: raise Exception("Bad access. Security risk.") - def get_all_vars(self): return self.ns.get_all_vars() @@ -190,7 +187,7 @@ def has_var(self, name): self.ns.get_var(name) except NameNotFound: return False - + return True def get_var(self, name): @@ -206,7 +203,6 @@ def get_var(self, name): raise Signal.make(T.NameError, name, f"Name '{name}' is not defined") - def set_var(self, name, value): try: return self.ns.set_var(name, value) @@ -226,36 +222,45 @@ def unique_name(self, obj): return self.state.unique_name(obj) - def set_var(name, value): return context.state.set_var(name, value) + def use_scope(scope): return context.state.use_scope(scope) + def get_var(name): return context.state.get_var(name) + def has_var(name): return context.state.has_var(name) + def get_db(): return context.state.db + def unique_name(prefix): return context.state.unique_name(prefix) + def require_access(access): - return context.state.require_access(access) + return context.state.require_access(access) + def catch_access(access): - return context.state.catch_access(access) + return context.state.catch_access(access) + def get_access_level(): - return context.state.access_level + return context.state.access_level + def reduce_access(new_level): - return context.state.reduce_access(new_level) + return context.state.reduce_access(new_level) + def get_display(): - return context.state.display \ No newline at end of file + return context.state.display diff --git a/preql/core/types_impl.py b/preql/core/types_impl.py index 72fb25d..d9dd2ce 100644 --- a/preql/core/types_impl.py +++ b/preql/core/types_impl.py @@ -1,43 +1,54 @@ -from .exceptions import Signal, pql_AttributeError - -from preql.utils import concat_for, classify_bool +from preql.utils import classify_bool, concat_for from .base import Object +from .exceptions import Signal, pql_AttributeError from .pql_types import ITEM_NAME, T, Type, dp_type def Object_get_attr(self, attr): raise pql_AttributeError(attr) + def Object_isa(self, t): if not isinstance(t, Type): - raise Signal.make(T.TypeError, None, f"'type' argument to isa() isn't a type. It is {t}") + raise Signal.make( + T.TypeError, None, f"'type' argument to isa() isn't a type. It is {t}" + ) return self.type <= t + Object.get_attr = Object_get_attr Object.isa = Object_isa + def _type_flatten_code(self): raise Signal.make(T.TypeError, None, f"Found type 'type' in unexpected place") + Type.flatten_code = _type_flatten_code + @dp_type def flatten_path(path, t): return [(path, t)] + @dp_type def flatten_path(path, t: T.union[T.table, T.struct]): elems = t.elems if t.maybe_null(): - elems = {k:v.as_nullable() for k, v in elems.items()} + elems = {k: v.as_nullable() for k, v in elems.items()} return concat_for(flatten_path(path + [name], col) for name, col in elems.items()) + + @dp_type def flatten_path(path, t: T.list): - return concat_for(flatten_path(path + [name], col) for name, col in [(ITEM_NAME, t.elem)]) + return concat_for( + flatten_path(path + [name], col) for name, col in [(ITEM_NAME, t.elem)] + ) -def flatten_type(tp, path = []): +def flatten_type(tp, path=[]): return [(join_names(path), t) for path, t in flatten_path(path, tp)] @@ -46,7 +57,6 @@ def table_params(t): return [(name, c) for name, c in t.elems.items() if not c <= T.t_id] - def table_flat_for_insert(table): read_only = {join_names(pk) for pk in table.options.get('pk', [])} names = [name for name, t in flatten_type(table)] @@ -64,7 +74,6 @@ def pql_repr(t, value): def kernel_type(t): - if t <= T.projected: # or t <= T.aggregated: + if t <= T.projected: # or t <= T.aggregated: return kernel_type(t.elems['item']) return t - diff --git a/preql/docstring/autodoc.py b/preql/docstring/autodoc.py index 3bd3cb3..b0b9523 100644 --- a/preql/docstring/autodoc.py +++ b/preql/docstring/autodoc.py @@ -2,11 +2,11 @@ from runtype import dataclass -from preql.utils import safezip, dsp -from preql.docstring.docstring import parse, Section, Defin, Text -from preql.core.pql_objects import Module, Function, T, MethodInstance +from preql.core.pql_objects import Function, MethodInstance, Module, T from preql.core.pql_types import Type, subtypes +from preql.docstring.docstring import Defin, Section, Text, parse from preql.settings import color_theme +from preql.utils import dsp, safezip from . import type_docs @@ -14,6 +14,7 @@ class AutoDocError(Exception): pass + @dataclass class ModuleDoc: module: object @@ -56,7 +57,7 @@ def print_text(self, indent=0): indent_str = ' ' * indent parent = (self.parent_type.repr() + '.') if self.parent_type else '' s = f'{indent_str}[{color_kw}]func[/{color_kw}] {parent}[bold white]{self.func.name}[/bold white]({params}) = ...\n\n' - return s + self.doc.print_text(indent+4) + return s + self.doc.print_text(indent + 4) def print_rst(self): is_method = bool(self.parent_type) @@ -73,6 +74,7 @@ def print_rst(self): s = ' ' + s.replace('\n', '\n ') # XXX hack to indent methods return s + @dataclass class TypeDoc: type: object @@ -87,20 +89,20 @@ def print_text(self, indent=0): params = f'\\[{params}]' indent_str = ' ' * indent s = f'{indent_str}[{color_kw}]type[/{color_kw}] [{color_class}]{self.type.typename}[/{color_class}]{params}\n\n' - return s + self.doc.print_text(indent+4) + return s + self.doc.print_text(indent + 4) def print_rst(self): type_name = str(self.type) # s = type_name + '\n' # s += '^' * len(type_name) + '\n' - s = f".. class:: {type_name}⁣\n\n" # includes an invisible unicode separator to trick sphinx + s = f".. class:: {type_name}⁣\n\n" # includes an invisible unicode separator to trick sphinx return s + self.doc.print_rst() - from lark import LarkError + def doc_func(f, parent_type=None): if isinstance(f, MethodInstance): f = f.func @@ -109,12 +111,22 @@ def doc_func(f, parent_type=None): except LarkError as e: raise AutoDocError(f"Error in docstring of function {f.name}: {e}") - assert {s.name for s in doc_tree.sections} <= {'Parameters', 'Example', 'Examples', 'Note', 'Returns', 'See Also'}, [s.name for s in doc_tree.sections] + assert {s.name for s in doc_tree.sections} <= { + 'Parameters', + 'Example', + 'Examples', + 'Note', + 'Returns', + 'See Also', + }, [s.name for s in doc_tree.sections] try: params_doc = doc_tree.get_section('Parameters') except KeyError: if f.params: - params_doc = Section('Parameters', [Defin(p.name, None, str(p.type) if p.type else '') for p in f.params]) + params_doc = Section( + 'Parameters', + [Defin(p.name, None, str(p.type) if p.type else '') for p in f.params], + ) doc_tree.sections.insert(0, params_doc) else: params = list(f.params) @@ -143,8 +155,10 @@ def doc_module(m): # p = Preql() # rich.print(doc_module(p('__builtins__')).print_text()) + def generate_rst(modules_fn, types_fn): from preql import Preql + p = Preql() with open(types_fn, 'w', encoding='utf8') as f: @@ -163,14 +177,17 @@ def generate_rst(modules_fn, types_fn): p('import graph') print(doc_module(p('graph')).print_rst(), file=f) + @dsp def autodoc(m: Module): return doc_module(m) + @dsp def autodoc(f: Function): return doc_func(f) + @dsp def autodoc(t: Type): try: @@ -182,24 +199,33 @@ def autodoc(t: Type): except LarkError as e: raise AutoDocError(f"Error in docstring of type {t}") - assert {s.name for s in doc_tree.sections} <= {'Example', 'Examples', 'Note', 'See Also'}, [s.name for s in doc_tree.sections] + assert {s.name for s in doc_tree.sections} <= { + 'Example', + 'Examples', + 'Note', + 'See Also', + }, [s.name for s in doc_tree.sections] if t.proto_attrs: - methods_doc = Section('Methods', [doc_func(f, t) for f in t.proto_attrs.values() if isinstance(f, Function)]) + methods_doc = Section( + 'Methods', + [doc_func(f, t) for f in t.proto_attrs.values() if isinstance(f, Function)], + ) doc_tree.sections.insert(0, methods_doc) if t in subtypes: - subtypes_doc = Section('Subtypes', [Text([str(st) + ", "]) for st in subtypes[t]]) + subtypes_doc = Section( + 'Subtypes', [Text([str(st) + ", "]) for st in subtypes[t]] + ) doc_tree.sections.insert(0, subtypes_doc) - if t.supertypes: supertypes_doc = Section('Supertypes', [Text([str(st)]) for st in t.supertypes]) doc_tree.sections.insert(0, supertypes_doc) - return TypeDoc(t, doc_tree) + # test_func() # test_module() if __name__ == '__main__': diff --git a/preql/docstring/docstring.py b/preql/docstring/docstring.py index 35933c7..b2beaf9 100644 --- a/preql/docstring/docstring.py +++ b/preql/docstring/docstring.py @@ -1,16 +1,15 @@ - -from typing import Optional, List, Union +from typing import List, Optional, Union from lark import Lark, Transformer, v_args from lark.indenter import Indenter - -from runtype import dataclass from rich.markup import escape as rich_esc +from runtype import dataclass def indent_str(indent): return ' ' * (indent) + @dataclass class Text: lines: List[str] @@ -35,7 +34,6 @@ def _print_rst(self, section=None): yield from self._print_text(8) - @dataclass(frozen=False) class Defin: name: str @@ -51,7 +49,7 @@ def _print_text(self, indent): decl = f'[bold]{self.name}[/bold]{type}' yield f'{indent_str(indent)}{decl}: ' if self.text: - yield from self.text._print_text(indent+len(decl)+2, True) + yield from self.text._print_text(indent + len(decl) + 2, True) else: yield '\n' @@ -64,7 +62,11 @@ def _print_html(self): yield from self.text._print_html() def _print_rst(self): - text = self.text.replace(lines = self.text.lines + [f'(default={self.default})']) if self.default else self.text + text = ( + self.text.replace(lines=self.text.lines + [f'(default={self.default})']) + if self.default + else self.text + ) text = ''.join(text._print_rst()) if self.text else '\n' yield f" :param {self.name}: {text}" if self.type: @@ -79,7 +81,7 @@ class Section: def _print_text(self, indent): l = [f'[bold white]{indent_str(indent)}{self.name}[/bold white]:\n'] for item in self.items: - l += item._print_text(indent+4) + l += item._print_text(indent + 4) return l def _print_html(self): @@ -103,6 +105,7 @@ def _print_rst(self): for item in self.items: yield from item._print_rst() + @dataclass class DocString: header: Text @@ -144,8 +147,9 @@ def print_rst(self): return ''.join(self._print_rst()) - _inline = v_args(inline=True) + + class DocTransformer(Transformer): def as_list(self, items): return items @@ -167,7 +171,6 @@ def defin(self, name, text): return Defin(name.rstrip(':'), text) - class DocIndenter(Indenter): NL_type = '_NL' OPEN_PAREN_types = [] @@ -177,18 +180,20 @@ class DocIndenter(Indenter): tab_len = 1 -parser = Lark.open('docstring.lark', rel_to=__file__, - parser='lalr', #lexer=_Lexer, - postlex=DocIndenter(), - maybe_placeholders=True, - ) +parser = Lark.open( + 'docstring.lark', + rel_to=__file__, + parser='lalr', # lexer=_Lexer, + postlex=DocIndenter(), + maybe_placeholders=True, +) def parse(s): s = s.strip() if not s: return DocString(Text([]), []) - tree = parser.parse(s+'\n') + tree = parser.parse(s + '\n') return DocTransformer().transform(tree) @@ -220,4 +225,4 @@ def parse(s): if __name__ == "__main__": - test_parser() \ No newline at end of file + test_parser() diff --git a/preql/docstring/type_docs.py b/preql/docstring/type_docs.py index c96ab58..a2275b7 100644 --- a/preql/docstring/type_docs.py +++ b/preql/docstring/type_docs.py @@ -1,7 +1,7 @@ from preql.core.pql_types import T DOCS = { -T.any: """A meta-type that can match any type. + T.any: """A meta-type that can match any type. Examples: >> isa(my_obj, any) // always returns true @@ -9,10 +9,8 @@ >> isa(my_type, any) // always returns true true """, - -# T.unknown: None, # TODO - -T.union: """A meta-type that means 'either one of the given types' + # T.unknown: None, # TODO + T.union: """A meta-type that means 'either one of the given types' Example: >> int <= union[int, string] @@ -23,8 +21,7 @@ true """, - -T.type: """The type of types + T.type: """The type of types Examples: >> type(int) == type(string) @@ -34,10 +31,9 @@ >> isa(int, type(int)) # int is an instance of `type` true """, -T.object: """The base object type + T.object: """The base object type """, - -T.nulltype: """The type of the singleton `null`. + T.nulltype: """The type of the singleton `null`. Represents SQL `NULL`, but behaves like Python's `None`, @@ -48,35 +44,26 @@ >> null + 1 TypeError: Operator '+' not implemented for nulltype and int """, - -T.primitive: "The base type for all primitives", - -T.text: "A text type (behaves the same as `string`)", -T.string: "A string type (behaves the same as `text`)", -T.number: "The base type for all numbers", -T.int: "An integer number", -T.float: "A floating-point number", -T.bool: "A boolean, which can be either `true` or `false`", -# T.decimal: "A decimal number", - -T.datetime: "A datetime type (date+time combined)", -T.timestamp: "A timestamp type (unix epoch)", - -T.container: """The base type of containers. + T.primitive: "The base type for all primitives", + T.text: "A text type (behaves the same as `string`)", + T.string: "A string type (behaves the same as `text`)", + T.number: "The base type for all numbers", + T.int: "An integer number", + T.float: "A floating-point number", + T.bool: "A boolean, which can be either `true` or `false`", + # T.decimal: "A decimal number", + T.datetime: "A datetime type (date+time combined)", + T.timestamp: "A timestamp type (unix epoch)", + T.container: """The base type of containers. A container holds other objects inside it. """, - -T.struct: "A structure type", - -T.row: "A row in a table. (essentially a named-tuple)", - -# T.collection: """The base class of collections. - -# A collection holds an array of other objects inside it. -# """, - -T.table: """A table type. + T.struct: "A structure type", + T.row: "A row in a table. (essentially a named-tuple)", + # T.collection: """The base class of collections. + # A collection holds an array of other objects inside it. + # """, + T.table: """A table type. Tables support the following operations - - Projection (or: map), using the `{}` operator @@ -87,32 +74,27 @@ - Delete, using the `delete[]` operator - `+` for concat, `&` for intersect, `|` for union """, - -T.list: """A list type""", -T.set: """A set type, in which all elements are unique""", -T.projected: """A meta-type to signify projected operations, i.e. operations inside a projection. + T.list: """A list type""", + T.set: """A set type, in which all elements are unique""", + T.projected: """A meta-type to signify projected operations, i.e. operations inside a projection. Example: >> x = [1] >> one one x{ repr(type(item)) } "projected[item: int]" """, -T.aggregated: """A meta-type to signify aggregated operations, i.e. operations inside a grouping + T.aggregated: """A meta-type to signify aggregated operations, i.e. operations inside a grouping Example: >> x = [1] >> one one x{ => repr(type(item))} "aggregated[item: int]" """, -T.t_id: "The type of a table id", -T.t_relation: "The type of a table relation", - - -T.json: "A json type", -T.json_array: "A json array type. Created by aggregation.", - -T.function: "A meta-type for all functions", -T.module: "A meta-type for all modules", -T.signal: "A meta-type for all signals (i.e. exceptions)", - + T.t_id: "The type of a table id", + T.t_relation: "The type of a table relation", + T.json: "A json type", + T.json_array: "A json array type. Created by aggregation.", + T.function: "A meta-type for all functions", + T.module: "A meta-type for all modules", + T.signal: "A meta-type for all signals (i.e. exceptions)", } diff --git a/preql/jup_kernel/__main__.py b/preql/jup_kernel/__main__.py index 45aa2d5..d9ca698 100644 --- a/preql/jup_kernel/__main__.py +++ b/preql/jup_kernel/__main__.py @@ -1,4 +1,5 @@ from ipykernel.kernelapp import IPKernelApp + from . import PreqlKernel IPKernelApp.launch_instance(kernel_class=PreqlKernel) diff --git a/preql/jup_kernel/install.py b/preql/jup_kernel/install.py index cdbf824..d8e88fe 100644 --- a/preql/jup_kernel/install.py +++ b/preql/jup_kernel/install.py @@ -3,8 +3,8 @@ import os import sys -from jupyter_client.kernelspec import KernelSpecManager from IPython.utils.tempdir import TemporaryDirectory +from jupyter_client.kernelspec import KernelSpecManager kernel_json = { "argv": [sys.executable, "-m", "preql.jup_kernel", "-f", "{connection_file}"], @@ -12,9 +12,10 @@ "language": "preql", } + def install_my_kernel_spec(user=True, prefix=None): with TemporaryDirectory() as td: - os.chmod(td, 0o755) # Starts off as 700, not user readable + os.chmod(td, 0o755) # Starts off as 700, not user readable with open(os.path.join(td, 'kernel.json'), 'w') as f: json.dump(kernel_json, f, sort_keys=True) # TODO: Copy any resources @@ -22,20 +23,30 @@ def install_my_kernel_spec(user=True, prefix=None): print('Installing Jupyter kernel spec') KernelSpecManager().install_kernel_spec(td, 'preql', user=user, prefix=prefix) + def _is_root(): try: return os.geteuid() == 0 except AttributeError: - return False # assume not an admin on non-Unix platforms + return False # assume not an admin on non-Unix platforms + def main(argv=None): ap = argparse.ArgumentParser() - ap.add_argument('--user', action='store_true', - help="Install to the per-user kernels registry. Default if not root.") - ap.add_argument('--sys-prefix', action='store_true', - help="Install to sys.prefix (e.g. a virtualenv or conda env)") - ap.add_argument('--prefix', - help="Install to the given prefix. Kernelspec will be installed in {PREFIX}/share/jupyter/kernels/") + ap.add_argument( + '--user', + action='store_true', + help="Install to the per-user kernels registry. Default if not root.", + ) + ap.add_argument( + '--sys-prefix', + action='store_true', + help="Install to sys.prefix (e.g. a virtualenv or conda env)", + ) + ap.add_argument( + '--prefix', + help="Install to the given prefix. Kernelspec will be installed in {PREFIX}/share/jupyter/kernels/", + ) args = ap.parse_args(argv) if args.sys_prefix: @@ -45,5 +56,6 @@ def main(argv=None): install_my_kernel_spec(user=args.user, prefix=args.prefix) + if __name__ == '__main__': main() diff --git a/preql/jup_kernel/kernel.py b/preql/jup_kernel/kernel.py index d0c6c90..75e2b00 100644 --- a/preql/jup_kernel/kernel.py +++ b/preql/jup_kernel/kernel.py @@ -10,6 +10,7 @@ pql.set_output_format('html') display = pql._display + class PreqlKernel(Kernel): implementation = 'Preql' implementation_version = __version__ @@ -23,8 +24,9 @@ class PreqlKernel(Kernel): } banner = "Preql" - def do_execute(self, code, silent, store_history=True, user_expressions=None, - allow_stdin=False): + def do_execute( + self, code, silent, store_history=True, user_expressions=None, allow_stdin=False + ): if not silent: # r = requests.post("http://127.0.0.1:8080/html", code) # json = r.json() @@ -41,18 +43,11 @@ def do_execute(self, code, silent, store_history=True, user_expressions=None, elif res is not None: res = res.repr() - json = { - 'output': display.as_html() + str(res), - 'success': True - } + json = {'output': display.as_html() + str(res), 'success': True} except preql.Signal as e: display.print_exception(e) - json = { - 'output': display.as_html(), - 'success': False - } - + json = {'output': display.as_html(), 'success': False} # if json['success']: # stream_content = {'name': 'stdout', 'text': json['output']} @@ -62,26 +57,29 @@ def do_execute(self, code, silent, store_history=True, user_expressions=None, html = json['output'] # self.send_info("Elapsed Time: {} !\n".format(elapsed_time)) - self.send_response(self.iopub_socket, 'display_data', { - 'data': { - "text/html": html, - }, - "metadata": { - "image/png": { - "width": 640, - "height": 480, + self.send_response( + self.iopub_socket, + 'display_data', + { + 'data': { + "text/html": html, }, - } - }) - - - return {'status': 'ok', - # The base class increments the execution count - 'execution_count': self.execution_count, - 'payload': [], - 'user_expressions': {}, - } + "metadata": { + "image/png": { + "width": 640, + "height": 480, + }, + }, + }, + ) + return { + 'status': 'ok', + # The base class increments the execution count + 'execution_count': self.execution_count, + 'payload': [], + 'user_expressions': {}, + } def do_complete(self, code, cursor_pos): context, fragment = last_word(code[:cursor_pos]) @@ -90,22 +88,24 @@ def do_complete(self, code, cursor_pos): matches = [f'{k}' for k in all_vars if k.startswith(fragment)] return { - 'status' : 'ok', - 'matches' : matches, - 'cursor_start' : cursor_pos - len(fragment), - 'cursor_end' : cursor_pos, - 'metadata' : {}, + 'status': 'ok', + 'matches': matches, + 'cursor_start': cursor_pos - len(fragment), + 'cursor_end': cursor_pos, + 'metadata': {}, } def is_name(s): return s.isalnum() or s in ('_', '!') + + def last_word(s): if not s: return '', '' i = len(s) - while i and is_name(s[i-1]): + while i and is_name(s[i - 1]): i -= 1 - if i < len(s) and s[i] == '!' : + if i < len(s) and s[i] == '!': i += 1 # hack to support ... !var and !in - return s[:i], s[i:] \ No newline at end of file + return s[:i], s[i:] diff --git a/preql/loggers.py b/preql/loggers.py index 59ebcdd..35823c1 100644 --- a/preql/loggers.py +++ b/preql/loggers.py @@ -1,5 +1,14 @@ -from logging import getLogger, Formatter, StreamHandler, basicConfig -from logging import DEBUG, INFO, WARN, ERROR, CRITICAL +from logging import ( + CRITICAL, + DEBUG, + ERROR, + INFO, + WARN, + Formatter, + StreamHandler, + basicConfig, + getLogger, +) basicConfig(level=INFO, format="(%(levelname)s) %(name)s -- %(message)s") # )#datefmt='%m-%d %H:%M') @@ -7,6 +16,7 @@ sh = StreamHandler() sh.setFormatter(Formatter('%(message)s')) + def make_logger(name, level): logger = getLogger(name) logger.propagate = False @@ -15,7 +25,6 @@ def make_logger(name, level): return logger - sql_log = make_logger('sql_output', DEBUG) ac_log = make_logger('autocomplete', CRITICAL) repl_log = make_logger('repl', INFO) diff --git a/preql/repl.py b/preql/repl.py index e3c44b7..7a71cd8 100644 --- a/preql/repl.py +++ b/preql/repl.py @@ -1,54 +1,61 @@ -from pathlib import Path -from time import time - ### XXX Fix for Python 3.8 bug (https://github.com/prompt-toolkit/python-prompt-toolkit/issues/1023) import asyncio import selectors +from pathlib import Path +from time import time + selector = selectors.SelectSelector() loop = asyncio.SelectorEventLoop(selector) asyncio.set_event_loop(loop) ### XXX End of fix -from pygments.lexers.go import GoLexer -from pygments.style import Style -from pygments.token import Keyword, Name, Comment, String, Error, Number, Operator, Generic -from prompt_toolkit.styles.pygments import style_from_pygments_cls from prompt_toolkit import PromptSession -from prompt_toolkit.lexers import PygmentsLexer -from prompt_toolkit.filters import Condition -from prompt_toolkit.validation import Validator, ValidationError from prompt_toolkit.application.current import get_app +from prompt_toolkit.auto_suggest import AutoSuggestFromHistory from prompt_toolkit.completion import Completer, Completion +from prompt_toolkit.filters import Condition from prompt_toolkit.formatted_text.html import HTML, html_escape from prompt_toolkit.history import FileHistory -from prompt_toolkit.auto_suggest import AutoSuggestFromHistory +from prompt_toolkit.lexers import PygmentsLexer from prompt_toolkit.output.color_depth import ColorDepth - -from . import __version__, __branch__ -from . import settings -from .utils import memoize -from .loggers import repl_log +from prompt_toolkit.styles.pygments import style_from_pygments_cls +from prompt_toolkit.validation import ValidationError, Validator +from pygments.lexers.go import GoLexer +from pygments.style import Style +from pygments.token import ( + Comment, + Error, + Generic, + Keyword, + Name, + Number, + Operator, + String, +) + +from . import __branch__, __version__, settings from .context import context - -from .core.exceptions import Signal, ExitInterp, pql_SyntaxError -from .core.autocomplete import autocomplete -from .core.parser import parse_stmts from .core import pql_objects as objects +from .core.autocomplete import autocomplete from .core.display import table_more -from .core.pql_types import Object -from .core.pql_types import T +from .core.exceptions import ExitInterp, Signal, pql_SyntaxError +from .core.parser import parse_stmts +from .core.pql_types import Object, T +from .loggers import repl_log +from .utils import memoize def is_name(s): return s.isalnum() or s in ('_', '!') + def last_word(s): if not s: return '', '' i = len(s) - while i and is_name(s[i-1]): + while i and is_name(s[i - 1]): i -= 1 - if i < len(s) and s[i] == '!' : + if i < len(s) and s[i] == '!': i += 1 # hack to support ... !var and !in return s[:i], s[i:] @@ -84,7 +91,7 @@ def _get_completions(self, document): for k, (_rank, v) in all_vars: if k.startswith(fragment): - a, b = k[:len(fragment)], k[len(fragment):] + a, b = k[: len(fragment)], k[len(fragment) :] if v is None: t = "" else: @@ -94,15 +101,20 @@ def _get_completions(self, document): t = type(v) yield Completion( - b, start_position=0, - display=HTML('%s%s : %s' % (a, b, html_escape(t))), + b, + start_position=0, + display=HTML( + '%s%s : %s' + % (a, b, html_escape(t)) + ), style='bg:ansigray fg:black', selected_style="fg:black bg:ansibrightyellow", - ) + ) def get_completions(self, document, complete_event): return self._get_completions(document) + class MyValidator(Validator): def validate(self, document): text = document.text @@ -119,8 +131,9 @@ def validate(self, document): # if pos <= len(text): raise ValidationError(message=e.message, cursor_position=pos) # except Exception as e: - # raise ValidationError(message=e.args[0], cursor_position=0) - # pass + # raise ValidationError(message=e.args[0], cursor_position=0) + # pass + # from prompt_toolkit.key_binding import KeyBindings # kb = KeyBindings() @@ -153,29 +166,32 @@ def make_preql_style(): class PreqlStyle(Style): default_style = "" styles = { - Generic: settings.color_theme['text'], - Comment: settings.color_theme['comment'], - Keyword: settings.color_theme['keyword'], - Name: settings.color_theme['name'], - Name.Function: settings.color_theme['name_func'], - Name.Class: settings.color_theme['name_class'], - String: settings.color_theme['string'], - Number: settings.color_theme['number'], - Operator: settings.color_theme['operator'], - Error: 'bg:ansired ansigray', + Generic: settings.color_theme['text'], + Comment: settings.color_theme['comment'], + Keyword: settings.color_theme['keyword'], + Name: settings.color_theme['name'], + Name.Function: settings.color_theme['name_func'], + Name.Class: settings.color_theme['name_class'], + String: settings.color_theme['string'], + Number: settings.color_theme['number'], + Operator: settings.color_theme['operator'], + Error: 'bg:ansired ansigray', } + return PreqlStyle def start_repl(p, prompt=' >> '): - save_last = '_' # XXX A little hacky + save_last = '_' # XXX A little hacky p.set_output_format('rich') display = p._display interp = p._interp console = display.console - console.print(f"[purple]Preql {__version__}{__branch__} interactive prompt. Type help() for help[/purple]") + console.print( + f"[purple]Preql {__version__}{__branch__} interactive prompt. Type help() for help[/purple]" + ) try: session = PromptSession( @@ -201,7 +217,6 @@ def multiline_filter(): if not code.strip(): continue - start_time = time() try: if code == '.': @@ -223,9 +238,13 @@ def multiline_filter(): res_repr = res.repr() # repl_log.info(res) - if isinstance(res_repr, str) and res.type == T.string: # Not text + if ( + isinstance(res_repr, str) and res.type == T.string + ): # Not text if len(res_repr) > 200: - res_repr = res_repr[:100] + "..." + res_repr[-100:] # smarter limit? + res_repr = ( + res_repr[:100] + "..." + res_repr[-100:] + ) # smarter limit? display.print(res_repr) except Signal as s: @@ -247,7 +266,5 @@ def multiline_filter(): except KeyboardInterrupt: repl_log.info("Interrupted (Ctrl+C)") - - except (KeyboardInterrupt, EOFError): repl_log.info('Exiting Preql interaction') diff --git a/preql/settings.py b/preql/settings.py index e9c2843..becf9ca 100644 --- a/preql/settings.py +++ b/preql/settings.py @@ -1,4 +1,3 @@ - import sys optimize = True @@ -6,34 +5,36 @@ debug = False print_sql = False -typecheck = False # not sys.flags.optimize +typecheck = False # not sys.flags.optimize autocomplete = True color_theme = { - 'text' : '#c0c0c0', - 'comment' : 'italic #808080', - 'keyword' : 'bold #0060f0', - 'name' : '#f0f0f0', - 'name_func' : 'bold #f0f0f0', - 'name_class' : 'bold #f0f0f0', - 'string' : '#40f040', - 'number' : '#40f0f0', - 'operator' : '#c0c0c0', - 'error' : '#f04040', + 'text': '#c0c0c0', + 'comment': 'italic #808080', + 'keyword': 'bold #0060f0', + 'name': '#f0f0f0', + 'name_func': 'bold #f0f0f0', + 'name_class': 'bold #f0f0f0', + 'string': '#40f040', + 'number': '#40f0f0', + 'operator': '#c0c0c0', + 'error': '#f04040', } update_color_theme = {} + class Display: TABLE_PREVIEW_SIZE_SHELL = 16 TABLE_PREVIEW_SIZE_HTML = 64 LIST_PREVIEW_SIZE = 128 MAX_AUTO_COUNT = 10000 + try: from .local_settings import * except ImportError: pass -color_theme.update(update_color_theme) \ No newline at end of file +color_theme.update(update_color_theme) diff --git a/preql/sql_interface.py b/preql/sql_interface.py index e6e62cd..d2b0072 100644 --- a/preql/sql_interface.py +++ b/preql/sql_interface.py @@ -1,24 +1,34 @@ +import json import operator -from pathlib import Path import subprocess -import json +from pathlib import Path import dsnparse -from .utils import classify, dataclass -from .loggers import sql_log from .context import context - -from .core.sql import Sql, QueryBuilder, sqlite, postgres, mysql, duck, bigquery, quote_id -from .core.sql_import_result import sql_result_to_python, type_from_sql -from .core.pql_types import T, Type, Object, Id from .core.exceptions import DatabaseQueryError, Signal +from .core.pql_types import Id, Object, T, Type +from .core.sql import ( + QueryBuilder, + Sql, + bigquery, + duck, + mysql, + postgres, + quote_id, + sqlite, +) +from .core.sql_import_result import sql_result_to_python, type_from_sql +from .loggers import sql_log +from .utils import classify, dataclass + @dataclass class Const(Object): type: Type value: object + class ConnectError(Exception): pass @@ -26,7 +36,7 @@ class ConnectError(Exception): def log_sql(sql): for i, s in enumerate(sql.split('\n')): prefix = '/**/ ' if i else '/**/;; ' - sql_log.debug(prefix+s) + sql_log.debug(prefix + s) class SqlInterface: @@ -36,7 +46,6 @@ class SqlInterface: requires_subquery_name = False id_type_decl = 'INTEGER' - def __init__(self, print_sql=False): self._print_sql = print_sql @@ -81,12 +90,12 @@ def quote_name(self, name): return f'"{name}"' - # from multiprocessing import Queue import queue import threading from time import sleep + class TaskQueue: def __init__(self): self._queue = queue.Queue() @@ -113,13 +122,12 @@ def _worker(self): except queue.Empty: continue try: - res = item(*args, **kwargs) + res = item(*args, **kwargs) except Exception as e: res = e self._task_results[task_id] = res self._queue.task_done() - def _get_result(self, task_id): while task_id not in self._task_results: sleep(0.001) @@ -136,10 +144,10 @@ def close(self): self._closed = True - class BaseConnection: pass + class ThreadedConnection(BaseConnection): def __init__(self, create_connection): self._queue = TaskQueue() @@ -158,7 +166,7 @@ def _import_result(self, sql_type, c): res = c.fetchall() except Exception as e: msg = "Exception when trying to fetch SQL result. Got error: %s" - raise DatabaseQueryError(msg%(e)) + raise DatabaseQueryError(msg % (e)) return sql_result_to_python(Const(sql_type, res)) @@ -169,13 +177,14 @@ def _execute_sql(self, state, sql_type, sql_code): c = self._backend_execute_sql(sql_code) except Exception as e: msg = "Exception when trying to execute SQL code:\n %s\n\nGot error: %s" - raise DatabaseQueryError(msg%(sql_code, e)) + raise DatabaseQueryError(msg % (sql_code, e)) return self._import_result(sql_type, c) def execute_sql(self, sql_type, sql_code): - return self._queue.run_task(self._execute_sql, context.state, sql_type, sql_code) - + return self._queue.run_task( + self._execute_sql, context.state, sql_type, sql_code + ) def commit(self): self._queue.run_task(self._conn.commit) @@ -188,8 +197,6 @@ def close(self): self._queue.close() - - class SqlInterfaceCursor(SqlInterface): "An interface that uses the standard SQL cursor interface" @@ -198,19 +205,17 @@ def __init__(self, *a, **kw): self._conn = ThreadedConnection(self._create_connection) - def _execute_sql(self, sql_type, sql_code): return self._conn.execute_sql(sql_type, sql_code) def ping(self): c = self._conn.cursor() c.execute('select 1') - row ,= c.fetchall() - n ,= row + (row,) = c.fetchall() + (n,) = row assert n == 1 - class MysqlInterface(SqlInterfaceCursor): target = mysql @@ -220,8 +225,10 @@ class MysqlInterface(SqlInterfaceCursor): def __init__(self, host, port, database, user, password, print_sql=False): self._print_sql = print_sql - args = dict(host=host, port=port, database=database, user=user, password=password) - self._args = {k:v for k, v in args.items() if v is not None} + args = dict( + host=host, port=port, database=database, user=user, password=password + ) + self._args = {k: v for k, v in args.items() if v is not None} super().__init__(print_sql) def _create_connection(self): @@ -229,7 +236,9 @@ def _create_connection(self): from mysql.connector import errorcode try: - return mysql.connector.connect(charset='utf8', use_unicode=True, **self._args) + return mysql.connector.connect( + charset='utf8', use_unicode=True, **self._args + ) except mysql.connector.Error as e: if e.errno == errorcode.ER_ACCESS_DENIED_ERROR: raise ConnectError("Bad user name or password") from e @@ -250,16 +259,18 @@ def list_tables(self): def import_table_type(self, name, columns_whitelist=None): assert isinstance(name, Id) - assert len(name.parts) == 1 # TODO ! - - columns_t = T.table(dict( - name=T.string, - type=T.string, - nullable=T.string, - key=T.string, - default=T.string, - extra=T.string, - )) + assert len(name.parts) == 1 # TODO ! + + columns_t = T.table( + dict( + name=T.string, + type=T.string, + nullable=T.string, + key=T.string, + default=T.string, + extra=T.string, + ) + ) columns_q = "desc %s" % name.name sql_columns = self._execute_sql(columns_t, columns_q) @@ -267,7 +278,10 @@ def import_table_type(self, name, columns_whitelist=None): wl = set(columns_whitelist) sql_columns = [c for c in sql_columns if c['name'] in wl] - cols = {c['name']: type_from_sql(c['type'].decode(), c['nullable']) for c in sql_columns} + cols = { + c['name']: type_from_sql(c['type'].decode(), c['nullable']) + for c in sql_columns + } return T.table(cols, name=name) @@ -282,30 +296,32 @@ class PostgresInterface(SqlInterfaceCursor): requires_subquery_name = True def __init__(self, host, port, database, user, password, print_sql=False): - self.args = dict(host=host, port=port, database=database, user=user, password=password) + self.args = dict( + host=host, port=port, database=database, user=user, password=password + ) super().__init__(print_sql) def _create_connection(self): import psycopg2 import psycopg2.extras + psycopg2.extensions.set_wait_callback(psycopg2.extras.wait_select) try: return psycopg2.connect(**self.args) except psycopg2.OperationalError as e: raise ConnectError(*e.args) from e - - - def table_exists(self, table_id): assert isinstance(table_id, Id) if len(table_id.parts) == 1: schema = 'public' - name ,= table_id.parts + (name,) = table_id.parts elif len(table_id.parts) == 2: schema, name = table_id.parts else: - raise Signal.make(T.DbError, None, "Postgres doesn't support nested schemas") + raise Signal.make( + T.DbError, None, "Postgres doesn't support nested schemas" + ) sql_code = f"SELECT count(*) FROM information_schema.tables where table_name='{name}' and table_schema='{schema}'" cnt = self._execute_sql(T.int, sql_code) @@ -317,26 +333,29 @@ def list_tables(self): names = self._execute_sql(T.list[T.string], sql_code) return list(map(Id, names)) - - _schema_columns_t = T.table(dict( - schema=T.string, - table=T.string, - name=T.string, - pos=T.int, - nullable=T.bool, - type=T.string, - )) + _schema_columns_t = T.table( + dict( + schema=T.string, + table=T.string, + name=T.string, + pos=T.int, + nullable=T.bool, + type=T.string, + ) + ) def import_table_type(self, table_id, columns_whitelist=None): assert isinstance(table_id, Id) if len(table_id.parts) == 1: schema = 'public' - name ,= table_id.parts + (name,) = table_id.parts elif len(table_id.parts) == 2: schema, name = table_id.parts else: - raise Signal.make(T.DbError, None, "Postgres doesn't support nested schemas") + raise Signal.make( + T.DbError, None, "Postgres doesn't support nested schemas" + ) columns_q = f"""SELECT table_schema, table_name, column_name, ordinal_position, is_nullable, data_type FROM information_schema.columns @@ -348,7 +367,10 @@ def import_table_type(self, table_id, columns_whitelist=None): wl = set(columns_whitelist) sql_columns = [c for c in sql_columns if c['name'] in wl] - cols = [(c['pos'], c['name'], type_from_sql(c['type'], c['nullable'])) for c in sql_columns] + cols = [ + (c['pos'], c['name'], type_from_sql(c['type'], c['nullable'])) + for c in sql_columns + ] cols.sort() cols = dict(c[1:] for c in cols) @@ -363,7 +385,10 @@ def import_table_types(self): columns_by_table = classify(sql_columns, lambda c: (c['schema'], c['table'])) for (schema, table_name), columns in columns_by_table.items(): - cols = [(c['pos'], c['name'], type_from_sql(c['type'], c['nullable'])) for c in columns] + cols = [ + (c['pos'], c['name'], type_from_sql(c['type'], c['nullable'])) + for c in columns + ] cols.sort() cols = dict(c[1:] for c in cols) @@ -371,7 +396,6 @@ def import_table_types(self): yield schema, table_name, T.table(cols, name=Id(schema, table_name)) - class BigQueryInterface(SqlInterface): target = bigquery @@ -399,13 +423,13 @@ def ensure_dataset(self): if self._dataset_ensured: return - self._client.delete_dataset(self.PREQL_DATASET, delete_contents=True, not_found_ok=True) + self._client.delete_dataset( + self.PREQL_DATASET, delete_contents=True, not_found_ok=True + ) self._client.create_dataset(self.PREQL_DATASET) self._dataset_ensured = True - - def _list_tables(self): for ds in self._client.list_datasets(): for t in self._client.list_tables(ds.reference): @@ -425,26 +449,26 @@ def _execute_sql(self, sql_type, sql_code): res = list(self._client.query(sql_code)) except Exception as e: msg = "Exception when trying to execute SQL code:\n %s\n\nGot error: %s" - raise DatabaseQueryError(msg%(sql_code, e)) + raise DatabaseQueryError(msg % (sql_code, e)) if sql_type is not T.nulltype: res = [list(i.values()) for i in res] return sql_result_to_python(Const(sql_type, res)) - - - _schema_columns_t = T.table(dict( - schema=T.string, - table=T.string, - name=T.string, - pos=T.int, - nullable=T.bool, - type=T.string, - )) - + _schema_columns_t = T.table( + dict( + schema=T.string, + table=T.string, + name=T.string, + pos=T.int, + nullable=T.bool, + type=T.string, + ) + ) def get_table(self, name): - from google.api_core.exceptions import NotFound, BadRequest + from google.api_core.exceptions import BadRequest, NotFound + try: return self._client.get_table('.'.join(name.parts)) except ValueError as e: @@ -454,7 +478,6 @@ def get_table(self, name): except BadRequest as e: raise Signal.make(T.DbQueryError, None, str(e)) - def table_exists(self, name): try: self.get_table(name) @@ -462,7 +485,6 @@ def table_exists(self, name): return False return True - def import_table_type(self, name, columns_whitelist=None): assert isinstance(name, Id) @@ -514,6 +536,7 @@ def qualified_name(self, name): def rollback(self): # XXX No error? No warning? pass + def commit(self): # XXX No error? No warning? pass @@ -521,13 +544,17 @@ def commit(self): def close(self): self._client.close() + class AbsSqliteInterface: def table_exists(self, name): assert isinstance(name, Id), name if len(name.parts) > 1: raise Signal.make(T.DbError, None, "Sqlite does not implement namespaces") - sql_code = "SELECT count(*) FROM sqlite_master where name='%s' and type='table'" % name.name + sql_code = ( + "SELECT count(*) FROM sqlite_master where name='%s' and type='table'" + % name.name + ) cnt = self._execute_sql(T.int, sql_code) return cnt > 0 @@ -535,14 +562,16 @@ def list_tables(self): sql_code = "SELECT name FROM sqlite_master where type='table'" return self._execute_sql(T.list[T.string], sql_code) - table_schema_type = T.table(dict( - pos=T.int, - name=T.string, - type=T.string, - notnull=T.bool, - default_value=T.string, - pk=T.bool, - )) + table_schema_type = T.table( + dict( + pos=T.int, + name=T.string, + type=T.string, + notnull=T.bool, + default_value=T.string, + pk=T.bool, + ) + ) def import_table_type(self, name, columns_whitelist=None): assert isinstance(name, Id), name @@ -554,7 +583,10 @@ def import_table_type(self, name, columns_whitelist=None): wl = set(columns_whitelist) sql_columns = [c for c in sql_columns if c['name'] in wl] - cols = [(c['pos'], c['name'], type_from_sql(c['type'], not c['notnull'])) for c in sql_columns] + cols = [ + (c['pos'], c['name'], type_from_sql(c['type'], not c['notnull'])) + for c in sql_columns + ] cols.sort() cols = dict(c[1:] for c in cols) @@ -573,7 +605,10 @@ def step(self, value): def finalize(self): return self.product + import math + + class _SqliteStddev: def __init__(self): self.M = 0.0 @@ -591,8 +626,7 @@ def step(self, value): def finalize(self): if self.k < 3: return None - return math.sqrt(self.S / (self.k-2)) - + return math.sqrt(self.S / (self.k - 2)) class SqliteInterface(SqlInterfaceCursor, AbsSqliteInterface): @@ -602,9 +636,9 @@ def __init__(self, filename=None, print_sql=False): self._filename = filename super().__init__(print_sql) - def _create_connection(self): import sqlite3 + # sqlite3.enable_callback_tracebacks(True) try: conn = sqlite3.connect(self._filename or ':memory:') @@ -632,13 +666,14 @@ class DuckInterface(SqliteInterface): def _create_connection(self): import duckdb + return duckdb.connect(self._filename or ':memory:') def quote_name(self, name): return f'"{name}"' def rollback(self): - pass # XXX + pass # XXX class GitInterface(AbsSqliteInterface): @@ -650,15 +685,16 @@ def __init__(self, path, print_sql): self.path = path self._print_sql = print_sql - - table_schema_type = T.table(dict( - cid=T.int, - dflt_value=T.string, - name=T.string, - notnull=T.bool, - pk=T.bool, - type=T.string, - )) + table_schema_type = T.table( + dict( + cid=T.int, + dflt_value=T.string, + name=T.string, + notnull=T.bool, + pk=T.bool, + type=T.string, + ) + ) def _execute_sql(self, sql_type, sql_code): assert context.state @@ -669,7 +705,7 @@ def _execute_sql(self, sql_type, sql_code): raise DatabaseQueryError(msg) except subprocess.CalledProcessError as e: msg = "Exception when trying to execute SQL code:\n %s\n\nGot error: %s" - raise DatabaseQueryError(msg%(sql_code, e)) + raise DatabaseQueryError(msg % (sql_code, e)) return self._import_result(sql_type, res) @@ -687,13 +723,15 @@ def _import_result(self, sql_type, c): assert "PLACEHOLDER" not in x, (x, row) res.append(x) else: - res = [list(json.loads(x).values()) for x in c.split(b'\n') if x.strip()] + res = [ + list(json.loads(x).values()) for x in c.split(b'\n') if x.strip() + ] return sql_result_to_python(Const(sql_type, res)) def import_table_type(self, name, columns_whitelist=None): assert isinstance(name, Id) - assert len(name.parts) == 1 # TODO ! + assert len(name.parts) == 1 # TODO ! # TODO merge with superclass columns_q = """pragma table_info('%s')""" % name @@ -703,7 +741,10 @@ def import_table_type(self, name, columns_whitelist=None): wl = set(columns_whitelist) sql_columns = [c for c in sql_columns if c['name'] in wl] - cols = [(c['cid'], c['name'], type_from_sql(c['type'], not c['notnull'])) for c in sql_columns] + cols = [ + (c['cid'], c['name'], type_from_sql(c['type'], not c['notnull'])) + for c in sql_columns + ] cols.sort() cols = dict(c[1:] for c in cols) @@ -727,28 +768,36 @@ def _drop_tables(state, *tables): _SQLITE_SCHEME = 'sqlite://' + def create_engine(db_uri, print_sql, auto_create): if db_uri.startswith(_SQLITE_SCHEME): # Parse sqlite:// ourselves, to allow for sqlite://c:/path/to/db - path = db_uri[len(_SQLITE_SCHEME):] + path = db_uri[len(_SQLITE_SCHEME) :] if not auto_create and path != ':memory:': if not Path(path).exists(): - raise ConnectError("File %r doesn't exist. To create it, set auto_create to True" % path) + raise ConnectError( + "File %r doesn't exist. To create it, set auto_create to True" + % path + ) return SqliteInterface(path, print_sql=print_sql) dsn = dsnparse.parse(db_uri) if len(dsn.schemes) > 1: raise NotImplementedError("Preql doesn't support multiple schemes") - scheme ,= dsn.schemes + (scheme,) = dsn.schemes if len(dsn.paths) != 1: raise ValueError("Bad value for uri: %s" % db_uri) - path ,= dsn.paths + (path,) = dsn.paths if scheme == 'postgres': - return PostgresInterface(dsn.host, dsn.port, path, dsn.user, dsn.password, print_sql=print_sql) + return PostgresInterface( + dsn.host, dsn.port, path, dsn.user, dsn.password, print_sql=print_sql + ) elif scheme == 'mysql': - return MysqlInterface(dsn.host, dsn.port, path, dsn.user, dsn.password, print_sql=print_sql) + return MysqlInterface( + dsn.host, dsn.port, path, dsn.user, dsn.password, print_sql=print_sql + ) elif scheme == 'git': return GitInterface(path, print_sql=print_sql) elif scheme == 'duck': diff --git a/preql/utils.py b/preql/utils.py index 946c86b..29b8006 100644 --- a/preql/utils.py +++ b/preql/utils.py @@ -1,12 +1,11 @@ -import time import re +import time from collections import deque from contextlib import contextmanager -from pathlib import Path - -from typing import Optional from functools import wraps from operator import getitem +from pathlib import Path +from typing import Optional import runtype from rich.text import Text @@ -17,6 +16,7 @@ dataclass = runtype.dataclass(check_types=settings.typecheck) dsp = runtype.Dispatch() + class SafeDict(dict): def __setitem__(self, key, value): if key in self: @@ -31,11 +31,15 @@ def update(self, *other_dicts): self[k] = v return self + def merge_dicts(dicts): return SafeDict().update(*dicts) + def concat(*iters): return [elem for it in iters for elem in it] + + def concat_for(iters): return [elem for it in iters for elem in it] @@ -44,6 +48,7 @@ def safezip(*args): assert len(set(map(len, args))) == 1 return zip(*args) + def split_at_index(arr, idx): return arr[:idx], arr[idx:] @@ -52,10 +57,11 @@ def listgen(f): @wraps(f) def _f(*args, **kwargs): return list(f(*args, **kwargs)) + return _f -def find_duplicate(seq, key=lambda x:x): +def find_duplicate(seq, key=lambda x: x): "Returns the first duplicate item in given sequence, or None if not found" found = set() for i in seq: @@ -66,22 +72,23 @@ def find_duplicate(seq, key=lambda x:x): class _X: - def __init__(self, path = None): + def __init__(self, path=None): self.path = path or [] def __getattr__(self, attr): x = getattr, attr - return type(self)(self.path+[x]) + return type(self)(self.path + [x]) def __getitem__(self, item): x = getitem, item - return type(self)(self.path+[x]) + return type(self)(self.path + [x]) def __call__(self, obj): for f, p in self.path: obj = f(obj, p) return obj + X = _X() @@ -106,6 +113,7 @@ def measure_func(self, f): def _f(*args, **kwargs): with benchmark.measure(f.__name__): return f(*args, **kwargs) + return _f def reset(self): @@ -131,6 +139,7 @@ def classify_bool(seq, pred): return true_elems, false_elems + benchmark = Benchmark() @@ -146,7 +155,6 @@ def classify(seq, key=None, value=None): return d - # @dataclasses.dataclass class TextPos: char_index: int @@ -160,6 +168,7 @@ def __init__(self, char_index, line, column): self.line = line self.column = column + # @dataclasses.dataclass class TextRange: start: TextPos @@ -175,6 +184,7 @@ def __init__(self, start, end): def expand_tab(s): return s.replace('\t', ' ') + @mut_dataclass class TextReference: text: str @@ -197,8 +207,12 @@ def get_pinpoint_text(self, span=80, rich=False): mark_before = mark_after = 0 if self.context: pos = self.ref.start.char_index - mark_before = max(0, min(len(text_before), pos - self.context.start.char_index)) - mark_after = max(0, min(len(text_after), self.context.end.char_index - pos - 1)) + mark_before = max( + 0, min(len(text_before), pos - self.context.start.char_index) + ) + mark_after = max( + 0, min(len(text_after), self.context.end.char_index - pos - 1) + ) assert mark_before >= 0 and mark_after >= 0 source = Path(self.source_file) @@ -208,22 +222,36 @@ def get_pinpoint_text(self, span=80, rich=False): return [ f" [red]~~~[/red] file '{source.name}' line {start.line}, column {start.column}", Text(text_before + text_after), - Text(' ' * (len(text_before)-mark_before) + MARK_CHAR*mark_before + '^' + MARK_CHAR*mark_after), + Text( + ' ' * (len(text_before) - mark_before) + + MARK_CHAR * mark_before + + '^' + + MARK_CHAR * mark_after + ), ] res = [ - " ~~~ file '%s' line %d, column %d:\n" % (source.name, start.line, start.column), - text_before, text_after, '\n', - ' ' * (len(text_before)-mark_before), MARK_CHAR*mark_before, '^', MARK_CHAR*mark_after, '\n' + " ~~~ file '%s' line %d, column %d:\n" + % (source.name, start.line, start.column), + text_before, + text_after, + '\n', + ' ' * (len(text_before) - mark_before), + MARK_CHAR * mark_before, + '^', + MARK_CHAR * mark_after, + '\n', ] return ''.join(res) def __str__(self): return '' + def __repr__(self): return '' + def bfs(initial, expand): open_q = deque(list(initial)) visited = set(open_q) @@ -235,6 +263,7 @@ def bfs(initial, expand): visited.add(next_node) open_q.append(next_node) + def bfs_all_unique(initial, expand): open_q = deque(list(initial)) while open_q: @@ -260,15 +289,13 @@ def inner(*args): def re_split(r, s): offset = 0 for m in re.finditer(r, s): - yield None, s[offset:m.start()] - yield m, s[m.start():m.end()] + yield None, s[offset : m.start()] + yield m, s[m.start() : m.end()] offset = m.end() yield None, s[offset:] - def method(f): assert len(f.__annotations__) == 1 - cls ,= f.__annotations__.values() + (cls,) = f.__annotations__.values() setattr(cls, f.__name__, f) -