From 1a289914ef53c7ebcbded44b7e9bd8c6b728a560 Mon Sep 17 00:00:00 2001 From: Chris Riccomini Date: Tue, 29 Aug 2023 10:09:46 -0700 Subject: [PATCH] Add Recap gateway Recap now ships with a little HTTP/JSON gateway. The gateway has two paths: - ls - schema These paths list subpaths and fetch schemas. For example: ``` GET http://127.0.0.1:8000/ls ["bq", "pg"] GET http://127.0.0.1:8000/ls/pg ["postgres","template0","template1","testdb"] GET http://127.0.0.1:8000/schema/pg/testdb/public/test_types {"type": "struct", "fields": ... } ``` The gateway is configured using environment variableas and supports a `.env` file (via `pydantic-settings`). ```bash RECAP_SYSTEMS__BQ=bigquery:// RECAP_SYSTEMS__PG=postgresql://localhost:5432/testdb ``` In the future, Recap's CLI will use the same environment variables. I'm leaving the gateway integration tests for a follow-on PR. --- pdm.lock | 170 ++++++++++++++++++- pyproject.toml | 7 + recap/clients/dbapi.py | 2 +- recap/gateway/__init__.py | 0 recap/gateway/app.py | 48 ++++++ recap/gateway/settings.py | 11 ++ tests/integration/clients/test_mysql.py | 2 +- tests/integration/clients/test_postgresql.py | 2 +- tests/unit/clients/test_snowflake.py | 2 +- 9 files changed, 235 insertions(+), 9 deletions(-) create mode 100644 recap/gateway/__init__.py create mode 100644 recap/gateway/app.py create mode 100644 recap/gateway/settings.py diff --git a/pdm.lock b/pdm.lock index a18d4568..30b45f56 100644 --- a/pdm.lock +++ b/pdm.lock @@ -2,11 +2,21 @@ # It is not intended for manual editing. [metadata] -groups = ["default", "bigquery", "hive", "json", "kafka", "proto", "style", "tests"] +groups = ["default", "bigquery", "hive", "json", "kafka", "proto", "style", "tests", "gateway"] cross_platform = true static_urls = false lock_version = "4.3" -content_hash = "sha256:59013da422ec59a31f06d6d69f8bbbcb31a5b4b49d0bda5d7e5de610488c16cc" +content_hash = "sha256:0cad20a8f1436dcb71a75d23c70b38084c2e412c2b00924bb0e9999ea1774661" + +[[package]] +name = "annotated-types" +version = "0.5.0" +requires_python = ">=3.7" +summary = "Reusable constraint types to use with typing.Annotated" +files = [ + {file = "annotated_types-0.5.0-py3-none-any.whl", hash = "sha256:58da39888f92c276ad970249761ebea80ba544b77acddaa1a4d6cf78287d45fd"}, + {file = "annotated_types-0.5.0.tar.gz", hash = "sha256:47cdc3490d9ac1506ce92c7aaa76c579dc3509ff11e098fc867e5130ab7be802"}, +] [[package]] name = "antlr4-python3-runtime" @@ -359,6 +369,21 @@ files = [ {file = "fakesnow-0.6.0.tar.gz", hash = "sha256:80729907940bd65dab24b9f852992a14d8a218b1d192ec27a1f0a85a34c0ba1e"}, ] +[[package]] +name = "fastapi" +version = "0.103.0" +requires_python = ">=3.7" +summary = "FastAPI framework, high performance, easy to learn, fast to code, ready for production" +dependencies = [ + "pydantic!=1.8,!=1.8.1,!=2.0.0,!=2.0.1,!=2.1.0,<3.0.0,>=1.7.4", + "starlette<0.28.0,>=0.27.0", + "typing-extensions>=4.5.0", +] +files = [ + {file = "fastapi-0.103.0-py3-none-any.whl", hash = "sha256:61ab72c6c281205dd0cbaccf503e829a37e0be108d965ac223779a8479243665"}, + {file = "fastapi-0.103.0.tar.gz", hash = "sha256:4166732f5ddf61c33e9fa4664f73780872511e0598d4d5434b1816dc1e6d9421"}, +] + [[package]] name = "filelock" version = "3.12.2" @@ -985,6 +1010,103 @@ files = [ {file = "pycryptodomex-3.18.0.tar.gz", hash = "sha256:3e3ecb5fe979e7c1bb0027e518340acf7ee60415d79295e5251d13c68dde576e"}, ] +[[package]] +name = "pydantic" +version = "2.3.0" +requires_python = ">=3.7" +summary = "Data validation using Python type hints" +dependencies = [ + "annotated-types>=0.4.0", + "pydantic-core==2.6.3", + "typing-extensions>=4.6.1", +] +files = [ + {file = "pydantic-2.3.0-py3-none-any.whl", hash = "sha256:45b5e446c6dfaad9444819a293b921a40e1db1aa61ea08aede0522529ce90e81"}, + {file = "pydantic-2.3.0.tar.gz", hash = "sha256:1607cc106602284cd4a00882986570472f193fde9cb1259bceeaedb26aa79a6d"}, +] + +[[package]] +name = "pydantic-core" +version = "2.6.3" +requires_python = ">=3.7" +summary = "" +dependencies = [ + "typing-extensions!=4.7.0,>=4.6.0", +] +files = [ + {file = "pydantic_core-2.6.3-cp310-cp310-macosx_10_7_x86_64.whl", hash = "sha256:1a0ddaa723c48af27d19f27f1c73bdc615c73686d763388c8683fe34ae777bad"}, + {file = "pydantic_core-2.6.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5cfde4fab34dd1e3a3f7f3db38182ab6c95e4ea91cf322242ee0be5c2f7e3d2f"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5493a7027bfc6b108e17c3383959485087d5942e87eb62bbac69829eae9bc1f7"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:84e87c16f582f5c753b7f39a71bd6647255512191be2d2dbf49458c4ef024588"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:522a9c4a4d1924facce7270c84b5134c5cabcb01513213662a2e89cf28c1d309"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:aaafc776e5edc72b3cad1ccedb5fd869cc5c9a591f1213aa9eba31a781be9ac1"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a750a83b2728299ca12e003d73d1264ad0440f60f4fc9cee54acc489249b728"}, + {file = "pydantic_core-2.6.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9e8b374ef41ad5c461efb7a140ce4730661aadf85958b5c6a3e9cf4e040ff4bb"}, + {file = "pydantic_core-2.6.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b594b64e8568cf09ee5c9501ede37066b9fc41d83d58f55b9952e32141256acd"}, + {file = "pydantic_core-2.6.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2a20c533cb80466c1d42a43a4521669ccad7cf2967830ac62c2c2f9cece63e7e"}, + {file = "pydantic_core-2.6.3-cp310-none-win32.whl", hash = "sha256:04fe5c0a43dec39aedba0ec9579001061d4653a9b53a1366b113aca4a3c05ca7"}, + {file = "pydantic_core-2.6.3-cp310-none-win_amd64.whl", hash = "sha256:6bf7d610ac8f0065a286002a23bcce241ea8248c71988bda538edcc90e0c39ad"}, + {file = "pydantic_core-2.6.3-cp311-cp311-macosx_10_7_x86_64.whl", hash = "sha256:6bcc1ad776fffe25ea5c187a028991c031a00ff92d012ca1cc4714087e575973"}, + {file = "pydantic_core-2.6.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:df14f6332834444b4a37685810216cc8fe1fe91f447332cd56294c984ecbff1c"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0b7486d85293f7f0bbc39b34e1d8aa26210b450bbd3d245ec3d732864009819"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a892b5b1871b301ce20d40b037ffbe33d1407a39639c2b05356acfef5536d26a"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:883daa467865e5766931e07eb20f3e8152324f0adf52658f4d302242c12e2c32"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d4eb77df2964b64ba190eee00b2312a1fd7a862af8918ec70fc2d6308f76ac64"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ce8c84051fa292a5dc54018a40e2a1926fd17980a9422c973e3ebea017aa8da"}, + {file = "pydantic_core-2.6.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:22134a4453bd59b7d1e895c455fe277af9d9d9fbbcb9dc3f4a97b8693e7e2c9b"}, + {file = "pydantic_core-2.6.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:02e1c385095efbd997311d85c6021d32369675c09bcbfff3b69d84e59dc103f6"}, + {file = "pydantic_core-2.6.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d79f1f2f7ebdb9b741296b69049ff44aedd95976bfee38eb4848820628a99b50"}, + {file = "pydantic_core-2.6.3-cp311-none-win32.whl", hash = "sha256:430ddd965ffd068dd70ef4e4d74f2c489c3a313adc28e829dd7262cc0d2dd1e8"}, + {file = "pydantic_core-2.6.3-cp311-none-win_amd64.whl", hash = "sha256:84f8bb34fe76c68c9d96b77c60cef093f5e660ef8e43a6cbfcd991017d375950"}, + {file = "pydantic_core-2.6.3-cp311-none-win_arm64.whl", hash = "sha256:5a2a3c9ef904dcdadb550eedf3291ec3f229431b0084666e2c2aa8ff99a103a2"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-macosx_10_7_x86_64.whl", hash = "sha256:d7050899026e708fb185e174c63ebc2c4ee7a0c17b0a96ebc50e1f76a231c057"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:99faba727727b2e59129c59542284efebbddade4f0ae6a29c8b8d3e1f437beb7"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5fa159b902d22b283b680ef52b532b29554ea2a7fc39bf354064751369e9dbd7"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:046af9cfb5384f3684eeb3f58a48698ddab8dd870b4b3f67f825353a14441418"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:930bfe73e665ebce3f0da2c6d64455098aaa67e1a00323c74dc752627879fc67"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:85cc4d105747d2aa3c5cf3e37dac50141bff779545ba59a095f4a96b0a460e70"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:b25afe9d5c4f60dcbbe2b277a79be114e2e65a16598db8abee2a2dcde24f162b"}, + {file = "pydantic_core-2.6.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e49ce7dc9f925e1fb010fc3d555250139df61fa6e5a0a95ce356329602c11ea9"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-macosx_10_7_x86_64.whl", hash = "sha256:2dd50d6a1aef0426a1d0199190c6c43ec89812b1f409e7fe44cb0fbf6dfa733c"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6595b0d8c8711e8e1dc389d52648b923b809f68ac1c6f0baa525c6440aa0daa"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ef724a059396751aef71e847178d66ad7fc3fc969a1a40c29f5aac1aa5f8784"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:3c8945a105f1589ce8a693753b908815e0748f6279959a4530f6742e1994dcb6"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:c8c6660089a25d45333cb9db56bb9e347241a6d7509838dbbd1931d0e19dbc7f"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:692b4ff5c4e828a38716cfa92667661a39886e71136c97b7dac26edef18767f7"}, + {file = "pydantic_core-2.6.3-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:f1a5d8f18877474c80b7711d870db0eeef9442691fcdb00adabfc97e183ee0b0"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-macosx_10_7_x86_64.whl", hash = "sha256:3796a6152c545339d3b1652183e786df648ecdf7c4f9347e1d30e6750907f5bb"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-macosx_11_0_arm64.whl", hash = "sha256:b962700962f6e7a6bd77e5f37320cabac24b4c0f76afeac05e9f93cf0c620014"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:56ea80269077003eaa59723bac1d8bacd2cd15ae30456f2890811efc1e3d4413"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75c0ebbebae71ed1e385f7dfd9b74c1cff09fed24a6df43d326dd7f12339ec34"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:252851b38bad3bfda47b104ffd077d4f9604a10cb06fe09d020016a25107bf98"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:6656a0ae383d8cd7cc94e91de4e526407b3726049ce8d7939049cbfa426518c8"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:d9140ded382a5b04a1c030b593ed9bf3088243a0a8b7fa9f071a5736498c5483"}, + {file = "pydantic_core-2.6.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:d38bbcef58220f9c81e42c255ef0bf99735d8f11edef69ab0b499da77105158a"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-macosx_10_7_x86_64.whl", hash = "sha256:c9d469204abcca28926cbc28ce98f28e50e488767b084fb3fbdf21af11d3de26"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-macosx_11_0_arm64.whl", hash = "sha256:48c1ed8b02ffea4d5c9c220eda27af02b8149fe58526359b3c07eb391cb353a2"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b2b1bfed698fa410ab81982f681f5b1996d3d994ae8073286515ac4d165c2e7"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf9d42a71a4d7a7c1f14f629e5c30eac451a6fc81827d2beefd57d014c006c4a"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:4292ca56751aebbe63a84bbfc3b5717abb09b14d4b4442cc43fd7c49a1529efd"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-musllinux_1_1_aarch64.whl", hash = "sha256:7dc2ce039c7290b4ef64334ec7e6ca6494de6eecc81e21cb4f73b9b39991408c"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-musllinux_1_1_x86_64.whl", hash = "sha256:615a31b1629e12445c0e9fc8339b41aaa6cc60bd53bf802d5fe3d2c0cda2ae8d"}, + {file = "pydantic_core-2.6.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1fa1f6312fb84e8c281f32b39affe81984ccd484da6e9d65b3d18c202c666149"}, + {file = "pydantic_core-2.6.3.tar.gz", hash = "sha256:1508f37ba9e3ddc0189e6ff4e2228bd2d3c3a4641cbe8c07177162f76ed696c7"}, +] + +[[package]] +name = "pydantic-settings" +version = "2.0.3" +requires_python = ">=3.7" +summary = "Settings management using Pydantic" +dependencies = [ + "pydantic>=2.0.1", + "python-dotenv>=0.21.0", +] +files = [ + {file = "pydantic_settings-2.0.3-py3-none-any.whl", hash = "sha256:ddd907b066622bd67603b75e2ff791875540dc485b7307c4fffc015719da8625"}, + {file = "pydantic_settings-2.0.3.tar.gz", hash = "sha256:962dc3672495aad6ae96a4390fac7e593591e144625e5112d359f8f67fb75945"}, +] + [[package]] name = "pyflakes" version = "3.1.0" @@ -1096,6 +1218,16 @@ files = [ {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, ] +[[package]] +name = "python-dotenv" +version = "1.0.0" +requires_python = ">=3.8" +summary = "Read key-value pairs from a .env file and set them as environment variables" +files = [ + {file = "python-dotenv-1.0.0.tar.gz", hash = "sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba"}, + {file = "python_dotenv-1.0.0-py3-none-any.whl", hash = "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"}, +] + [[package]] name = "pytz" version = "2023.3" @@ -1302,6 +1434,19 @@ files = [ {file = "sqlglot-16.8.1.tar.gz", hash = "sha256:2a7451f0ed2feac6e7a832bafee852a25e743c35a637e5020f3235c1feb9b700"}, ] +[[package]] +name = "starlette" +version = "0.27.0" +requires_python = ">=3.7" +summary = "The little ASGI library that shines." +dependencies = [ + "anyio<5,>=3.4.0", +] +files = [ + {file = "starlette-0.27.0-py3-none-any.whl", hash = "sha256:918416370e846586541235ccd38a474c08b80443ed31c578a418e2209b3eef91"}, + {file = "starlette-0.27.0.tar.gz", hash = "sha256:6a6b0d042acb8d469a01eba54e9cda6cbd24ac602c4cd016723117d6a7e73b75"}, +] + [[package]] name = "thrift" version = "0.16.0" @@ -1335,12 +1480,12 @@ files = [ [[package]] name = "typing-extensions" -version = "4.4.0" +version = "4.7.1" requires_python = ">=3.7" summary = "Backported and Experimental Type Hints for Python 3.7+" files = [ - {file = "typing_extensions-4.4.0-py3-none-any.whl", hash = "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e"}, - {file = "typing_extensions-4.4.0.tar.gz", hash = "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa"}, + {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, + {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, ] [[package]] @@ -1353,6 +1498,21 @@ files = [ {file = "urllib3-1.26.16.tar.gz", hash = "sha256:8f135f6502756bde6b2a9b28989df5fbe87c9970cecaa69041edcce7f0589b14"}, ] +[[package]] +name = "uvicorn" +version = "0.23.2" +requires_python = ">=3.8" +summary = "The lightning-fast ASGI server." +dependencies = [ + "click>=7.0", + "h11>=0.8", + "typing-extensions>=4.0; python_version < \"3.11\"", +] +files = [ + {file = "uvicorn-0.23.2-py3-none-any.whl", hash = "sha256:1f9be6558f01239d4fdf22ef8126c39cb1ad0addf76c40e760549d2c2f43ab53"}, + {file = "uvicorn-0.23.2.tar.gz", hash = "sha256:4d3cc12d7727ba72b64d12d3cc7743124074c0a69f7b201512fc50c3e3f1569a"}, +] + [[package]] name = "wrapt" version = "1.14.1" diff --git a/pyproject.toml b/pyproject.toml index c5a3daee..2cad7868 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,12 @@ json = [ "referencing>=0.30.0", "httpx>=0.24.1", ] +gateway = [ + "fastapi>=0.103.0", + "pydantic>=2.3.0", + "pydantic-settings>=2.0.3", + "uvicorn>=0.23.2", +] [build-system] requires = ["pdm-backend"] @@ -107,6 +113,7 @@ unit = "pytest tests/unit -vv" spec = "pytest tests/spec -vv" integration = "pytest tests/integration -vv" test = {composite = ["unit", "spec"]} +serve ="uvicorn recap.gateway.app:app --reload" [tool.pytest.ini_options] addopts = [ diff --git a/recap/clients/dbapi.py b/recap/clients/dbapi.py index a24fb164..54c5a353 100644 --- a/recap/clients/dbapi.py +++ b/recap/clients/dbapi.py @@ -58,7 +58,7 @@ def ls_tables(self, catalog: str, schema: str) -> list[str]: ) return [row[0] for row in cursor.fetchall()] - def get_schema(self, table: str, schema: str, catalog: str) -> StructType: + def get_schema(self, catalog: str, schema: str, table: str) -> StructType: cursor = self.connection.cursor() cursor.execute( f""" diff --git a/recap/gateway/__init__.py b/recap/gateway/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/recap/gateway/app.py b/recap/gateway/app.py new file mode 100644 index 00000000..236a4ceb --- /dev/null +++ b/recap/gateway/app.py @@ -0,0 +1,48 @@ +from fastapi import Depends, FastAPI + +from recap.clients import Client, create_client +from recap.gateway.settings import RecapSettings +from recap.types import to_dict + +app = FastAPI() +settings = RecapSettings() + + +async def get_reader(system_name: str | None = None): + if system_name and (url := settings.systems.get(system_name)): + with create_client(url.unicode_string()) as client: + yield client + else: + yield None + + +@app.get("/ls") +@app.get("/ls/{system_name}") +@app.get("/ls/{system_name}/{path:path}") +async def ls( + system_name: str | None = None, + path: str | None = None, + client: Client | None = Depends(get_reader), +) -> list[str]: + if system_name is None: + return list(settings.systems.keys()) + if client is None: + raise ValueError(f"Unknown system: {system_name}") + return client.ls(*_args(path)) + + +@app.get("/schema/{system_name}/{path:path}") +async def schema( + path: str, + client: Client = Depends(get_reader), +) -> dict: + print(path) + recap_struct = client.get_schema(*_args(path)) + recap_dict = to_dict(recap_struct) + if not isinstance(recap_dict, dict): + raise ValueError(f"Expected dict, got {type(recap_dict)}") + return recap_dict + + +def _args(path: str | None) -> list[str]: + return path.strip("/").split("/") if path else [] diff --git a/recap/gateway/settings.py b/recap/gateway/settings.py new file mode 100644 index 00000000..9778508b --- /dev/null +++ b/recap/gateway/settings.py @@ -0,0 +1,11 @@ +from pydantic import AnyUrl, Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class RecapSettings(BaseSettings): + systems: dict[str, AnyUrl] = Field(default_factory=dict) + model_config = SettingsConfigDict( + env_file=".env", + env_prefix="recap_", + env_nested_delimiter="__", + ) diff --git a/tests/integration/clients/test_mysql.py b/tests/integration/clients/test_mysql.py index 1521e9d6..f0638fa0 100644 --- a/tests/integration/clients/test_mysql.py +++ b/tests/integration/clients/test_mysql.py @@ -77,7 +77,7 @@ def test_struct_method(self): client = MysqlClient(self.connection) # type: ignore # Test 'test_types' table. MySQL catalog is always 'def'. - test_types_struct = client.get_schema("test_types", "testdb", "def") + test_types_struct = client.get_schema("def", "testdb", "test_types") # Define the expected output for 'test_types' table expected_fields = [ diff --git a/tests/integration/clients/test_postgresql.py b/tests/integration/clients/test_postgresql.py index 265c9972..8a18a3fe 100644 --- a/tests/integration/clients/test_postgresql.py +++ b/tests/integration/clients/test_postgresql.py @@ -67,7 +67,7 @@ def test_struct_method(self): client = PostgresqlClient(self.connection) # type: ignore # Test 'test_types' table - test_types_struct = client.get_schema("test_types", "public", "testdb") + test_types_struct = client.get_schema("testdb", "public", "test_types") # Define the expected output for 'test_types' table expected_fields = [ diff --git a/tests/unit/clients/test_snowflake.py b/tests/unit/clients/test_snowflake.py index e684aa5b..2b9e0e77 100644 --- a/tests/unit/clients/test_snowflake.py +++ b/tests/unit/clients/test_snowflake.py @@ -66,7 +66,7 @@ def setup_class(cls): def test_struct_method(self): client = SnowflakeClient(self.connection) # type: ignore - test_types_struct = client.get_schema("TEST_TYPES", "PUBLIC", "TESTDB") + test_types_struct = client.get_schema("TESTDB", "PUBLIC", "TEST_TYPES") expected_fields = [ UnionType( default=None,