diff --git a/Makefile b/Makefile index 63a3ab54..0e9a79aa 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,10 @@ devdocker: ## Builds the docker for dev .PHONY:init init: devdocker up setup-pre-commit ## Initializes the local developer environment +.PHONY:build +build: ## Builds the docker for dev + docker compose build --progress=plain --parallel + .PHONY: up up: Pipfile.lock docker compose up --build -d @@ -35,9 +39,10 @@ run-pre-commit: setup-pre-commit ./.git/hooks/pre-commit .PHONY:update -update: down Pipfile.lock setup-pre-commit ## Updates all the packages using Pipfile +update: Pipfile.lock setup-pre-commit ## Updates all the packages using Pipfile docker compose run --rm --name sfs_pipenv dev pipenv sync --dev && \ - make devdocker + make devdocker && \ + make pipenv-setup .PHONY:tests tests: up @@ -51,6 +56,11 @@ sphinx-html: @touch docs/.nojekyll cp -a docsrc/_build/html/. docs +.PHONY:pipenv-setup +pipenv-setup:devdocker ## Run pipenv-setup to update setup.py with latest dependencies + docker compose run --rm --name spark_data_frame_comparer dev sh -c "pipenv run pipenv install --skip-lock --categories \"pipenvsetup\" && pipenv run pipenv-setup sync --pipfile" && \ + make run-pre-commit + .PHONY:shell shell:devdocker ## Brings up the bash shell in dev docker docker compose run --rm --name sfs_shell dev /bin/bash diff --git a/Pipfile b/Pipfile index 1a5b6ae5..d749349e 100644 --- a/Pipfile +++ b/Pipfile @@ -44,6 +44,16 @@ lxml=">=4.9.1" jinja2=">=3.1.2" #jsonpickle=">=2.2.0" +# These dependencies are required for pipenv-setup. They conflict with ones above, so we install these +# only when running pipenv-setup +[pipenvsetup] +# vistr is needed for visualizing the dependency graph +vistir=">=0.6.1, <0.7.0" # https://github.com/Madoshakalaka/pipenv-setup/issues/138 +# plete is needed for tab completion +plette = "<1.0.0" # https://github.com/Madoshakalaka/pipenv-setup/issues/138 +# pipenv-setup is needed for updating setup.py with the dependencies for anyone installing this package +pipenv-setup = ">=3.2.0" + [requires] python_version = "3.12" diff --git a/Pipfile.lock b/Pipfile.lock index 6737e2ef..63cc23ba 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "9776e7e7e4cbf20237e9db8e78dca3b91b2fde5f777680124958bcbfae552008" + "sha256": "ded50c361e570185d5057bc7ce56bd3034198973f8807eccc881836457214d0f" }, "pipfile-spec": 6, "requires": { @@ -379,12 +379,12 @@ }, "importlib-metadata": { "hashes": [ - "sha256:11901fa0c2f97919b288679932bb64febaeacf289d18ac84dd68cb2e74213369", - "sha256:72e8d4399996132204f9a16dcc751af254a48f8d1b20b9ff0f98d4a8f901e73d" + "sha256:66f342cc6ac9818fc6ff340576acd24d65ba0b3efabb2b4ac08b598965a4a2f1", + "sha256:9a547d3bc3608b025f93d403fdd1aae741c24fbb8314df4b155675742ce303c5" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==8.2.0" + "version": "==8.4.0" }, "iniconfig": { "hashes": [ @@ -404,11 +404,11 @@ }, "jaraco.context": { "hashes": [ - "sha256:3e16388f7da43d384a1a7cd3452e72e14732ac9fe459678773a3608a812bf266", - "sha256:c2f67165ce1f9be20f32f650f25d8edfc1646a8aeee48ae06fb35f90763576d2" + "sha256:9bae4ea555cf0b14938dc0aee7c9f32ed303aa20a3b73e7dc80111628792d1b3", + "sha256:f797fc481b490edb305122c9181830a3a5b76d84ef6d1aef2fb9b47ab956f9e4" ], "markers": "python_version >= '3.8'", - "version": "==5.3.0" + "version": "==6.0.1" }, "jaraco.functools": { "hashes": [ @@ -959,12 +959,12 @@ }, "setuptools": { "hashes": [ - "sha256:80aacbf633704e9c8bfa1d99fa5dd4dc59573efcf9e4042c13d3bcef91ac2ef9", - "sha256:f11dd94b7bae3a156a95ec151f24e4637fb4fa19c878e4d191bfb8b2d82728c4" + "sha256:b208925fcb9f7af924ed2dc04708ea89791e24bde0d3020b27df0e116088b34e", + "sha256:d59a3e788ab7e012ab2c4baed1b376da6366883ee20d7a5fc426816e3d7b1193" ], "index": "pypi", "markers": "python_version >= '3.8'", - "version": "==72.2.0" + "version": "==73.0.1" }, "snowballstemmer": { "hashes": [ @@ -1124,5 +1124,327 @@ "markers": "python_version >= '3.8'", "version": "==3.20.0" } + }, + "pipenvsetup": { + "attrs": { + "hashes": [ + "sha256:5cfb1b9148b5b086569baec03f20d7b6bf3bcacc9a42bebf87ffaaca362f6346", + "sha256:81921eb96de3191c8258c199618104dd27ac608d9366f5e35d011eae1867ede2" + ], + "markers": "python_version >= '3.7'", + "version": "==24.2.0" + }, + "cached-property": { + "hashes": [ + "sha256:9fa5755838eecbb2d234c3aa390bd80fbd3ac6b6869109bfc1b499f7bd89a130", + "sha256:df4f613cf7ad9a588cc381aaf4a512d26265ecebd5eb9e1ba12f1319eb85a6a0" + ], + "version": "==1.5.2" + }, + "cerberus": { + "hashes": [ + "sha256:7649a5815024d18eb7c6aa5e7a95355c649a53aacfc9b050e9d0bf6bfa2af372", + "sha256:81011e10266ef71b6ec6d50e60171258a5b134d69f8fb387d16e4936d0d47642" + ], + "version": "==1.3.5" + }, + "certifi": { + "hashes": [ + "sha256:5a1e7645bc0ec61a09e26c36f6106dd4cf40c6db3a1fb6352b0244e7fb057c7b", + "sha256:c198e21b1289c2ab85ee4e67bb4b4ef3ead0892059901a8d5b622f24a1101e90" + ], + "markers": "python_version >= '3.6'", + "version": "==2024.7.4" + }, + "chardet": { + "hashes": [ + "sha256:0368df2bfd78b5fc20572bb4e9bb7fb53e2c094f60ae9993339e8671d0afb8aa", + "sha256:d3e64f022d254183001eccc5db4040520c0f23b1a3f33d6413e099eb7f126557" + ], + "markers": "python_version >= '3.6'", + "version": "==5.0.0" + }, + "charset-normalizer": { + "hashes": [ + "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027", + "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087", + "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786", + "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8", + "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09", + "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185", + "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574", + "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e", + "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519", + "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898", + "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269", + "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3", + "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f", + "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6", + "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8", + "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a", + "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73", + "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc", + "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714", + "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2", + "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc", + "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce", + "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d", + "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e", + "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6", + "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269", + "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96", + "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d", + "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a", + "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4", + "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77", + "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d", + "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0", + "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed", + "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068", + "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac", + "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25", + "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8", + "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab", + "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26", + "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2", + "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db", + "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f", + "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5", + "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99", + "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c", + "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d", + "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811", + "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa", + "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a", + "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03", + "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b", + "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04", + "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c", + "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001", + "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458", + "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389", + "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99", + "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985", + "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537", + "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238", + "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f", + "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d", + "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796", + "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a", + "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143", + "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8", + "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c", + "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5", + "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5", + "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711", + "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4", + "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6", + "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c", + "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7", + "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4", + "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b", + "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae", + "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12", + "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c", + "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae", + "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8", + "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887", + "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b", + "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4", + "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f", + "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5", + "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33", + "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519", + "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561" + ], + "markers": "python_full_version >= '3.7.0'", + "version": "==3.3.2" + }, + "colorama": { + "hashes": [ + "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", + "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", + "version": "==0.4.6" + }, + "distlib": { + "hashes": [ + "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784", + "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64" + ], + "version": "==0.3.8" + }, + "idna": { + "hashes": [ + "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc", + "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0" + ], + "markers": "python_version >= '3.5'", + "version": "==3.7" + }, + "orderedmultidict": { + "hashes": [ + "sha256:04070bbb5e87291cc9bfa51df413677faf2141c73c61d2a5f7b26bea3cd882ad", + "sha256:43c839a17ee3cdd62234c47deca1a8508a3f2ca1d0678a3bf791c87cf84adbf3" + ], + "version": "==1.0.1" + }, + "packaging": { + "hashes": [ + "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5", + "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.9" + }, + "pep517": { + "hashes": [ + "sha256:1b2fa2ffd3938bb4beffe5d6146cbcb2bda996a5a4da9f31abffd8b24e07b317", + "sha256:31b206f67165b3536dd577c5c3f1518e8fbaf38cbc57efff8369a392feff1721" + ], + "markers": "python_version >= '3.6'", + "version": "==0.13.1" + }, + "pip": { + "hashes": [ + "sha256:2cd581cf58ab7fcfca4ce8efa6dcacd0de5bf8d0a3eb9ec927e07405f4d9e2a2", + "sha256:5b5e490b5e9cb275c879595064adce9ebd31b854e3e803740b72f9ccf34a45b8" + ], + "markers": "python_version >= '3.8'", + "version": "==24.2" + }, + "pip-shims": { + "hashes": [ + "sha256:089e3586a92b1b8dbbc16b2d2859331dc1c412d3e3dbcd91d80e6b30d73db96c", + "sha256:2ae9f21c0155ca5c37d2734eb5f9a7d98c4c42a122d1ba3eddbacc9d9ea9fbae" + ], + "markers": "python_version >= '3.6'", + "version": "==0.7.3" + }, + "pipenv-setup": { + "hashes": [ + "sha256:0def7ec3363f58b38a43dc59b2078fcee67b47301fd51a41b8e34e6f79812b1a", + "sha256:6ceda7145a3088494d8ca68fded4b0473022dc62eb786a021c137632c44298b5" + ], + "index": "pypi", + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2' and python_version < '4'", + "version": "==3.2.0" + }, + "pipfile": { + "hashes": [ + "sha256:f7d9f15de8b660986557eb3cc5391aa1a16207ac41bc378d03f414762d36c984" + ], + "version": "==0.0.2" + }, + "platformdirs": { + "hashes": [ + "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee", + "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3" + ], + "markers": "python_version >= '3.8'", + "version": "==4.2.2" + }, + "plette": { + "extras": [ + "validation" + ], + "hashes": [ + "sha256:06b8c09eb90293ad0b8101cb5c95c4ea53e9b2b582901845d0904ff02d237454", + "sha256:42d68ce8c6b966874b68758d87d7f20fcff2eff0d861903eea1062126be4d98f" + ], + "index": "pypi", + "markers": "python_version >= '3.7'", + "version": "==0.4.4" + }, + "pyparsing": { + "hashes": [ + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.4.7" + }, + "python-dateutil": { + "hashes": [ + "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", + "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==2.9.0.post0" + }, + "requests": { + "hashes": [ + "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760", + "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6" + ], + "markers": "python_version >= '3.8'", + "version": "==2.32.3" + }, + "requirementslib": { + "hashes": [ + "sha256:28924cf11a2fa91adb03f8431d80c2a8c3dc386f1c48fb2be9a58e4c39072354", + "sha256:d26ec6ad45e1ffce9532303543996c9c71a99dc65f783908f112e3f2aae7e49c" + ], + "markers": "python_version >= '3.7'", + "version": "==1.6.9" + }, + "setuptools": { + "hashes": [ + "sha256:b208925fcb9f7af924ed2dc04708ea89791e24bde0d3020b27df0e116088b34e", + "sha256:d59a3e788ab7e012ab2c4baed1b376da6366883ee20d7a5fc426816e3d7b1193" + ], + "markers": "python_version >= '3.8'", + "version": "==73.0.1" + }, + "six": { + "hashes": [ + "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", + "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", + "version": "==1.16.0" + }, + "toml": { + "hashes": [ + "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", + "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f" + ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'", + "version": "==0.10.2" + }, + "tomlkit": { + "hashes": [ + "sha256:7a974427f6e119197f670fbbbeae7bef749a6c14e793db934baefc1b5f03efde", + "sha256:fff5fe59a87295b278abd31bec92c15d9bc4a06885ab12bcea52c71119392e79" + ], + "markers": "python_version >= '3.8'", + "version": "==0.13.2" + }, + "urllib3": { + "hashes": [ + "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472", + "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168" + ], + "markers": "python_version >= '3.8'", + "version": "==2.2.2" + }, + "vistir": { + "hashes": [ + "sha256:1a89a612fb667c26ed6b4ed415b01e0261e13200a350c43d1990ace0ef44d35b", + "sha256:a8beb7643d07779cdda3941a08dad77d48de94883dbd3cb2b9b5ecb7eb7c0994" + ], + "index": "pypi", + "markers": "python_version not in '3.0, 3.1, 3.2, 3.3' and python_version >= '3.7'", + "version": "==0.6.1" + }, + "wheel": { + "hashes": [ + "sha256:2376a90c98cc337d18623527a97c31797bd02bad0033d41547043a1cbfbe448f", + "sha256:a29c3f2817e95ab89aa4660681ad547c0e9547f20e75b0562fe7723c9a2a9d49" + ], + "markers": "python_version >= '3.8'", + "version": "==0.44.0" + } } } diff --git a/docker-compose.yml b/docker-compose.yml index b3a2c25b..53f19b76 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,6 +8,5 @@ services: image: sparkfhirschemas_dev volumes: - ./:/sfs/ - - ~/.local/share/virtualenvs:/.local/share/virtualenvs:rw - container_name: sfs_dev + container_name: sparkfhirschemas_dev working_dir: /sfs diff --git a/setup.cfg b/setup.cfg index 8d446dba..5ea1ffa2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [tool:pytest] addopts = [mypy] -python_version = 3.6 +python_version = 3.12 warn_return_any = True warn_unused_configs = True [mypy-lxml.*] diff --git a/setup.py b/setup.py index b59bb7ef..e0efbe77 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,4 @@ # noinspection Mypy -from typing import Any from setuptools import setup, find_packages from os import path, getcwd @@ -19,33 +18,11 @@ raise -def fix_setuptools() -> None: - """Work around bugs in setuptools. - - Some versions of setuptools are broken and raise SandboxViolation for normal - operations in a virtualenv. We therefore disable the sandbox to avoid these - issues. - """ - try: - from setuptools.sandbox import DirectorySandbox - - # noinspection PyUnusedLocal - def violation(operation: Any, *args: Any, **_: Any) -> None: - print("SandboxViolation: %s" % (args,)) - - DirectorySandbox._violation = violation - except ImportError: - pass - - -# Fix bugs in setuptools. -fix_setuptools() - # classifiers list is here: https://pypi.org/classifiers/ # create the package setup setup( - install_requires=["pyspark==3.5.1"], + install_requires=["py4j==0.10.9.7", "pyspark==3.5.1"], name=package_name, version=version, author="Imran Qureshi", diff --git a/spark.Dockerfile b/spark.Dockerfile index 7850a86a..648379d4 100644 --- a/spark.Dockerfile +++ b/spark.Dockerfile @@ -1,4 +1,4 @@ -FROM imranq2/helix.spark:3.5.1.1-slim +FROM imranq2/helix.spark:3.5.1.3-slim # https://github.com/icanbwell/helix.spark USER root @@ -9,22 +9,12 @@ COPY Pipfile* /sfs/ WORKDIR /sfs RUN df -h # for space monitoring -RUN pipenv sync --dev --system && pipenv run pip install pyspark==3.5.1 +RUN pipenv sync --dev --system --extra-pip-args="--prefer-binary" # override entrypoint to remove extra logging RUN mv /opt/minimal_entrypoint.sh /opt/entrypoint.sh USER root -# install python 3.12 - it's not available in normal ubuntu repositories -# https://github.com/deadsnakes/issues/issues/53 -RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys F23C5A6CF475977595C89F51BA6932366A755776 && \ - echo "deb https://ppa.launchpadcontent.net/deadsnakes/ppa/ubuntu/ jammy main" | tee /etc/apt/sources.list.d/deadsnakes-ubuntu-ppa-lunar.list && \ - apt-get update && apt-get install -y python3.12 && \ - update-alternatives --install /usr/bin/python python /usr/bin/python3.12 1 && \ - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1 - -RUN pip install pyspark==3.5.1 -RUN pip install pytest>=8.2.2 COPY . /sfs @@ -32,3 +22,7 @@ RUN df -h # for space monitoring RUN mkdir -p /fhir && chmod 777 /fhir RUN mkdir -p /.local/share/virtualenvs && chmod 777 /.local/share/virtualenvs # USER 1001 + +# Run as non-root user +# https://spark.apache.org/docs/latest/running-on-kubernetes.html#user-identity +USER 185 \ No newline at end of file