From 8df121f93b20512417bf754259b05101e8e89c52 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Sat, 27 Jan 2024 14:49:37 +0100 Subject: [PATCH 01/10] Add reclass-rs as an optional dependency --- poetry.lock | 110 +++++++++++++++++++++++++++++++++++++------------ pyproject.toml | 4 +- 2 files changed, 86 insertions(+), 28 deletions(-) diff --git a/poetry.lock b/poetry.lock index e9ede092f..8cff1c675 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.0 and should not be changed by hand. [[package]] name = "addict" @@ -416,6 +416,7 @@ urllib3 = ">=1.26.0" [package.extras] ssh = ["paramiko (>=2.4.3)"] +websockets = ["websocket-client (>=1.3.0)"] [[package]] name = "enum34" @@ -428,7 +429,6 @@ files = [ {file = "enum34-1.1.10-py3-none-any.whl", hash = "sha256:c3858660960c984d6ab0ebad691265180da2b43f07e061c0f8dca9ef3cffd328"}, {file = "enum34-1.1.10.tar.gz", hash = "sha256:cce6a7477ed816bd2542d03d53db9f0db935dd013b70f336a95c73979289f248"}, ] -websockets = ["websocket-client (>=1.3.0)"] [[package]] name = "exceptiongroup" @@ -751,16 +751,6 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, - {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1127,7 +1117,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1135,16 +1124,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1161,8 +1142,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1170,8 +1149,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -1199,6 +1176,84 @@ url = "https://github.com/kapicorp/reclass" reference = "HEAD" resolved_reference = "856b34cb77811d665c6346883238d436ac5c4924" +[[package]] +name = "reclass-rs" +version = "0.4.0" +description = "Reclass defines a syntax and directory structure for recursively merging YAML data sources." +optional = true +python-versions = ">=3.8" +files = [ + {file = "reclass_rs-0.4.0-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:8f853eaf3b9af92c0eb9b3b25770ca35c3da7e411dfea6529651195b8d37c308"}, + {file = "reclass_rs-0.4.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0052213a2f3c5fd8f0d41458a772356b99fc277684a2af3d65a9fe209a068d0d"}, + {file = "reclass_rs-0.4.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9d00dec807866b1f86d9a27d062d2bd036b30b07a282ea136a02adeecbf27fc0"}, + {file = "reclass_rs-0.4.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dec6527849282a39992150c4dfd359c9facb5df8d5b55249f7043d65abaad7c1"}, + {file = "reclass_rs-0.4.0-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b7b0d104599aca9799b1053aca51bbbb415c786ab65b77274d48abbfd75a4556"}, + {file = "reclass_rs-0.4.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5bdb4750823dc766a207031d6fe9d665813084c9d5313a79f72f5566186fb1f8"}, + {file = "reclass_rs-0.4.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a794f2980140e4c26706a5271290b985664e588d68827c4aa77e65da937bae1a"}, + {file = "reclass_rs-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3ee5dd459ebebe83a63178b6fde241cc2ff37d74f78907ffdc9e6012cbe87ee"}, + {file = "reclass_rs-0.4.0-cp310-none-win32.whl", hash = "sha256:b20e15499071279733db2bce6029605b0edbe8d1c68bead384f14b37b1ad52a5"}, + {file = "reclass_rs-0.4.0-cp310-none-win_amd64.whl", hash = "sha256:16c1a7b0ef954d1613a2d44b7e4714ff728a2eaa3c3cec50e341f82a8a216369"}, + {file = "reclass_rs-0.4.0-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:d205cdf06dc10816dd3d2c0ad2eda35e2acca90432a8191a2a73ee6ee9bbe20a"}, + {file = "reclass_rs-0.4.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a6f7439851472d3f056964f7e1d45a7c3a0249e27a123288cf78445ae71b22bf"}, + {file = "reclass_rs-0.4.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f2c41e7119f26beae4fb4f086f84abeb90fb068ac21ff1e33aaf2e1e4db4ba02"}, + {file = "reclass_rs-0.4.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f616e4afba3d91371971784672f9b8d8ce39a8aada44bf9186b4b36346b9537"}, + {file = "reclass_rs-0.4.0-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:ef005a277013737d7bc01e0abb9b48295177ad3292d5eb20d4fc4e936f9a83e2"}, + {file = "reclass_rs-0.4.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3d87e40ec7d612efc33922db0fa073a5d3e6cb46a1537cb27756d51acdfc9338"}, + {file = "reclass_rs-0.4.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2fdf68cca29590cc342479a6d24ab8fe8941a5a448092baa7b032f1297be704"}, + {file = "reclass_rs-0.4.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8107a6cbfe26ccb363ec22a4257d76146c616dd4e0f45569c819a3f58854a17f"}, + {file = "reclass_rs-0.4.0-cp311-none-win32.whl", hash = "sha256:1ddc7bfcbbbe9508e1715cf7c934f9f64ae42ff3bbb9d0bc9ac83deb4ed681d7"}, + {file = "reclass_rs-0.4.0-cp311-none-win_amd64.whl", hash = "sha256:869f2fc72998b1eed104f24709d7248452b7a2d8ffc9c5ff240ae5884f7f1af7"}, + {file = "reclass_rs-0.4.0-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:b8135a1d60019d6aa498cb4a88b981b9725c98e9b30c376647182cc87d0fa31a"}, + {file = "reclass_rs-0.4.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:922a94eef9cccea33a26a24078abc7ed76af7aa3b585534147cf34417d078241"}, + {file = "reclass_rs-0.4.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:5c4175f1069ff17cffca508d716e80a635b965d6f1e47ae225d15f82094ac030"}, + {file = "reclass_rs-0.4.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fb79137810a91ede53ff9eb43e2532be96b8d2443eb2e5e955b2622495e95d7"}, + {file = "reclass_rs-0.4.0-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bee4b607b6c4b0a3972bb6c55687387bf6bbd21bab322a00b8ff405493a240af"}, + {file = "reclass_rs-0.4.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:628f85d667de40a3b97c8c670de7e5313a3d51fefbe45bacf9396a9a7719965a"}, + {file = "reclass_rs-0.4.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bcd88789835e3f04021458b4a741ce48d62bbeb9a0cdf353a57e359ea11f48f"}, + {file = "reclass_rs-0.4.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4037eb2bf5c66a89ee611c8298fee782f700e95198e29c5900a2495737d0e2d0"}, + {file = "reclass_rs-0.4.0-cp312-none-win32.whl", hash = "sha256:f654949bf9cd28835b0b31eecda5cc2db058ef791fde4a5bd315078a15ec1de0"}, + {file = "reclass_rs-0.4.0-cp312-none-win_amd64.whl", hash = "sha256:bfde34f826891d755f8da233bd7000a6567dfaf16a27fa50d5ba305fa10293a8"}, + {file = "reclass_rs-0.4.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4120a731aefa21fd012f314f96d383fe47496964955561b15f7283a9d05bb26"}, + {file = "reclass_rs-0.4.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7ba35ea61cd3f56cfe4edfd3772096ae331053318645463a4ba6b1268c8aa9d6"}, + {file = "reclass_rs-0.4.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4307eb7b37c69e5c3a0dc1b5cbede945c528db2da388627218f02511dee68f9a"}, + {file = "reclass_rs-0.4.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d882e57d42f8ac845af2fd396347e76dde84c48bf23c1e077317e66498c961d2"}, + {file = "reclass_rs-0.4.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ca99650751a014f60d975648ca668e5f267c994f1a370b79e00c9931e6ac7dbf"}, + {file = "reclass_rs-0.4.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c9708c8ca8f519ec0533de6b591edba7d76bf3695c9e27678e5da9633f500c1"}, + {file = "reclass_rs-0.4.0-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c404f113f6d28506faa932210a5447da29051ca394fd079cb5bc9252b34976bc"}, + {file = "reclass_rs-0.4.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2399a0e64f0af4eaf488d7159c217363bf087a2d4c8820f82000f766ff828bb6"}, + {file = "reclass_rs-0.4.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d1285830b85e7091f3728230f9f8626de2d2e678fc21b3a89076e5ca2ba46f40"}, + {file = "reclass_rs-0.4.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:84d43ccfa82f8b26ee1cb298ef1686524ba63758fc818d7f1ccc3fdeecacd540"}, + {file = "reclass_rs-0.4.0-cp38-none-win32.whl", hash = "sha256:9f83b38790e05530883c9bb0b6f80586e11e69639c1f3785e1d2aae417dfc957"}, + {file = "reclass_rs-0.4.0-cp38-none-win_amd64.whl", hash = "sha256:19dff5fe1cf020aa6a24b1a647ed0ccb7e1a1cf1d421e957ed128fda0bacafe9"}, + {file = "reclass_rs-0.4.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:ed70d329f140de9c699d307862d04c65219bf6e9f6e39be6d407d9c9a314e3ac"}, + {file = "reclass_rs-0.4.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:148f650cf55f575d0c109138627cd09add0fdbd9e967f8bf0ca41bace1a20098"}, + {file = "reclass_rs-0.4.0-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cbbf461073edec8320e337001586ef8fe3bd6a42733ded68997a66898cfb72fe"}, + {file = "reclass_rs-0.4.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1e79282bee0290f4436d851d9a3df7d72b4a454f4a05be6c524ac09823f14eb5"}, + {file = "reclass_rs-0.4.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d1b557216eb1fedc426efec5dbd59f9fd4ce7ee50b805cb8a9528a3e7793633"}, + {file = "reclass_rs-0.4.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63d05f43b29e62b8a2e8ef961aedfe05af6c7ae540ab9cc24a8f90fafeac2d53"}, + {file = "reclass_rs-0.4.0-cp39-none-win32.whl", hash = "sha256:94a1c7965e9d88b18c7bdd335e67984c630b369fcb2e5e1be94886a5dcae2009"}, + {file = "reclass_rs-0.4.0-cp39-none-win_amd64.whl", hash = "sha256:1bb4a3b802497722cd1f444297d066fda5ca3ec50eaa7646dbe1e01cf642c3e5"}, + {file = "reclass_rs-0.4.0-pp310-pypy310_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:fb37f7b5d41ab3e315fc525d23d23cc8202bdacd0019d864bd76493f7afd7533"}, + {file = "reclass_rs-0.4.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aa108eb00bd9a5a842a668878947777ec2ba7d126c3922074e54a76638981474"}, + {file = "reclass_rs-0.4.0-pp310-pypy310_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:fa792b5b624967863ba12defec84522c52bb4c5201a952e14e541e6bb510c91c"}, + {file = "reclass_rs-0.4.0-pp310-pypy310_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c75a51c6ae4978be21c25b4d440f4359b450bd846ef86a01623d5e08c60a119"}, + {file = "reclass_rs-0.4.0-pp310-pypy310_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:20bc17f03e91d369fafbed33a1e6282cfb155e0aa3789161313115c98af89d7f"}, + {file = "reclass_rs-0.4.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:333dbc0b3502d586457d39b6b424042e0529c6591aa59ddd65ee4c03b6fc0a6e"}, + {file = "reclass_rs-0.4.0-pp38-pypy38_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8b22ea8a00ab9102f1f6611c568f1fa34eec7cc4b3fe77c1be879582a832c36c"}, + {file = "reclass_rs-0.4.0-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98f07c37459f6ac536e9eba959f7be22f812849a94b3305c8df823e8c61b0159"}, + {file = "reclass_rs-0.4.0-pp38-pypy38_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:08fdeea644e5e5e27c69f0dbd1aa769d9c16af93a2419c3e0ee2e692016c608b"}, + {file = "reclass_rs-0.4.0-pp38-pypy38_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:75c0b9ce6b8bfbddd0a9fe670fa440c1d8cd2ae9687a96827dc2d968869c52b3"}, + {file = "reclass_rs-0.4.0-pp38-pypy38_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2ef6c1af4c993f1b7fd69f26425e77f98a6519a747024084fcd38dcb070c6641"}, + {file = "reclass_rs-0.4.0-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:020bed0189328aae7ed12647232c6821d440b8ec81e7b737067adb6bdbfd3233"}, + {file = "reclass_rs-0.4.0-pp39-pypy39_pp73-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:8a473b1a634d7032b4e5ba6e1eaa5aecef07795b8601ed1f35715c5bb6339a88"}, + {file = "reclass_rs-0.4.0-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae5d9ec7fa64e231af20d23efcd0a1dc89eaeb3594611f1bdca2096ca368d67b"}, + {file = "reclass_rs-0.4.0-pp39-pypy39_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:23a3f2a710a9590f1ad79d99a9d264d4917e4f8b8a64d09de92eb2b7f8c3372f"}, + {file = "reclass_rs-0.4.0-pp39-pypy39_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4ae7a1ac4b78a994cb0d119d454caf93776528eb0ad5fd742785d6503e96ffab"}, + {file = "reclass_rs-0.4.0-pp39-pypy39_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:068b33a19652260c1e551d5ef415eb73493b810dec77250393c31753388ba94c"}, + {file = "reclass_rs-0.4.0-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9573ff1eeb8f7f103c7a3b292a9348537e17d1b8e4e2ca792ae3776d4533bb76"}, + {file = "reclass_rs-0.4.0.tar.gz", hash = "sha256:6fe5c7f24c0f6f0baac91404620cf32c103658030213deee4fb6e9ed725d8cc8"}, +] + [[package]] name = "referencing" version = "0.32.1" @@ -1495,9 +1550,10 @@ dev = ["doc8", "flake8", "flake8-import-order", "rstcheck[sphinx]", "sphinx"] [extras] gojsonnet = ["gojsonnet"] -test = ["docker"] +reclass-rs = ["reclass-rs"] +test = ["docker", "reclass-rs"] [metadata] lock-version = "2.0" python-versions = "^3.10" -content-hash = "2fe358199e545c228705fdc013d4d8591be337d462cb95a29f171cb3beeabf9c" +content-hash = "15eda653356f12e08a8a4e3a5cb83c230f038b84147bc5793581b0ec9d897e8e" diff --git a/pyproject.toml b/pyproject.toml index 3b62a82f5..94b7ffcd8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,10 +63,12 @@ typing-extensions = "^4.0.0" gojsonnet = { version = "^0.20.0", optional = true } docker = { version = ">=5,<8", optional = true } reclass = {git = "https://github.com/kapicorp/reclass"} +reclass-rs = {version = "^0.4.0", optional = true} [tool.poetry.extras] gojsonnet = ["gojsonnet"] -test = ["docker"] +test = ["docker", "reclass-rs"] +reclass-rs = ["reclass-rs"] [tool.black] line-length = 110 From acce1312f51e81e71b75524bd31e9a4a55a923f6 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Mon, 26 Feb 2024 14:11:08 +0100 Subject: [PATCH 02/10] Introduce parameters `ignore_class_notfound` and `compose_node_name` for `get_reclass_config()` By introducing these parameters, we don't need to override as many options in the returned reclass config. Additionally, this change introduces a clean interface for getting Kapitan's expected Reclass config for the reclass-rs integration. --- kapitan/inventory/inv_reclass.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/kapitan/inventory/inv_reclass.py b/kapitan/inventory/inv_reclass.py index 11e6f1608..3847014b5 100644 --- a/kapitan/inventory/inv_reclass.py +++ b/kapitan/inventory/inv_reclass.py @@ -24,9 +24,9 @@ def render_targets(self, targets: list = None, ignore_class_notfound: bool = Fal Does not throw errors if a class is not found while ignore_class_notfound is specified """ - reclass_config = get_reclass_config(self.inventory_path) - reclass_config.setdefault("ignore_class_notfound", ignore_class_notfound) - reclass_config["compose_node_name"] = self.compose_target_name + reclass_config = get_reclass_config( + self.inventory_path, ignore_class_notfound, self.compose_target_name + ) try: storage = reclass.get_storage( @@ -54,15 +54,20 @@ def render_targets(self, targets: list = None, ignore_class_notfound: bool = Fal raise InventoryError(e.message) -def get_reclass_config(inventory_path: str) -> dict: +def get_reclass_config( + inventory_path: str, + ignore_class_notfound: bool = False, + compose_node_name: bool = False, +) -> dict: # set default values initially reclass_config = { "storage_type": "yaml_fs", "inventory_base_uri": inventory_path, "nodes_uri": "targets", "classes_uri": "classes", - "compose_node_name": False, + "compose_node_name": compose_node_name, "allow_none_override": True, + "ignore_class_notfound": ignore_class_notfound, } try: from yaml import CSafeLoader as YamlLoader From 100e575e1326ed71466a0f2002ae839c7e861631 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Mon, 26 Feb 2024 14:12:40 +0100 Subject: [PATCH 03/10] Make normalisation of `nodes_uri` and `classes_uri` optional This commit introduces a new optional parameter for `get_reclass_config()` which allows callers to disable the normalisation of the `nodes_uri` and `classes_uri` parameters. This option is required for reclass-rs, which expects these parameters to be relative to the inventory path, rather than relative to the working directory. --- kapitan/inventory/inv_reclass.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kapitan/inventory/inv_reclass.py b/kapitan/inventory/inv_reclass.py index 3847014b5..d47f287dd 100644 --- a/kapitan/inventory/inv_reclass.py +++ b/kapitan/inventory/inv_reclass.py @@ -58,6 +58,7 @@ def get_reclass_config( inventory_path: str, ignore_class_notfound: bool = False, compose_node_name: bool = False, + normalise_nodes_classes: bool = True, ) -> dict: # set default values initially reclass_config = { @@ -92,7 +93,8 @@ def get_reclass_config( logger.debug("Inventory reclass: No config file found. Using reclass inventory config defaults") # normalise relative nodes_uri and classes_uri paths - for uri in ("nodes_uri", "classes_uri"): - reclass_config[uri] = os.path.normpath(os.path.join(inventory_path, reclass_config[uri])) + if normalise_nodes_classes: + for uri in ("nodes_uri", "classes_uri"): + reclass_config[uri] = os.path.normpath(os.path.join(inventory_path, reclass_config[uri])) return reclass_config From d655b12335201bb05c3f38bd5b54fb61ac162f29 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Mon, 26 Feb 2024 14:14:22 +0100 Subject: [PATCH 04/10] Implement inventory backend for reclass-rs We reuse `get_reclass_config()` in the `ReclassRsInventory` backend. Additionally, we make the backend selectable through the `--inventory-backend` CLI option. --- kapitan/inventory/__init__.py | 2 ++ kapitan/inventory/inv_reclass_rs.py | 39 +++++++++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 kapitan/inventory/inv_reclass_rs.py diff --git a/kapitan/inventory/__init__.py b/kapitan/inventory/__init__.py index c3c0f662d..ed25bdbee 100644 --- a/kapitan/inventory/__init__.py +++ b/kapitan/inventory/__init__.py @@ -1,10 +1,12 @@ from typing import Type from .inv_reclass import ReclassInventory +from .inv_reclass_rs import ReclassRsInventory from .inventory import Inventory # Dict mapping values for command line flag `--inventory-backend` to the # associated `Inventory` subclass. AVAILABLE_BACKENDS: dict[str, Type[Inventory]] = { "reclass": ReclassInventory, + "reclass-rs": ReclassRsInventory, } diff --git a/kapitan/inventory/inv_reclass_rs.py b/kapitan/inventory/inv_reclass_rs.py new file mode 100644 index 000000000..76bf3212f --- /dev/null +++ b/kapitan/inventory/inv_reclass_rs.py @@ -0,0 +1,39 @@ +import logging +import os +import reclass_rs + +from kapitan.errors import InventoryError + +from .inventory import Inventory, InventoryTarget +from .inv_reclass import get_reclass_config + +logger = logging.getLogger(__name__) + + +class ReclassRsInventory(Inventory): + def _make_reclass_rs(self, ignore_class_notfound: bool): + # Get Reclass config options with the same method that's used for `ReclassInventory`, but + # disable the logic to normalise the `nodes_uri` and `classes_uri` options, since reclass-rs + # expects those fields to be relative to the inventory path. + config_dict = get_reclass_config( + self.inventory_path, ignore_class_notfound, self.compose_target_name, False + ) + + # Turn on verbose config loading only if Kapitan loglevel is set to at least DEBUG. + config = reclass_rs.Config.from_dict( + self.inventory_path, config_dict, logger.isEnabledFor(logging.DEBUG) + ) + return reclass_rs.Reclass.from_config(config) + + def render_targets(self, targets: list = None, ignore_class_notfound: bool = False): + try: + r = self._make_reclass_rs(ignore_class_notfound) + inv = r.inventory() + + for target_name, nodeinfo in inv.nodes.items(): + self.targets[target_name].parameters = nodeinfo.parameters + self.targets[target_name].classes = nodeinfo.classes + + except ValueError as e: + logger.error(f"Reclass-rs error: {e}") + raise InventoryError(f"{e}") From 112b492278a48610630eed101cef73eb110c802d Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Wed, 20 Sep 2023 13:06:04 +0200 Subject: [PATCH 05/10] Add test cases for `inventory` and remote inventory with reclass-rs --- tests/test_inventory.py | 20 +++++++++++++++++++ tests/test_remote_inventory.py | 35 +++++++++++++++++++++++++++++----- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/tests/test_inventory.py b/tests/test_inventory.py index c8216a0d1..45f9fdf6a 100644 --- a/tests/test_inventory.py +++ b/tests/test_inventory.py @@ -7,12 +7,23 @@ "inventory tests" +import argparse +import importlib import unittest +from kapitan import cached from kapitan.resources import inventory class InventoryTargetTest(unittest.TestCase): + def setUp(self): + # Setup `cached.args` if it's not setup yet + cached.args.setdefault("compile", argparse.Namespace()) + # Configure `compile.inventory_path` and `inventory-backend`. This + # allows us to reuse the tests by inheriting from this test class. + cached.args["compile"].inventory_path = "inventory" + cached.args["inventory-backend"] = "reclass" + def test_inventory_target(self): inv = inventory(["examples/kubernetes"], "minikube-es") self.assertEqual(inv["parameters"]["cluster"]["name"], "minikube") @@ -20,3 +31,12 @@ def test_inventory_target(self): def test_inventory_all_targets(self): inv = inventory(["examples/kubernetes"], None) self.assertNotEqual(inv.get("minikube-es"), None) + + +class InventoryTargetTestReclassRs(InventoryTargetTest): + def setUp(self): + if not importlib.util.find_spec("reclass_rs"): + self.skipTest("reclass-rs not available") + + super().setUp() + cached.args["inventory-backend"] = "reclass-rs" diff --git a/tests/test_remote_inventory.py b/tests/test_remote_inventory.py index 923235d58..863e9b343 100644 --- a/tests/test_remote_inventory.py +++ b/tests/test_remote_inventory.py @@ -1,3 +1,4 @@ +import argparse import os import sys import unittest @@ -17,7 +18,7 @@ ) -class RemoteInventoryTest(unittest.TestCase): +class RemoteInventoryTestFetch(unittest.TestCase): def setUp(self): os.chdir( os.path.join( @@ -89,6 +90,17 @@ def test_unpack_http_inv(self): rmtree(temp_dir) rmtree(output_dir) + +class RemoteInventoryTest(unittest.TestCase): + extraArgv = [] + + def setUp(self): + os.chdir( + os.path.join( + os.path.abspath(os.path.dirname(__file__)), "test_remote_inventory", "environment_one" + ) + ) + def test_compile_fetch(self): """Run $ kapitan compile --force-fetch --output-path=some/dir/ --inventory-path=another/dir --targets remoteinv-example remoteinv-nginx zippedinv were some/dir/ & another/dir/ are directories chosen by the user @@ -119,7 +131,7 @@ def test_compile_fetch(self): "remoteinv-example", "remoteinv-nginx", "zippedinv", - ] + ] + self.extraArgv main() self.assertTrue(os.path.isfile(os.path.join(temp_inv, "targets", "remoteinv-nginx.yml"))) @@ -152,7 +164,7 @@ def test_compile_fetch_classes_that_doesnot_exist_yet(self): os.path.join(temp_dir, "inventory"), "-t", "nginx", - ] + ] + self.extraArgv main() self.assertTrue(os.path.exists(os.path.join(temp_dir, "compiled", "nginx"))) rmtree(temp_dir) @@ -186,7 +198,7 @@ def test_force_fetch(self): temp_inv, "--targets", "remoteinv-example", - ] + ] + self.extraArgv main() fname = os.path.join(temp_inv, "targets", "zippedinv.yml") @@ -212,7 +224,7 @@ def test_force_fetch(self): temp_inv, "--targets", "remoteinv-example", - ] + ] + self.extraArgv main() # zippedinv.yml overwritten force_fetched_data = json.loads(yaml_load([os.path.dirname(fname)], os.path.basename(fname))) @@ -224,3 +236,16 @@ def test_force_fetch(self): def tearDown(self): os.chdir("../../../") reset_cache() + + +class RemoteInventoryTestReclassRs(RemoteInventoryTest): + def setUp(self): + super().setUp() + self.extraArgv = ["--inventory-backend=reclass-rs"] + + @unittest.skip( + "Broken with reclass-rs due to duplicate key in class `component.mysql` " + + "(`parameters.mysql.storage` is duplicated)" + ) + def test_compile_fetch(self): + pass From 251e82b5878bc87e51e40f46881d44b26ae399e3 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Mon, 4 Mar 2024 17:56:00 +0100 Subject: [PATCH 06/10] Add test case for `componse-node-name` This commit is based on the test case introduced in PR #1138. We move the compose-node-name test implementation to `tests/` so it's stored in the same place as the other tests and make the test class inheritable so that we can reuse the test for reclass-rs. Additionally we extend the test to check that all targets found by `search_targets()` are rendered by `render_targets()`. This is needed since we don't use reclass(-rs)'s target discovery logic in `search_targets()` but instead use a simplified version that's implemented directly in Kapitan's inventory backend base class. However, `render_targets()` then renders whatever targets reclass(-rs)'s target discovery finds, so if there's a mismatch we'd want to be informed. Co-authored-by: Matteo Voges Co-authored-by: Simon Gerber --- tests/test_compose_node_name.py | 68 +++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 tests/test_compose_node_name.py diff --git a/tests/test_compose_node_name.py b/tests/test_compose_node_name.py new file mode 100644 index 000000000..c8d5aec13 --- /dev/null +++ b/tests/test_compose_node_name.py @@ -0,0 +1,68 @@ +import glob +import logging +import os +import shutil +import tempfile +import unittest + +from kapitan.inventory import ReclassInventory +from kapitan.inventory import ReclassRsInventory +from kapitan.inventory.inventory import InventoryError +from kapitan import setup_logging + + +class ReclassComposeNodeNameTest(unittest.TestCase): + def setUp(self): + self.inventory = ReclassInventory + + def test_compose_target_name(self): + inventory_path = "examples/kubernetes/inventory" + example_target_names = [ + os.path.splitext(f)[0] for f in os.listdir(os.path.join(inventory_path, "targets")) + ] + + temp_inventory_dir = tempfile.mkdtemp() + shutil.copytree(inventory_path, temp_inventory_dir, dirs_exist_ok=True) + + # ensure normal rendering works + compose_target_name = True + inv = self.inventory(temp_inventory_dir, compose_target_name) + found_targets = inv.search_targets() + self.assertEqual(example_target_names, list(found_targets.keys())) + # ensure that actual rendering finds the same nodes as `search_targets()` + for t in example_target_names: + nodeinfo = inv.get_target(t) + self.assertTrue(nodeinfo is not None) + + # create compose_target_name setup + targets_path = os.path.join(inventory_path, "targets") + shutil.copytree(targets_path, os.path.join(temp_inventory_dir, "targets", "env1")) + shutil.copytree(targets_path, os.path.join(temp_inventory_dir, "targets", "env2")) + + composed_target_names = [] + for name in example_target_names: + composed_target_names.extend([name, f"env1.{name}", f"env2.{name}"]) + + # ensure inventory detects name collision + compose_target_name = False + inv = self.inventory(temp_inventory_dir, compose_target_name) + with self.assertRaises(InventoryError): + inv.search_targets() + + # ensure compose_target_name works as intended + compose_target_name = True + inv = self.inventory(temp_inventory_dir, compose_target_name) + found_targets = inv.search_targets() + + self.assertEqual(set(composed_target_names), set(found_targets.keys())) + # ensure that actual rendering finds the same nodes as `search_targets()` + for t in composed_target_names: + nodeinfo = inv.get_target(t) + self.assertTrue(nodeinfo is not None) + + shutil.rmtree(temp_inventory_dir) + + +class ReclassRsComposeNodeNameTest(ReclassComposeNodeNameTest): + def setUp(self): + self.inventory = ReclassRsInventory From 1e1dcf5d4e4c9d7e39be73a8bd940b5bd4906fcd Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Mon, 4 Mar 2024 17:51:44 +0100 Subject: [PATCH 07/10] Log timing information for reclass(-rs) if verbose logging is requested We implement the timing simply by executing `datetime.now()` before and after the call to the method which renders the reclass inventory. --- kapitan/inventory/inv_reclass.py | 5 +++++ kapitan/inventory/inv_reclass_rs.py | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/kapitan/inventory/inv_reclass.py b/kapitan/inventory/inv_reclass.py index d47f287dd..63751f428 100644 --- a/kapitan/inventory/inv_reclass.py +++ b/kapitan/inventory/inv_reclass.py @@ -1,6 +1,8 @@ import logging import os +from datetime import datetime + import reclass import reclass.core import yaml @@ -37,7 +39,10 @@ def render_targets(self, targets: list = None, ignore_class_notfound: bool = Fal ) class_mappings = reclass_config.get("class_mappings") # this defaults to None (disabled) _reclass = reclass.core.Core(storage, class_mappings, reclass.settings.Settings(reclass_config)) + start = datetime.now() rendered_inventory = _reclass.inventory() + elapsed = datetime.now() - start + logger.debug(f"Inventory rendering with reclass took {elapsed}") # store parameters and classes for target_name, rendered_target in rendered_inventory["nodes"].items(): diff --git a/kapitan/inventory/inv_reclass_rs.py b/kapitan/inventory/inv_reclass_rs.py index 76bf3212f..f7d714439 100644 --- a/kapitan/inventory/inv_reclass_rs.py +++ b/kapitan/inventory/inv_reclass_rs.py @@ -2,6 +2,8 @@ import os import reclass_rs +from datetime import datetime + from kapitan.errors import InventoryError from .inventory import Inventory, InventoryTarget @@ -28,7 +30,10 @@ def _make_reclass_rs(self, ignore_class_notfound: bool): def render_targets(self, targets: list = None, ignore_class_notfound: bool = False): try: r = self._make_reclass_rs(ignore_class_notfound) + start = datetime.now() inv = r.inventory() + elapsed = datetime.now() - start + logger.debug(f"Inventory rendering with reclass-rs took {elapsed}") for target_name, nodeinfo in inv.nodes.items(): self.targets[target_name].parameters = nodeinfo.parameters From f8abe41a6efc850915dbed470f189e503a95925e Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Mon, 4 Mar 2024 18:04:42 +0100 Subject: [PATCH 08/10] Update `test_coverage` make target to not ignore files matching `*reclass*` This is not required anymore since we don't import reclass as a Python submodule anymore. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7434d33ea..af4f314e8 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ test_docker: .PHONY: test_coverage test_coverage: @echo ----- Testing code coverage ----- - coverage run --source=kapitan --omit="*reclass*" -m unittest discover + coverage run --source=kapitan -m unittest discover coverage report --fail-under=65 -m .PHONY: test_formatting From dd01d2bdb89feb63754d7055c0e3a147c7cc84a3 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 5 Mar 2024 11:05:48 +0100 Subject: [PATCH 09/10] Update documentation --- docs/pages/commands/kapitan_compile.md | 2 +- docs/pages/inventory/introduction.md | 5 ++ docs/pages/inventory/reclass-rs.md | 71 ++++++++++++++++++++++++++ mkdocs.yml | 1 + 4 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 docs/pages/inventory/reclass-rs.md diff --git a/docs/pages/commands/kapitan_compile.md b/docs/pages/commands/kapitan_compile.md index aaf410f13..e3ab697e3 100644 --- a/docs/pages/commands/kapitan_compile.md +++ b/docs/pages/commands/kapitan_compile.md @@ -168,7 +168,7 @@ The `--embed-refs` flags tells **Kapitan** to embed these references on compile, options: -h, --help show this help message and exit - --inventory-backend {reclass} + --inventory-backend {reclass,reclass-rs} Select the inventory backend to use (default=reclass) --search-paths JPATH [JPATH ...], -J JPATH [JPATH ...] set search paths, default is ["."] diff --git a/docs/pages/inventory/introduction.md b/docs/pages/inventory/introduction.md index 8c788c5f9..3858c9d7d 100644 --- a/docs/pages/inventory/introduction.md +++ b/docs/pages/inventory/introduction.md @@ -17,6 +17,11 @@ By combining [**target**](#targets) and [**classes**](#classes), the **Inventory !!! info The **Kapitan** **Inventory** is based on an open source project called [reclass](https://github.com/kapicorp/reclass) and you can find the full documentation on our Github clone. However we discourage you to look directly at the reclass documentation before you learn more about **Kapitan**, because **Kapitan** uses a fork of reclass and greatly simplifies the reclass experience. +!!! info + Kapitan allows users to switch the inventory backend to [reclass-rs](https://github.com/projectsyn/reclass-rs). You can switch the backend to reclass-rs by passing `--inventory-backend=reclass-rs` on the command line. Alternatively, you can define the backend in the [.kapitan config file](../commands/kapitan_dotfile.md). + + See the [reclass-rs inventory backend](reclass-rs.md) documentation for more details. + !!! note Kapitan enforces very little structure for the **Inventory**, so that you can adapt it to your specific needs: this might be overwhelming at the beginning: don’t worry, we will explain best practice and give guidelines soon. diff --git a/docs/pages/inventory/reclass-rs.md b/docs/pages/inventory/reclass-rs.md new file mode 100644 index 000000000..0fbe36fdb --- /dev/null +++ b/docs/pages/inventory/reclass-rs.md @@ -0,0 +1,71 @@ +# The reclass-rs inventory backend + +## Overview + +[Reclass-rs](https://github.com/projectsyn/reclass-rs) is a reimplementation of Kapitan's Reclass fork in Rust. +Please note that the Rust implementation doesn't support all the features of Kapitan's Reclass fork yet. + +However, reclass-rs improves rendering time for the inventory significantly, especially if you're making heavy use of parameter references in class includes. +If some of the Reclass features or options that you're using are missing in reclass-rs, don't hesitate to open an issue in the [reclass-rs project](https://github.com/projectsyn/reclass-rs/issues/new?assignees=&labels=enhancement&projects=&template=03_missing_reclass_feature.md). + +## Installation + +The `reclass-rs` Python package is an optional dependency of Kapitan. +You can install it as follows: + +```shell +pip install kapitan[reclass-rs] +``` + +## Usage + +To use the reclass-rs inventory backend, you need to pass `--inventory-backend=reclass-rs` on the command line. +If you want to permanently switch to the reclass-rs inventory backend, you can select the inventory backend in the [.kapitan config file](../commands/kapitan_dotfile.md): + +```yaml +global: + inventory-backend: reclass-rs +``` + +## Performance comparison + +For the performance comparison, a real Kapitan inventory which makes heavy use of parameter interpolation in class includes was rendered with both Reclass and reclass-rs. +The example inventory that was used for the performance comparison contains 325 classes and 56 targets. +The example inventory renders to a total of 25MB of YAML. + +### Reclass + +``` +$ time kapitan inventory -v --inventory-backend=reclass > inv.yml +[ ... some output omitted ... ] +kapitan.resources DEBUG Using reclass as inventory backend +kapitan.inventory.inv_reclass DEBUG Inventory reclass: No config file found. Using reclass inventory config defaults +kapitan.inventory.inv_reclass DEBUG Inventory rendering with reclass took 0:01:06.037057 + +real 1m23.840s +user 1m23.520s +sys 0m0.287s +``` + +Reclass takes 1 minute and 6 seconds to render the example inventory. +The rest of the runtime (roughly 18 seconds) is spent in writing the resulting 25MB of YAML to the output file. + +### reclass-rs + +``` +$ time kapitan inventory -v --inventory-backend=reclass-rs > inv-rs.yml +[ ... some output omitted ... ] +kapitan.resources DEBUG Using reclass-rs as inventory backend +kapitan.inventory.inv_reclass DEBUG Inventory reclass: No config file found. Using reclass inventory config defaults +reclass-config.yml entry 'storage_type=yaml_fs' not implemented yet, ignoring... +reclass-config.yml entry 'inventory_base_uri=./inventory' not implemented yet, ignoring... +reclass-config.yml entry 'allow_none_override=true' not implemented yet, ignoring... +kapitan.inventory.inv_reclass_rs DEBUG Inventory rendering with reclass-rs took 0:00:01.717107 + +real 0m19.921s +user 0m35.586s +sys 0m1.066s +``` + +reclass-rs takes 1.7 seconds to render the example inventory. +The rest of the runtime (roughly 18 seconds) is spent in writing the resulting 25MB of YAML to the output file. diff --git a/mkdocs.yml b/mkdocs.yml index c24f16b96..04d380bc8 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -91,6 +91,7 @@ nav: - Classes: pages/inventory/classes.md - Parameters Interpolation: pages/inventory/parameters_interpolation.md - Advanced: pages/inventory/advanced.md + - reclass-rs Backend: pages/inventory/reclass-rs.md - Input Types: - Introduction: pages/input_types/introduction.md - Kadet: pages/input_types/kadet.md From 7f01abc4f71219316a13a4b74666b3b750ebfc11 Mon Sep 17 00:00:00 2001 From: Simon Gerber Date: Tue, 5 Mar 2024 13:05:33 +0100 Subject: [PATCH 10/10] Install `reclass-rs` in container image --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b99206ade..b3a9bd7f4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,7 @@ COPY ./poetry.lock ./poetry.lock COPY ./README.md ./README.md # Installs and caches dependencies -RUN poetry install --no-root --extras=gojsonnet +RUN poetry install --no-root --extras=gojsonnet --extras=reclass-rs COPY ./kapitan ./kapitan