From 9899217e163e7a13ddfe3269c5c163ebc26c3748 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Sat, 18 Jun 2022 21:06:44 -0500 Subject: [PATCH 1/6] stub pants-plugins/macros.py Pants macros are used to define functions that can be used in BUILD files to inject additional functions that can add multiple targets to the BUILD file at once. These functions look like "targets" without building a whole plugin for them. --- pants-plugins/BUILD | 2 ++ pants-plugins/macros.py | 13 +++++++++++++ pants.toml | 1 + 3 files changed, 16 insertions(+) create mode 100644 pants-plugins/macros.py diff --git a/pants-plugins/BUILD b/pants-plugins/BUILD index 66302cd2e0..cb52cb88d8 100644 --- a/pants-plugins/BUILD +++ b/pants-plugins/BUILD @@ -11,3 +11,5 @@ pants_requirements( name="pants", testutil=True, ) + +python_sources() diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py new file mode 100644 index 0000000000..ff2cbcd6b9 --- /dev/null +++ b/pants-plugins/macros.py @@ -0,0 +1,13 @@ +# Copyright 2023 The StackStorm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/pants.toml b/pants.toml index f99b8d7daf..b88ae3e04a 100644 --- a/pants.toml +++ b/pants.toml @@ -8,6 +8,7 @@ repo_id = "de0dea7a-9f6a-4c6e-aa20-6ba5ad969b8a" [GLOBAL] pants_version = "2.14.0" pythonpath = ["%(buildroot)s/pants-plugins"] +build_file_prelude_globs = ["pants-plugins/macros.py"] backend_packages = [ # python "pants.backend.python", From 0ebf2d6ebb4c6528af03f88e4cf8b61236ac4ec4 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Jun 2022 11:24:20 -0500 Subject: [PATCH 2/6] add shell+resources pants macro to fix including shell in python_distributions --- pants-plugins/macros.py | 15 +++++++++++++++ st2actions/bin/BUILD | 3 ++- st2common/bin/BUILD | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index ff2cbcd6b9..7b0d008a5c 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -11,3 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + +def st2_shell_sources_and_resources(**kwargs): + """This creates a shell_sources and a resources target. + + This is needed because python_sources dependencies on shell_sources + are silently ignored. So, we also need the resources target + to allow depending on them. + """ + shell_sources(**kwargs) + + kwargs.pop("skip_shellcheck") + + kwargs["name"] += "_resources" + resources(**kwargs) diff --git a/st2actions/bin/BUILD b/st2actions/bin/BUILD index c81519023d..5a138aad51 100644 --- a/st2actions/bin/BUILD +++ b/st2actions/bin/BUILD @@ -5,7 +5,8 @@ python_sources( skip_pylint=True, ) -shell_sources( +st2_shell_sources_and_resources( name="shell", + sources=["*.sh"], skip_shellcheck=True, ) diff --git a/st2common/bin/BUILD b/st2common/bin/BUILD index 05d1f908c1..a05ce529eb 100644 --- a/st2common/bin/BUILD +++ b/st2common/bin/BUILD @@ -5,7 +5,7 @@ python_sources( skip_pylint=True, ) -shell_sources( +st2_shell_sources_and_resources( name="shell", sources=["st2ctl", "st2-self-check", "st2-run-pack-tests"], skip_shellcheck=True, From 3bd92868eed03fea6d62fb592bb2e26928f06b3a Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Jun 2022 14:10:56 -0500 Subject: [PATCH 3/6] resolve lint issues in pants-plugins/macros.py --- pants-plugins/macros.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index 7b0d008a5c..4820e9ee81 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -20,9 +20,9 @@ def st2_shell_sources_and_resources(**kwargs): are silently ignored. So, we also need the resources target to allow depending on them. """ - shell_sources(**kwargs) + shell_sources(**kwargs) # noqa: F821 kwargs.pop("skip_shellcheck") kwargs["name"] += "_resources" - resources(**kwargs) + resources(**kwargs) # noqa: F821 From 1df917681f33ae4fc14700130713a05dd8a439de Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Tue, 21 Jun 2022 17:54:09 -0500 Subject: [PATCH 4/6] add st2_shell_sources_and_resources() to fixtures --- pants-plugins/macros.py | 2 +- st2tests/st2tests/fixtures/generic/BUILD | 4 ++++ st2tests/st2tests/fixtures/generic/actions/BUILD | 5 ++++- st2tests/st2tests/fixtures/packs/BUILD | 3 ++- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/pants-plugins/macros.py b/pants-plugins/macros.py index 4820e9ee81..454bbaa05f 100644 --- a/pants-plugins/macros.py +++ b/pants-plugins/macros.py @@ -22,7 +22,7 @@ def st2_shell_sources_and_resources(**kwargs): """ shell_sources(**kwargs) # noqa: F821 - kwargs.pop("skip_shellcheck") + kwargs.pop("skip_shellcheck", None) kwargs["name"] += "_resources" resources(**kwargs) # noqa: F821 diff --git a/st2tests/st2tests/fixtures/generic/BUILD b/st2tests/st2tests/fixtures/generic/BUILD index 99d651ce3c..48fcf06310 100644 --- a/st2tests/st2tests/fixtures/generic/BUILD +++ b/st2tests/st2tests/fixtures/generic/BUILD @@ -1,5 +1,9 @@ pack_metadata( name="metadata", + dependencies=[ + "./actions:shell", + "./actions:shell_resources", + ], ) python_sources( diff --git a/st2tests/st2tests/fixtures/generic/actions/BUILD b/st2tests/st2tests/fixtures/generic/actions/BUILD index 6c95f66377..de6d193a62 100644 --- a/st2tests/st2tests/fixtures/generic/actions/BUILD +++ b/st2tests/st2tests/fixtures/generic/actions/BUILD @@ -1 +1,4 @@ -shell_sources() +st2_shell_sources_and_resources( + name="shell", + sources=["*.sh"], +) diff --git a/st2tests/st2tests/fixtures/packs/BUILD b/st2tests/st2tests/fixtures/packs/BUILD index bbcae502dd..025cf82aac 100644 --- a/st2tests/st2tests/fixtures/packs/BUILD +++ b/st2tests/st2tests/fixtures/packs/BUILD @@ -13,7 +13,7 @@ pack_metadata_in_git_submodule( ], ) -shell_sources( +st2_shell_sources_and_resources( name="test_content_version_shell", # do not check across git submodule boundary skip_shellcheck=True, @@ -29,6 +29,7 @@ python_sources( dependencies=[ ":test_content_version_metadata", ":test_content_version_shell", + ":test_content_version_shell_resources", ], sources=[ "test_content_version/**/*.py", From 34d54d8e580ab2c5d0cd19d4dcc20056afd66893 Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Wed, 8 Feb 2023 12:56:35 -0600 Subject: [PATCH 5/6] Add pants-plugins/macros.py to pants-plugins/README.md --- pants-plugins/README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pants-plugins/README.md b/pants-plugins/README.md index de85a3c855..222f8d69ed 100644 --- a/pants-plugins/README.md +++ b/pants-plugins/README.md @@ -16,6 +16,7 @@ These StackStorm-specific plugins might be useful in other StackStorm-related re These StackStorm-specific plugins are probably only useful for the st2 repo. - `api_spec` +- `macros.py` (not a plugin - see pants.toml `[GLOBAL].build_file_prelude_globs`) - `release` - `sample_conf` - `schemas` @@ -33,6 +34,16 @@ This plugin also wires up pants so that the `lint` goal runs additional api spec validation on `st2common/st2common/openapi.yaml` with something like `./pants lint st2common/st2common/openapi.yaml`. +### `macros.py` macros + +[Macros](https://www.pantsbuild.org/docs/macros) are a pants feature +that can reduce "boilerplate"/duplication in BUILD files. The functions +defined in `macros.py` are available in all the BUILD files, and using +them looks just like using the normal BUILD targets. + +For documentation about our macros, please refer to the function docstrings +in the `macros.py` file. + ### `pack_metadata` plugin This plugin adds two new targets to pants: From 96a3c8aa0122eb9a3611061712a40e785b1da13c Mon Sep 17 00:00:00 2001 From: Jacob Floyd Date: Mon, 6 Feb 2023 13:18:03 -0600 Subject: [PATCH 6/6] update changelog entry --- CHANGELOG.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f7beac4b7c..754bd680d7 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -16,6 +16,7 @@ Added to pants' use of PEX lockfiles. This is not a user-facing addition. #5778 #5789 #5817 #5795 #5830 #5833 #5834 #5841 #5840 #5838 #5842 #5837 #5849 #5850 #5846 #5853 #5848 #5847 #5858 #5857 #5860 #5868 #5871 #5864 #5874 #5884 #5893 #5891 + #5890 Contributed by @cognifloyd * Added a joint index to solve the problem of slow mongo queries for scheduled executions. #5805