diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 8d0eff683a..babf1d766a 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -57,3 +57,32 @@ jobs: name: pants-log-py${{ matrix.python-version }} path: .pants.d/pants.log if: always() # We want the log even on failures. + + set_merge_ok: + name: Set Merge OK (Lint) + if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') + needs: + - lint-checks + outputs: + merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }} + runs-on: ubuntu-latest + steps: + - id: set_merge_ok + run: echo 'merge_ok=true' >> ${GITHUB_OUTPUT} + + merge_ok: + name: Merge OK (Lint) + if: always() + needs: + - set_merge_ok + runs-on: ubuntu-latest + steps: + - run: | + merge_ok="${{ needs.set_merge_ok.outputs.merge_ok }}" + if [[ "${merge_ok}" == "true" ]]; then + echo "Merge OK" + exit 0 + else + echo "Merge NOT OK" + exit 1 + fi diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7b39b61a72..9be21ee106 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,21 +23,17 @@ on: # - cron: '0 0 * * *' jobs: - test: - name: '${{ matrix.name }} - Python ${{ matrix.python-version-short }}' + pants-plugins-tests: + name: 'Pants Plugins Tests (pants runs: pytest) - Python ${{ matrix.python.version-short }}' runs-on: ubuntu-20.04 strategy: fail-fast: false matrix: # NOTE: We need to use full Python version as part of Python deps cache key otherwise # setup virtualenv step will fail. - include: - - name: 'Test (pants runs: pytest)' - python-version-short: '3.8' - python-version: '3.8.10' - - name: 'Test (pants runs: pytest)' - python-version-short: '3.9' - python-version: '3.9.14' + python: + # Pants itself uses only 3.9 + - {version-short: '3.9', version: '3.9.14'} services: mongo: @@ -87,11 +83,109 @@ jobs: git submodule foreach 'git fetch --all --tags' git submodule foreach 'git tag' - - name: 'Set up Python (${{ matrix.python-version }})' + - name: 'Set up Python (${{ matrix.python.version }})' + id: python uses: actions/setup-python@v5 with: - python-version: '${{ matrix.python-version }}' + python-version: '${{ matrix.python.version }}' + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages + + - name: Initialize Pants and its GHA caches + uses: ./.github/actions/init-pants + with: + # To ignore a bad cache, bump the cache* integer. + gha-cache-key: cache0-py${{ matrix.python.version }} + + - name: Test pants-plugins + env: + # Github Actions uses the 'runner' user, so use that instead of stanley. + ST2TESTS_SYSTEM_USER: 'runner' + ST2TESTS_REDIS_HOST: '127.0.0.1' + ST2TESTS_REDIS_PORT: '6379' + run: | + pants test pants-plugins/:: + + - name: Upload pants log + uses: actions/upload-artifact@v4 + with: + name: pants-log-py${{ matrix.python.version }}-pants-plugins-tests + path: .pants.d/pants.log + if: always() # We want the log even on failures. + + unit-tests: + name: 'Unit Tests Shard ${{ matrix.shard.k }}/${{ matrix.shard.n }} (pants runs: pytest) - Python ${{ matrix.python.version-short }}' + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + # NOTE: We need to use full Python version as part of Python deps cache key otherwise + # setup virtualenv step will fail. + python: + - {version-short: '3.8', version: '3.8.10'} + - {version-short: '3.9', version: '3.9.14'} + shard: + # Sharding of tests is handled by pants: + # https://www.pantsbuild.org/stable/docs/using-pants/advanced-target-selection#sharding-the-input-targets + - {k: '0', n: '4'} + - {k: '1', n: '4'} + - {k: '2', n: '4'} + - {k: '3', n: '4'} + + services: + mongo: + image: mongo:7.0 + ports: + - 27017:27017 + + rabbitmq: + image: rabbitmq:3.8-management + options: >- + --name rabbitmq + ports: + - 5671:5671/tcp # AMQP SSL port + - 5672:5672/tcp # AMQP standard port + - 15672:15672/tcp # Management: HTTP, CLI + + redis: + # Docker Hub image + image: redis + # Set health checks to wait until redis has started + options: >- + --name "redis" + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379/tcp + + env: + COLUMNS: '120' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # a test uses a submodule, and pants needs access to it to calculate deps. + submodules: 'recursive' + # sadly, the submodule will only have fetch-depth=1, which is what we want + # for st2.git, but not for the submodules. We still want actions/checkout + # to do the initial checkout, however, so that it adds auth for fetching + # in the submodule. + + - name: Fetch repository submodules + run: | + git submodule status + git submodule foreach 'git fetch --all --tags' + git submodule foreach 'git tag' + + - name: 'Set up Python (${{ matrix.python.version }})' + id: python + uses: actions/setup-python@v5 + with: + python-version: '${{ matrix.python.version }}' - name: Cache and Install APT Dependencies uses: ./.github/actions/apt-packages @@ -100,21 +194,154 @@ jobs: uses: ./.github/actions/init-pants with: # To ignore a bad cache, bump the cache* integer. - gha-cache-key: cache0-py${{ matrix.python-version }} + gha-cache-key: cache0-py${{ matrix.python.version }} - - name: Test + - name: Unit Tests env: # Github Actions uses the 'runner' user, so use that instead of stanley. ST2TESTS_SYSTEM_USER: 'runner' - # We do not support running pytest everywhere yet. When we do it will be simply: - # pants test :: - # Until then, we need to manually adjust this command line to test what we can. + ST2TESTS_REDIS_HOST: '127.0.0.1' + ST2TESTS_REDIS_PORT: '6379' + run: > + pants + --python-bootstrap-search-path=[] + --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} + --tag=unit + --test-shard=${{ matrix.shard.k }}/${{ matrix.shard.n }} + test '::' + + - name: Upload pants log + uses: actions/upload-artifact@v4 + with: + name: pants-log-py${{ matrix.python.version }}-unit-tests-shard-${{ matrix.shard.k }}_${{ matrix.shard.n }} + path: .pants.d/pants.log + if: always() # We want the log even on failures. + + pack-tests: + name: 'Pack Tests (pants runs: pytest) - Python ${{ matrix.python.version-short }}' + runs-on: ubuntu-20.04 + strategy: + fail-fast: false + matrix: + # NOTE: We need to use full Python version as part of Python deps cache key otherwise + # setup virtualenv step will fail. + python: + - {version-short: '3.8', version: '3.8.10'} + - {version-short: '3.9', version: '3.9.14'} + + services: + mongo: + image: mongo:7.0 + ports: + - 27017:27017 + + rabbitmq: + image: rabbitmq:3.8-management + options: >- + --name rabbitmq + ports: + - 5671:5671/tcp # AMQP SSL port + - 5672:5672/tcp # AMQP standard port + - 15672:15672/tcp # Management: HTTP, CLI + + redis: + # Docker Hub image + image: redis + # Set health checks to wait until redis has started + options: >- + --name "redis" + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + ports: + - 6379:6379/tcp + + env: + COLUMNS: '120' + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + # a test uses a submodule, and pants needs access to it to calculate deps. + submodules: 'recursive' + # sadly, the submodule will only have fetch-depth=1, which is what we want + # for st2.git, but not for the submodules. We still want actions/checkout + # to do the initial checkout, however, so that it adds auth for fetching + # in the submodule. + + - name: Fetch repository submodules run: | - pants test pylint_plugins/:: pants-plugins/:: + git submodule status + git submodule foreach 'git fetch --all --tags' + git submodule foreach 'git tag' + + - name: 'Set up Python (${{ matrix.python.version }})' + id: python + uses: actions/setup-python@v5 + with: + python-version: '${{ matrix.python.version }}' + + - name: Cache and Install APT Dependencies + uses: ./.github/actions/apt-packages + + - name: Initialize Pants and its GHA caches + uses: ./.github/actions/init-pants + with: + # To ignore a bad cache, bump the cache* integer. + gha-cache-key: cache0-py${{ matrix.python.version }} + + - name: Pack Tests + env: + # Github Actions uses the 'runner' user, so use that instead of stanley. + ST2TESTS_SYSTEM_USER: 'runner' + ST2TESTS_REDIS_HOST: '127.0.0.1' + ST2TESTS_REDIS_PORT: '6379' + run: > + pants + --python-bootstrap-search-path=[] + --python-bootstrap-search-path=${{ steps.python.outputs.python-path }} + --tag=pack + test '::' - name: Upload pants log uses: actions/upload-artifact@v4 with: - name: pants-log-py${{ matrix.python-version }} + name: pants-log-py${{ matrix.python.version }}-pack-tests path: .pants.d/pants.log if: always() # We want the log even on failures. + + #integration-tests: TODO: run integration tests + + set_merge_ok: + name: Set Merge OK (Tests) + if: always() && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') + needs: + - pants-plugins-tests + - unit-tests + - pack-tests + #- integration-tests + outputs: + merge_ok: ${{ steps.set_merge_ok.outputs.merge_ok }} + runs-on: ubuntu-latest + steps: + - id: set_merge_ok + run: echo 'merge_ok=true' >> ${GITHUB_OUTPUT} + + merge_ok: + name: Merge OK (Tests) + if: always() + needs: + - set_merge_ok + runs-on: ubuntu-latest + steps: + - run: | + merge_ok="${{ needs.set_merge_ok.outputs.merge_ok }}" + if [[ "${merge_ok}" == "true" ]]; then + echo "Merge OK" + exit 0 + else + echo "Merge NOT OK" + exit 1 + fi diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 02f80e1d79..abd62f9cc8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -69,7 +69,7 @@ Added working on StackStorm, improve our security posture, and improve CI reliability thanks in part to pants' use of PEX lockfiles. This is not a user-facing addition. #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253 - #6254 #6258 #6259 #6260 + #6254 #6258 #6259 #6260 #6269 Contributed by @cognifloyd * Build of ST2 EL9 packages #6153 Contributed by @amanda11 diff --git a/pants-plugins/uses_services/redis_rules.py b/pants-plugins/uses_services/redis_rules.py index 9530011b0d..5863bb1aa3 100644 --- a/pants-plugins/uses_services/redis_rules.py +++ b/pants-plugins/uses_services/redis_rules.py @@ -27,6 +27,7 @@ VenvPexProcess, rules as pex_rules, ) +from pants.core.goals.test import TestExtraEnv from pants.engine.fs import CreateDigest, Digest, FileContent from pants.engine.rules import collect_rules, Get, MultiGet, rule from pants.engine.process import FallibleProcessResult, ProcessCacheScope @@ -57,7 +58,12 @@ class UsesRedisRequest: # with our version of oslo.config (newer are slower) we can't directly override opts w/ environment variables. - coord_url: str = "redis://127.0.0.1:6379" + host: str = "127.0.0.1" + port: str = "6379" + + @property + def coord_url(self) -> str: + return f"redis://{self.host}:{self.port}" @dataclass(frozen=True) @@ -80,9 +86,13 @@ def is_applicable(cls, target: Target) -> bool: ) async def redis_is_running_for_pytest( request: PytestUsesRedisRequest, + test_extra_env: TestExtraEnv, ) -> PytestPluginSetup: + redis_host = test_extra_env.env.get("ST2TESTS_REDIS_HOST", "127.0.0.1") + redis_port = test_extra_env.env.get("ST2TESTS_REDIS_PORT", "6379") + # this will raise an error if redis is not running - _ = await Get(RedisIsRunning, UsesRedisRequest()) + _ = await Get(RedisIsRunning, UsesRedisRequest(host=redis_host, port=redis_port)) return PytestPluginSetup() @@ -133,6 +143,13 @@ async def redis_is_running( if is_running: return RedisIsRunning() + env_vars_hint = dedent( + """ + You can also export the ST2TESTS_REDIS_HOST and ST2TESTS_REDIS_PORT + env vars to automatically use any redis host, local or remote, + while running unit and integration tests. + """ + ) # redis is not running, so raise an error with instructions. raise ServiceMissingError.generate( platform=platform, @@ -145,16 +162,20 @@ async def redis_is_running( """\ sudo yum -y install redis # Don't forget to start redis. + """ - ), + ) + + env_vars_hint, service_start_cmd_deb="systemctl start redis", not_installed_clause_deb="this is one way to install it:", install_instructions_deb=dedent( """\ sudo apt-get install -y mongodb redis # Don't forget to start redis. + """ - ), + ) + + env_vars_hint, service_start_cmd_generic="systemctl start redis", ), ) diff --git a/pants-plugins/uses_services/redis_rules_test.py b/pants-plugins/uses_services/redis_rules_test.py index 53e8808c37..93d7668698 100644 --- a/pants-plugins/uses_services/redis_rules_test.py +++ b/pants-plugins/uses_services/redis_rules_test.py @@ -73,7 +73,8 @@ def test_redis_is_running(rule_runner: RuleRunner) -> None: @pytest.mark.parametrize("mock_platform", platform_samples) def test_redis_not_running(rule_runner: RuleRunner, mock_platform: Platform) -> None: request = UsesRedisRequest( - coord_url="redis://127.100.20.7:10", # 10 is an unassigned port, unlikely to be used + host="127.100.20.7", + port="10", # 10 is an unassigned port, unlikely to be used ) with pytest.raises(ExecutionError) as exception_info: diff --git a/pants.toml b/pants.toml index fb6272b5b0..72e5fcb6c8 100644 --- a/pants.toml +++ b/pants.toml @@ -240,6 +240,9 @@ extra_env_vars = [ # Use this so that the test system does not require the stanley user. # For example: export ST2TESTS_SYSTEM_USER=${USER} "ST2TESTS_SYSTEM_USER", + # Use these to override the redis host and port + "ST2TESTS_REDIS_HOST", + "ST2TESTS_REDIS_PORT", ] [twine] diff --git a/pylint_plugins/BUILD b/pylint_plugins/BUILD index 1f7bfde6c6..9705181eec 100644 --- a/pylint_plugins/BUILD +++ b/pylint_plugins/BUILD @@ -8,6 +8,7 @@ python_sources() python_tests( name="tests", + tags=["unit"], dependencies=[ "./fixtures", "!//conftest.py:test_utils", diff --git a/st2actions/tests/unit/BUILD b/st2actions/tests/unit/BUILD index e8c10aa3c7..ae3f66d70a 100644 --- a/st2actions/tests/unit/BUILD +++ b/st2actions/tests/unit/BUILD @@ -8,10 +8,17 @@ python_tests( uses=["mongo"], overrides={ ( - "test_execution_cancellation.py", - "test_runner_container.py", - "test_worker.py", + "test_executions.py", + "test_policies.py", + "test_scheduler.py", + "test_workflow_engine.py", ): dict( + uses=["mongo", "rabbitmq", "redis"], + ), + ("test_execution_cancellation.py", "test_worker.py"): dict( + uses=["mongo", "rabbitmq", "redis", "system_user"], + ), + "test_runner_container.py": dict( uses=["mongo", "system_user"], ), ( diff --git a/st2actions/tests/unit/policies/BUILD b/st2actions/tests/unit/policies/BUILD index 66a22040d1..729db50258 100644 --- a/st2actions/tests/unit/policies/BUILD +++ b/st2actions/tests/unit/policies/BUILD @@ -4,5 +4,5 @@ python_tests( "st2common.runners.runner", "st2common.metrics.driver", ], - uses=["mongo"], + uses=["mongo", "rabbitmq", "redis"], ) diff --git a/st2api/tests/unit/controllers/v1/BUILD b/st2api/tests/unit/controllers/v1/BUILD index 5ef4f983f2..dd007bca19 100644 --- a/st2api/tests/unit/controllers/v1/BUILD +++ b/st2api/tests/unit/controllers/v1/BUILD @@ -8,12 +8,26 @@ python_tests( uses=["mongo"], overrides={ ( - "test_alias_execution.py", - "test_auth.py", - "test_auth_api_keys.py", - "test_executions.py", - "test_inquiries.py", + "test_actions.py", + "test_action_alias.py", + "test_executions_filters.py", + "test_kvps.py", + "test_packs.py", + "test_rules.py", + "test_sensortypes.py", + "test_triggers.py", + "test_triggertypes.py", + "test_triggerinstances.py", ): dict( + uses=["mongo", "rabbitmq", "redis"], + ), + ("test_alias_execution.py", "test_executions.py", "test_inquiries.py"): dict( + uses=["mongo", "rabbitmq", "redis", "system_user"], + ), + "test_service_registry.py": dict( + uses=["mongo", "redis"], + ), + ("test_auth.py", "test_auth_api_keys.py"): dict( uses=["mongo", "system_user"], ), "test_webhooks.py": dict( diff --git a/st2api/tests/unit/controllers/v1/test_triggertypes.py b/st2api/tests/unit/controllers/v1/test_triggertypes.py index 2461796e20..99d224c709 100644 --- a/st2api/tests/unit/controllers/v1/test_triggertypes.py +++ b/st2api/tests/unit/controllers/v1/test_triggertypes.py @@ -21,6 +21,7 @@ import six from st2api.controllers.v1.triggers import TriggerTypeController +from st2common.models.db.trigger import TriggerTypeDB from st2tests.api import FunctionalTest from st2tests.api import APIControllerWithIncludeAndExcludeFilterTestCase @@ -57,6 +58,9 @@ class TriggerTypeControllerTestCase( include_attribute_field_name = "payload_schema" exclude_attribute_field_name = "parameters_schema" + ensure_indexes = True + ensure_indexes_models = [TriggerTypeDB] + @classmethod def setUpClass(cls): # super's setUpClass does the following: diff --git a/st2common/st2common/services/coordination.py b/st2common/st2common/services/coordination.py index 8f693f56e5..803b8f694d 100644 --- a/st2common/st2common/services/coordination.py +++ b/st2common/st2common/services/coordination.py @@ -15,6 +15,8 @@ from __future__ import absolute_import +import sys + import six from oslo_config import cfg @@ -244,9 +246,17 @@ def get_coordinator(start_heart=True, use_cache=True): global COORDINATOR if not configured(): + extra_msg = "" + # sys._called_from_test set in conftest.py for pytest runs + if "nose" in sys.modules.keys() or hasattr(sys, "_called_from_test"): + extra_msg = ( + " Set ST2TESTS_REDIS_HOST and ST2TESTS_REDIS_PORT env vars to " + "configure the coordination backend for unit and integration tests." + ) LOG.warning( "Coordination backend is not configured. Code paths which use coordination " "service will use best effort approach and race conditions are possible." + f"{extra_msg}" ) if not use_cache: