diff --git a/.circleci/config.yml b/.circleci/config.yml index 9735a38198..0e3c582b6a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -7,8 +7,7 @@ jobs: - image: udata/circleci:py3.11 - image: mongo:6.0.4 - image: redis:alpine - environment: - BASH_ENV: /root/.bashrc + parallelism: 2 steps: - checkout - run: @@ -19,9 +18,14 @@ jobs: command: export BASE_BRANCH=$(base_branch) - restore_cache: keys: - - py3-cache-v12-{{ arch }}-{{ checksum "python.deps" }} - - py3-cache-v12-{{ arch }}-{{ .Branch }} - - py3-cache-v12-{{ arch }}-{{ .Environment.BASE_BRANCH }} + # 1. Exact match of dependencies + - py3-cache-v12-{{ arch }}-{{ checksum "python.deps" }} + # 2. Latest cache from this branch + - py3-cache-v12-{{ arch }}-{{ .Branch }} + # 3. Latest cache from base branch (e.g., master) + - py3-cache-v12-{{ arch }}-{{ .Environment.BASE_BRANCH }} + # 4. Any latest cache for this architecture (fallback) + - py3-cache-v12-{{ arch }}- - run: name: Install Python dependencies command: | @@ -33,11 +37,11 @@ jobs: - save_cache: key: py3-cache-v12-{{ arch }}-{{ checksum "python.deps" }} paths: - - venv + - venv - save_cache: key: py3-cache-v12-{{ arch }}-{{ .Branch }} paths: - - venv + - venv - run: name: Lint and format code and sort imports # ruff check --select I . : check linting and imports sorting without fixing (to fix, use --fix) @@ -51,7 +55,62 @@ jobs: command: | mkdir -p reports/python source venv/bin/activate - inv test --report --ci + + # Place here the tests to execute in parallel (paths relative to udata/) + # In order to gain some time when executing the tests in the CI, you should put here the tests files that: + # - takes a longer time to run than average + # - modify the global app configuration (so that they don't interfere with other tests) + PARALLEL_TESTS=( + "harvest/tests/test_actions.py" + "harvest/tests/test_api.py" + "harvest/tests/test_base_backend.py" + "harvest/tests/test_dcat_backend.py" + "tests/api/test_activities_api.py" + "tests/api/test_auth_api.py" + "tests/api/test_datasets_api.py" + "tests/api/test_organizations_api.py" + "tests/api/test_reports_api.py" + "tests/api/test_reuses_api.py" + "tests/api/test_user_api.py" + "tests/api/test_transfer_api.py" + "tests/apiv2/test_swagger.py" + "tests/dataset/test_dataset_model.py" + "tests/dataset/test_dataset_rdf.py" + "tests/forms/test_form_field.py" + "tests/forms/test_model_field.py" + "tests/forms/test_reference_field.py" + "tests/frontend/test_markdown.py" + "tests/test_model.py" + "tests/test_rdf.py" + "tests/test_routing.py" + "tests/test_uris.py" + "tests/test_utils.py" + # This test modifies the global app configuration + "tests/test_i18n.py" + ) + + # Convert array to string with | as separator for grep + PARALLEL_PATTERN=$(printf "%s|" "${PARALLEL_TESTS[@]}" | sed 's/|$//') + + if [ "$CIRCLE_NODE_INDEX" = "0" ]; then + echo "=== EXECUTOR 0: Running all tests except isolated ones ===" + cd ${CIRCLE_WORKING_DIRECTORY:-$(pwd)}/udata + + # Collect test files + find . -name "test_*.py" | grep -v -E "$PARALLEL_PATTERN" | sed 's|^./||' > serial_tests.txt + + # Run pytest with the files + cd .. + python -m pytest $(sed 's|^|udata/|' udata/serial_tests.txt) + else + echo "=== EXECUTOR 1: Running isolated tests ===" + PARALLEL_PATHS=() + for test_path in "${PARALLEL_TESTS[@]}"; do + PARALLEL_PATHS+=("udata/$test_path") + done + PATHS_STRING="${PARALLEL_PATHS[*]}" + inv test --report --ci --paths "$PATHS_STRING" + fi - store_test_results: path: reports/python - store_artifacts: @@ -61,7 +120,7 @@ jobs: - persist_to_workspace: root: . paths: - - venv + - venv assets: docker: @@ -79,9 +138,14 @@ jobs: command: export BASE_BRANCH=$(base_branch) - restore_cache: keys: + # 1. Exact match of dependencies - js-cache-{{ arch }}-{{ checksum "js.deps" }} + # 2. Latest cache from this branch - js-cache-{{ arch }}-{{ .Branch }} + # 3. Latest cache from base branch (e.g., master) - js-cache-{{ arch }}-{{ .Environment.BASE_BRANCH }} + # 4. Any latest cache for this architecture (fallback) + - js-cache-{{ arch }}- - run: name: Install NodeJS and dependencies command: | @@ -91,11 +155,11 @@ jobs: - save_cache: key: js-cache-{{ arch }}-{{ checksum "js.deps" }} paths: - - node_modules + - node_modules - save_cache: key: js-cache-{{ arch }}-{{ .Branch }} paths: - - node_modules + - node_modules - run: name: Compile assets command: | @@ -111,8 +175,6 @@ jobs: dist: docker: - image: udata/circleci:py3.11 - environment: - BASH_ENV: /root/.bashrc steps: - checkout - attach_workspace: diff --git a/tasks/__init__.py b/tasks/__init__.py index eba91fb21f..9b53395e07 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -134,10 +134,14 @@ def i18nc(ctx): @task(i18nc) -def test(ctx, fast=False, report=False, verbose=False, ci=False): +def test(ctx, fast=False, report=False, verbose=False, ci=False, paths=None): """Run tests suite""" header("Run tests suite") - cmd = ["pytest udata"] + cmd = ["pytest"] + if paths: + cmd.extend(paths.split()) + else: + cmd.append("udata") if ci: cmd.append("-p no:sugar --color=yes") if verbose: