From cf5cfe2fd163a39cb9e3c21657504e6d9d6ba533 Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Fri, 20 Dec 2024 19:04:11 +0900
Subject: [PATCH 01/22] ci: add more cache rules

---
 .circleci/config.yml | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9735a38198..08b43d8b71 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -19,9 +19,14 @@ jobs:
           command: export BASE_BRANCH=$(base_branch)
       - restore_cache:
           keys:
-          - py3-cache-v12-{{ arch }}-{{ checksum "python.deps" }}
-          - py3-cache-v12-{{ arch }}-{{ .Branch }}
-          - py3-cache-v12-{{ arch }}-{{ .Environment.BASE_BRANCH }}
+            # 1. Exact match of dependencies
+            - py3-cache-v12-{{ arch }}-{{ checksum "python.deps" }}
+            # 2. Latest cache from this branch
+            - py3-cache-v12-{{ arch }}-{{ .Branch }}
+            # 3. Latest cache from base branch (e.g., master)
+            - py3-cache-v12-{{ arch }}-{{ .Environment.BASE_BRANCH }}
+            # 4. Any latest cache for this architecture (fallback)
+            - py3-cache-v12-{{ arch }}-
       - run:
           name: Install Python dependencies
           command: |
@@ -79,9 +84,14 @@ jobs:
           command: export BASE_BRANCH=$(base_branch)
       - restore_cache:
           keys:
+            # 1. Exact match of dependencies
             - js-cache-{{ arch }}-{{ checksum "js.deps" }}
+            # 2. Latest cache from this branch
             - js-cache-{{ arch }}-{{ .Branch }}
+            # 3. Latest cache from base branch (e.g., master)
             - js-cache-{{ arch }}-{{ .Environment.BASE_BRANCH }}
+            # 4. Any latest cache for this architecture (fallback)
+            - js-cache-{{ arch }}-
       - run:
           name: Install NodeJS and dependencies
           command: |

From 91c1575fda63d5b963778d1a8e36cd53c0897e9d Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Fri, 20 Dec 2024 20:50:05 +0900
Subject: [PATCH 02/22] ci: parallelize tests

---
 .circleci/config.yml | 28 +++++++++++++++++++++++++++-
 tasks/__init__.py    |  8 ++++++--
 2 files changed, 33 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 08b43d8b71..cf7a2bbe09 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -9,6 +9,7 @@ jobs:
       - image: redis:alpine
     environment:
       BASH_ENV: /root/.bashrc
+    parallelism: 3
     steps:
       - checkout
       - run:
@@ -56,7 +57,32 @@ jobs:
           command: |
             mkdir -p reports/python
             source venv/bin/activate
-            inv test --report --ci
+
+            echo "Debug: Current working directory"
+            pwd
+
+            echo "Debug: Listing project root contents"
+            ls -la
+
+            echo "Debug: Looking for all test files"
+            find . -name "test_*.py"
+
+            echo "Debug: Attempting to find and split tests"
+            TESTFILES=$(find udata/tests -name "test_*.py" | circleci tests split --split-by=timings)
+            echo "Debug: TESTFILES=$TESTFILES"
+
+            echo "Debug: Current node index: $CIRCLE_NODE_INDEX"
+            echo "Debug: Total nodes: $CIRCLE_NODE_TOTAL"
+
+            if [ -n "$TESTFILES" ]; then
+              echo "Running tests: $TESTFILES"
+              inv test --report --ci --paths "$TESTFILES"
+            else
+              echo "Error: No tests found to run in this split"
+              echo "Debug: Listing all Python files"
+              find . -name "*.py" | grep -i test
+              exit 1
+            fi
       - store_test_results:
           path: reports/python
       - store_artifacts:
diff --git a/tasks/__init__.py b/tasks/__init__.py
index eba91fb21f..9b53395e07 100644
--- a/tasks/__init__.py
+++ b/tasks/__init__.py
@@ -134,10 +134,14 @@ def i18nc(ctx):
 
 
 @task(i18nc)
-def test(ctx, fast=False, report=False, verbose=False, ci=False):
+def test(ctx, fast=False, report=False, verbose=False, ci=False, paths=None):
     """Run tests suite"""
     header("Run tests suite")
-    cmd = ["pytest udata"]
+    cmd = ["pytest"]
+    if paths:
+        cmd.extend(paths.split())
+    else:
+        cmd.append("udata")
     if ci:
         cmd.append("-p no:sugar --color=yes")
     if verbose:

From 111976bd2516601c4ea6ebbc6fbf279eba133e49 Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Fri, 20 Dec 2024 20:59:44 +0900
Subject: [PATCH 03/22] ci: remove useless bashrc loading for building with
 python

---
 .circleci/config.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index cf7a2bbe09..8d2db733f6 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -147,8 +147,6 @@ jobs:
   dist:
     docker:
       - image: udata/circleci:py3.11
-    environment:
-      BASH_ENV: /root/.bashrc
     steps:
       - checkout
       - attach_workspace:

From b61f3d2fb5fc95d86ffc5574776f1d2838dffe0d Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Fri, 20 Dec 2024 20:59:57 +0900
Subject: [PATCH 04/22] ci: parallelize assets compiling

---
 .circleci/config.yml | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 8d2db733f6..ebb91dc8ee 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -100,6 +100,7 @@ jobs:
       - image: udata/circleci:py3.11
     environment:
       BASH_ENV: /root/.bashrc
+    parallelism: 3
     steps:
       - checkout
       - run:
@@ -135,10 +136,12 @@ jobs:
       - run:
           name: Compile assets
           command: |
-            npm run assets:build
-            npm run widgets:build
-            npm run oembed:build
-
+            COMMANDS=("npm run assets:build" "npm run widgets:build" "npm run oembed:build")
+            INDEX=$((CIRCLE_NODE_INDEX))
+            if [ $INDEX -lt ${#COMMANDS[@]} ]; then
+              echo "Running: ${COMMANDS[$INDEX]}"
+              eval "${COMMANDS[$INDEX]}"
+            fi
       - persist_to_workspace:
           root: .
           paths:

From fa03998c7575de313954f85e36075935527bbdf9 Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Fri, 20 Dec 2024 21:04:13 +0900
Subject: [PATCH 05/22] ci: 3->4 executors for tests job

---
 .circleci/config.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index ebb91dc8ee..c80e20d0ec 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -9,7 +9,7 @@ jobs:
       - image: redis:alpine
     environment:
       BASH_ENV: /root/.bashrc
-    parallelism: 3
+    parallelism: 4
     steps:
       - checkout
       - run:

From f46644e656076d1ed565fe68b90815c660f94910 Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Fri, 20 Dec 2024 21:13:21 +0900
Subject: [PATCH 06/22] ci: remove debug lines

---
 .circleci/config.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index c80e20d0ec..e4b797fcec 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -71,9 +71,6 @@ jobs:
             TESTFILES=$(find udata/tests -name "test_*.py" | circleci tests split --split-by=timings)
             echo "Debug: TESTFILES=$TESTFILES"
 
-            echo "Debug: Current node index: $CIRCLE_NODE_INDEX"
-            echo "Debug: Total nodes: $CIRCLE_NODE_TOTAL"
-
             if [ -n "$TESTFILES" ]; then
               echo "Running tests: $TESTFILES"
               inv test --report --ci --paths "$TESTFILES"

From 7c7f5ae9d1421268d17289e5dffae8fb7daaeca5 Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Fri, 20 Dec 2024 21:19:25 +0900
Subject: [PATCH 07/22] tests: mark some tests are non paralellizable

---
 .circleci/config.yml | 37 +++++++++++++++++--------------------
 1 file changed, 17 insertions(+), 20 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index e4b797fcec..7a9bf039f1 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -39,11 +39,11 @@ jobs:
       - save_cache:
           key: py3-cache-v12-{{ arch }}-{{ checksum "python.deps" }}
           paths:
-          - venv
+            - venv
       - save_cache:
           key: py3-cache-v12-{{ arch }}-{{ .Branch }}
           paths:
-          - venv
+            - venv
       - run:
           name: Lint and format code and sort imports
           # ruff check --select I . : check linting and imports sorting without fixing (to fix, use --fix)
@@ -58,27 +58,24 @@ jobs:
             mkdir -p reports/python
             source venv/bin/activate
 
-            echo "Debug: Current working directory"
-            pwd
+            # List of tests that must run serially (one per line for easier maintenance)
+            SERIAL_TESTS="
+            udata/tests/dataset/test_dataset_tasks.py
+            "
 
-            echo "Debug: Listing project root contents"
-            ls -la
-
-            echo "Debug: Looking for all test files"
-            find . -name "test_*.py"
+            if [ "$CIRCLE_NODE_INDEX" = "0" ]; then
+              # First executor runs the non-parallel safe tests
+              echo "Running serial tests..."
+              echo "$SERIAL_TESTS" | tr -d '\n' | xargs inv test --report --ci --paths
+            fi
 
-            echo "Debug: Attempting to find and split tests"
-            TESTFILES=$(find udata/tests -name "test_*.py" | circleci tests split --split-by=timings)
-            echo "Debug: TESTFILES=$TESTFILES"
+            # Then run the rest of the tests split across executors
+            echo "Running parallel tests..."
+            TESTFILES=$(find udata/tests -name "test_*.py" $(echo "$SERIAL_TESTS" | sed 's/^/! -path /') | circleci tests split --split-by=timings)
 
             if [ -n "$TESTFILES" ]; then
               echo "Running tests: $TESTFILES"
               inv test --report --ci --paths "$TESTFILES"
-            else
-              echo "Error: No tests found to run in this split"
-              echo "Debug: Listing all Python files"
-              find . -name "*.py" | grep -i test
-              exit 1
             fi
       - store_test_results:
           path: reports/python
@@ -89,7 +86,7 @@ jobs:
       - persist_to_workspace:
           root: .
           paths:
-          - venv
+            - venv
 
   assets:
     docker:
@@ -125,11 +122,11 @@ jobs:
       - save_cache:
           key: js-cache-{{ arch }}-{{ checksum "js.deps" }}
           paths:
-          - node_modules
+            - node_modules
       - save_cache:
           key: js-cache-{{ arch }}-{{ .Branch }}
           paths:
-          - node_modules
+            - node_modules
       - run:
           name: Compile assets
           command: |

From 1f5d711842f6c39e0607d5690fe447ab025fd87f Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Fri, 20 Dec 2024 21:35:16 +0900
Subject: [PATCH 08/22] ci: remove parallelism for tests

---
 .circleci/config.yml | 22 +---------------------
 1 file changed, 1 insertion(+), 21 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 7a9bf039f1..373dfd2c7f 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -9,7 +9,6 @@ jobs:
       - image: redis:alpine
     environment:
       BASH_ENV: /root/.bashrc
-    parallelism: 4
     steps:
       - checkout
       - run:
@@ -57,26 +56,7 @@ jobs:
           command: |
             mkdir -p reports/python
             source venv/bin/activate
-
-            # List of tests that must run serially (one per line for easier maintenance)
-            SERIAL_TESTS="
-            udata/tests/dataset/test_dataset_tasks.py
-            "
-
-            if [ "$CIRCLE_NODE_INDEX" = "0" ]; then
-              # First executor runs the non-parallel safe tests
-              echo "Running serial tests..."
-              echo "$SERIAL_TESTS" | tr -d '\n' | xargs inv test --report --ci --paths
-            fi
-
-            # Then run the rest of the tests split across executors
-            echo "Running parallel tests..."
-            TESTFILES=$(find udata/tests -name "test_*.py" $(echo "$SERIAL_TESTS" | sed 's/^/! -path /') | circleci tests split --split-by=timings)
-
-            if [ -n "$TESTFILES" ]; then
-              echo "Running tests: $TESTFILES"
-              inv test --report --ci --paths "$TESTFILES"
-            fi
+            inv test --report --ci
       - store_test_results:
           path: reports/python
       - store_artifacts:

From 878b1bf88ebd98ddd33c03a7fc5c9ebd9d5375af Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Fri, 20 Dec 2024 21:38:04 +0900
Subject: [PATCH 09/22] ci: debug serial tests

---
 .circleci/config.yml | 32 +++++++++++++++++++++++++++++++-
 1 file changed, 31 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 373dfd2c7f..e7ead07fa6 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -9,6 +9,7 @@ jobs:
       - image: redis:alpine
     environment:
       BASH_ENV: /root/.bashrc
+    parallelism: 4
     steps:
       - checkout
       - run:
@@ -56,7 +57,36 @@ jobs:
           command: |
             mkdir -p reports/python
             source venv/bin/activate
-            inv test --report --ci
+
+            # List of tests that must run serially (one per line for easier maintenance)
+            SERIAL_TESTS="
+            udata/tests/dataset/test_dataset_tasks.py
+            "
+
+            if [ "$CIRCLE_NODE_INDEX" = "0" ]; then
+              # First executor runs the non-parallel safe tests
+              echo "Running serial tests..."
+              echo "Debug: Current environment:"
+              env | sort
+              echo "Debug: Python path:"
+              python -c "import sys; print('\n'.join(sys.path))"
+              echo "Debug: Running tests:"
+              echo "$SERIAL_TESTS" | tr -d '\n'
+              echo ""
+
+              # Run serial tests with more verbose output
+              echo "$SERIAL_TESTS" | tr -d '\n' | xargs inv test --report --ci --paths -- -vv
+            fi
+
+            # Then run the rest of the tests split across executors
+            echo "Running parallel tests..."
+            TESTFILES=$(find udata/tests -name "test_*.py" $(echo "$SERIAL_TESTS" | sed 's/^/! -path /') | circleci tests split --split-by=timings)
+
+            if [ -n "$TESTFILES" ]; then
+              echo "Debug: Running parallel tests on node $CIRCLE_NODE_INDEX:"
+              echo "$TESTFILES"
+              inv test --report --ci --paths "$TESTFILES"
+            fi
       - store_test_results:
           path: reports/python
       - store_artifacts:

From 547664b379371f29f6bf7c599c2051b6e64a0778 Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Fri, 20 Dec 2024 21:40:10 +0900
Subject: [PATCH 10/22] ci: fix serial tests

---
 .circleci/config.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index e7ead07fa6..db587985e1 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -74,8 +74,9 @@ jobs:
               echo "$SERIAL_TESTS" | tr -d '\n'
               echo ""
 
-              # Run serial tests with more verbose output
-              echo "$SERIAL_TESTS" | tr -d '\n' | xargs inv test --report --ci --paths -- -vv
+              # Fix: properly pass the paths argument
+              SERIAL_TEST_FILES=$(echo "$SERIAL_TESTS" | tr -d '\n')
+              inv test --report --ci --paths "${SERIAL_TEST_FILES}"
             fi
 
             # Then run the rest of the tests split across executors
@@ -85,7 +86,7 @@ jobs:
             if [ -n "$TESTFILES" ]; then
               echo "Debug: Running parallel tests on node $CIRCLE_NODE_INDEX:"
               echo "$TESTFILES"
-              inv test --report --ci --paths "$TESTFILES"
+              inv test --report --ci --paths "${TESTFILES}"
             fi
       - store_test_results:
           path: reports/python

From 662ae5d64fa42b8b8365f4d066e29f2bd8f54939 Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Fri, 20 Dec 2024 21:44:48 +0900
Subject: [PATCH 11/22] ci: fix parallel tests

---
 .circleci/config.yml | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index db587985e1..7f087b9ca9 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -66,22 +66,22 @@ jobs:
             if [ "$CIRCLE_NODE_INDEX" = "0" ]; then
               # First executor runs the non-parallel safe tests
               echo "Running serial tests..."
-              echo "Debug: Current environment:"
-              env | sort
-              echo "Debug: Python path:"
-              python -c "import sys; print('\n'.join(sys.path))"
-              echo "Debug: Running tests:"
-              echo "$SERIAL_TESTS" | tr -d '\n'
-              echo ""
-
-              # Fix: properly pass the paths argument
               SERIAL_TEST_FILES=$(echo "$SERIAL_TESTS" | tr -d '\n')
               inv test --report --ci --paths "${SERIAL_TEST_FILES}"
             fi
 
             # Then run the rest of the tests split across executors
             echo "Running parallel tests..."
-            TESTFILES=$(find udata/tests -name "test_*.py" $(echo "$SERIAL_TESTS" | sed 's/^/! -path /') | circleci tests split --split-by=timings)
+            # Fix: properly construct the find command with exclusions
+            EXCLUSIONS=""
+            while read -r test_file; do
+              if [ -n "$test_file" ]; then
+                EXCLUSIONS="$EXCLUSIONS ! -path $test_file"
+              fi
+            done <<< "$SERIAL_TESTS"
+
+            echo "Debug: Find command exclusions: $EXCLUSIONS"
+            TESTFILES=$(find udata/tests -name "test_*.py" $EXCLUSIONS | circleci tests split --split-by=timings)
 
             if [ -n "$TESTFILES" ]; then
               echo "Debug: Running parallel tests on node $CIRCLE_NODE_INDEX:"

From bcb6e1770b995715bf5d0c8515d4a55876f88e22 Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Sat, 21 Dec 2024 15:28:34 +0900
Subject: [PATCH 12/22] ci: run only one test in parallel

---
 .circleci/config.yml | 45 ++++++++++++++++++--------------------------
 1 file changed, 18 insertions(+), 27 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 7f087b9ca9..d1359e5261 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -9,7 +9,7 @@ jobs:
       - image: redis:alpine
     environment:
       BASH_ENV: /root/.bashrc
-    parallelism: 4
+    parallelism: 2
     steps:
       - checkout
       - run:
@@ -58,35 +58,26 @@ jobs:
             mkdir -p reports/python
             source venv/bin/activate
 
-            # List of tests that must run serially (one per line for easier maintenance)
-            SERIAL_TESTS="
-            udata/tests/dataset/test_dataset_tasks.py
+            # List of tests to run in parallel (one per line)
+            PARALLEL_TESTS="
+            udata/harvest/tests/test_actions.py
             "
+            # Remove leading/trailing whitespace and convert to space-separated list
+            PARALLEL_TESTS_LIST=$(echo "$PARALLEL_TESTS" | tr -d '\n' | xargs)
 
             if [ "$CIRCLE_NODE_INDEX" = "0" ]; then
-              # First executor runs the non-parallel safe tests
-              echo "Running serial tests..."
-              SERIAL_TEST_FILES=$(echo "$SERIAL_TESTS" | tr -d '\n')
-              inv test --report --ci --paths "${SERIAL_TEST_FILES}"
-            fi
-
-            # Then run the rest of the tests split across executors
-            echo "Running parallel tests..."
-            # Fix: properly construct the find command with exclusions
-            EXCLUSIONS=""
-            while read -r test_file; do
-              if [ -n "$test_file" ]; then
-                EXCLUSIONS="$EXCLUSIONS ! -path $test_file"
-              fi
-            done <<< "$SERIAL_TESTS"
-
-            echo "Debug: Find command exclusions: $EXCLUSIONS"
-            TESTFILES=$(find udata/tests -name "test_*.py" $EXCLUSIONS | circleci tests split --split-by=timings)
-
-            if [ -n "$TESTFILES" ]; then
-              echo "Debug: Running parallel tests on node $CIRCLE_NODE_INDEX:"
-              echo "$TESTFILES"
-              inv test --report --ci --paths "${TESTFILES}"
+              echo "Running all tests except parallel ones..."
+              # Create a temporary file with all test files
+              find . -name "test_*.py" > all_tests.txt
+              # Remove parallel tests from the list
+              for test in $PARALLEL_TESTS_LIST; do
+                sed -i "\:$test:d" all_tests.txt
+              done
+              # Run remaining tests - all at once instead of trying to pass them individually
+              inv test --report --ci
+            else
+              echo "Running parallel tests: $PARALLEL_TESTS_LIST"
+              inv test --report --ci --paths "$PARALLEL_TESTS_LIST"
             fi
       - store_test_results:
           path: reports/python

From 4d7d715c6e3df6435a36b0b275045f3f61874258 Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Sat, 21 Dec 2024 15:46:43 +0900
Subject: [PATCH 13/22] ci: more tests in parallel

---
 .circleci/config.yml | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index d1359e5261..745f839208 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -58,9 +58,16 @@ jobs:
             mkdir -p reports/python
             source venv/bin/activate
 
-            # List of tests to run in parallel (one per line)
+            # List of longer-running tests to execute in parallel (one per line)
             PARALLEL_TESTS="
             udata/harvest/tests/test_actions.py
+            udata/tests/api/test_activies_api.py
+            udata/tests/api/test_datasets_api.py
+            udata/tests/api/test_organizations_api.py
+            udata/tests/api/test_reports_api.py
+            udata/tests/apiv2/test_swagger.py
+            udata/tests/api/test_transfer_api.py
+            udata/tests/forms/test_form_field.py
             "
             # Remove leading/trailing whitespace and convert to space-separated list
             PARALLEL_TESTS_LIST=$(echo "$PARALLEL_TESTS" | tr -d '\n' | xargs)

From 47b4b78e4ba3f1246c0e69296b928b4eb9688c65 Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Sat, 21 Dec 2024 15:47:30 +0900
Subject: [PATCH 14/22] Revert "ci: parallelize assets compiling"

This reverts commit b61f3d2fb5fc95d86ffc5574776f1d2838dffe0d.
---
 .circleci/config.yml | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 745f839208..fd021339cd 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -103,7 +103,6 @@ jobs:
       - image: udata/circleci:py3.11
     environment:
       BASH_ENV: /root/.bashrc
-    parallelism: 3
     steps:
       - checkout
       - run:
@@ -139,12 +138,10 @@ jobs:
       - run:
           name: Compile assets
           command: |
-            COMMANDS=("npm run assets:build" "npm run widgets:build" "npm run oembed:build")
-            INDEX=$((CIRCLE_NODE_INDEX))
-            if [ $INDEX -lt ${#COMMANDS[@]} ]; then
-              echo "Running: ${COMMANDS[$INDEX]}"
-              eval "${COMMANDS[$INDEX]}"
-            fi
+            npm run assets:build
+            npm run widgets:build
+            npm run oembed:build
+
       - persist_to_workspace:
           root: .
           paths:

From 9e36eaa67283a0bc77a6d933c8377f058dcae7bf Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Sat, 21 Dec 2024 15:49:58 +0900
Subject: [PATCH 15/22] ci: splitting test among 2 executors working, but
 parallel tests are still being executed in first executor as well

---
 .circleci/config.yml | 55 +++++++++++++++++++++++---------------------
 1 file changed, 29 insertions(+), 26 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index fd021339cd..b60490a771 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -7,8 +7,6 @@ jobs:
       - image: udata/circleci:py3.11
       - image: mongo:6.0.4
       - image: redis:alpine
-    environment:
-      BASH_ENV: /root/.bashrc
     parallelism: 2
     steps:
       - checkout
@@ -58,33 +56,38 @@ jobs:
             mkdir -p reports/python
             source venv/bin/activate
 
-            # List of longer-running tests to execute in parallel (one per line)
-            PARALLEL_TESTS="
-            udata/harvest/tests/test_actions.py
-            udata/tests/api/test_activies_api.py
-            udata/tests/api/test_datasets_api.py
-            udata/tests/api/test_organizations_api.py
-            udata/tests/api/test_reports_api.py
-            udata/tests/apiv2/test_swagger.py
-            udata/tests/api/test_transfer_api.py
-            udata/tests/forms/test_form_field.py
-            "
-            # Remove leading/trailing whitespace and convert to space-separated list
-            PARALLEL_TESTS_LIST=$(echo "$PARALLEL_TESTS" | tr -d '\n' | xargs)
+            # List of longer-running tests to execute in parallel (paths relative to udata/)
+            PARALLEL_TESTS=(
+              "harvest/tests/test_actions.py"
+              "tests/api/test_activities_api.py"
+              "tests/api/test_datasets_api.py"
+              "tests/api/test_organizations_api.py"
+              "tests/api/test_reports_api.py"
+              "tests/apiv2/test_swagger.py"
+              "tests/api/test_transfer_api.py"
+              "tests/forms/test_form_field.py"
+            )
+
+            # Convert array to string with | as separator for grep
+            PARALLEL_PATTERN=$(printf "%s|" "${PARALLEL_TESTS[@]}" | sed 's/|$//')
 
             if [ "$CIRCLE_NODE_INDEX" = "0" ]; then
-              echo "Running all tests except parallel ones..."
-              # Create a temporary file with all test files
-              find . -name "test_*.py" > all_tests.txt
-              # Remove parallel tests from the list
-              for test in $PARALLEL_TESTS_LIST; do
-                sed -i "\:$test:d" all_tests.txt
-              done
-              # Run remaining tests - all at once instead of trying to pass them individually
-              inv test --report --ci
+              echo "=== EXECUTOR 0: Running all tests except parallel ones ==="
+              cd ${CIRCLE_WORKING_DIRECTORY:-$(pwd)}/udata
+
+              # Collect all tests except parallel ones
+              python -m pytest --collect-only --quiet --no-header --no-summary | grep -v "$PARALLEL_PATTERN" > serial_tests.txt
+              cd ..
+              inv test --report --ci -- --durations=0
             else
-              echo "Running parallel tests: $PARALLEL_TESTS_LIST"
-              inv test --report --ci --paths "$PARALLEL_TESTS_LIST"
+              echo "=== EXECUTOR 1: Running parallel tests ==="
+              # Prepend udata/ to each test path for correct resolution from project root
+              PARALLEL_PATHS=()
+              for test_path in "${PARALLEL_TESTS[@]}"; do
+                PARALLEL_PATHS+=("udata/$test_path")
+              done
+              PATHS_STRING="${PARALLEL_PATHS[*]}"
+              inv test --report --ci --paths "$PATHS_STRING" -- --durations=0
             fi
       - store_test_results:
           path: reports/python

From c54132a5c3b1aba7309e529e9b343cbc91131c97 Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Mon, 23 Dec 2024 17:28:51 +0900
Subject: [PATCH 16/22] ci: try to exclude the parallel tests from the serial
 executor

---
 .circleci/config.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index b60490a771..9b11b7a9fe 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -78,7 +78,7 @@ jobs:
               # Collect all tests except parallel ones
               python -m pytest --collect-only --quiet --no-header --no-summary | grep -v "$PARALLEL_PATTERN" > serial_tests.txt
               cd ..
-              inv test --report --ci -- --durations=0
+              inv test --report --ci --paths "@serial_tests.txt"
             else
               echo "=== EXECUTOR 1: Running parallel tests ==="
               # Prepend udata/ to each test path for correct resolution from project root
@@ -87,7 +87,7 @@ jobs:
                 PARALLEL_PATHS+=("udata/$test_path")
               done
               PATHS_STRING="${PARALLEL_PATHS[*]}"
-              inv test --report --ci --paths "$PATHS_STRING" -- --durations=0
+              inv test --report --ci --paths "$PATHS_STRING"0
             fi
       - store_test_results:
           path: reports/python

From 6153958dcec281c8595ce62fcd2445f2fb80e4da Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Mon, 23 Dec 2024 17:31:34 +0900
Subject: [PATCH 17/22] ci: add timings and verbose output

---
 .circleci/config.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 9b11b7a9fe..2b85b6bae5 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -78,7 +78,8 @@ jobs:
               # Collect all tests except parallel ones
               python -m pytest --collect-only --quiet --no-header --no-summary | grep -v "$PARALLEL_PATTERN" > serial_tests.txt
               cd ..
-              inv test --report --ci --paths "@serial_tests.txt"
+              # Use the collected tests file
+              inv test --report --ci --paths "@serial_tests.txt" -- -v --durations=0
             else
               echo "=== EXECUTOR 1: Running parallel tests ==="
               # Prepend udata/ to each test path for correct resolution from project root
@@ -87,7 +88,7 @@ jobs:
                 PARALLEL_PATHS+=("udata/$test_path")
               done
               PATHS_STRING="${PARALLEL_PATHS[*]}"
-              inv test --report --ci --paths "$PATHS_STRING"0
+              inv test --report --ci --paths "$PATHS_STRING" -- -v --durations=0
             fi
       - store_test_results:
           path: reports/python

From dd46bad3c9879dd6fb8f61fa4f94ec8291181d02 Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Mon, 23 Dec 2024 17:51:06 +0900
Subject: [PATCH 18/22] ci: use pytest instead of inv test

---
 .circleci/config.yml | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 2b85b6bae5..fb8ae13db9 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -75,11 +75,19 @@ jobs:
               echo "=== EXECUTOR 0: Running all tests except parallel ones ==="
               cd ${CIRCLE_WORKING_DIRECTORY:-$(pwd)}/udata
 
-              # Collect all tests except parallel ones
-              python -m pytest --collect-only --quiet --no-header --no-summary | grep -v "$PARALLEL_PATTERN" > serial_tests.txt
+              # Collect tests with error checking
+              python -m pytest --collect-only --quiet --no-header --no-summary > all_tests.txt
+
+              # Filter out parallel tests and save to serial_tests.txt
+              grep -v -E "$PARALLEL_PATTERN" all_tests.txt > serial_tests.txt
+
+              echo "Content of serial_tests.txt:"
+              cat serial_tests.txt
+
+              # Run pytest directly reading from the file
               cd ..
-              # Use the collected tests file
-              inv test --report --ci --paths "@serial_tests.txt" -- -v --durations=0
+              source venv/bin/activate
+              python -m pytest --verbose --durations=0 $(cat udata/serial_tests.txt)
             else
               echo "=== EXECUTOR 1: Running parallel tests ==="
               # Prepend udata/ to each test path for correct resolution from project root

From 27bddd8e9ec57eb8ba9b31a440f307c846a61c26 Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Mon, 23 Dec 2024 22:34:41 +0900
Subject: [PATCH 19/22] ci: fix paths

---
 .circleci/config.yml | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index fb8ae13db9..fde6779cc5 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -75,28 +75,21 @@ jobs:
               echo "=== EXECUTOR 0: Running all tests except parallel ones ==="
               cd ${CIRCLE_WORKING_DIRECTORY:-$(pwd)}/udata
 
-              # Collect tests with error checking
-              python -m pytest --collect-only --quiet --no-header --no-summary > all_tests.txt
+              # Collect test files
+              find . -name "test_*.py" | grep -v -E "$PARALLEL_PATTERN" | sed 's|^./||' > serial_tests.txt
 
-              # Filter out parallel tests and save to serial_tests.txt
-              grep -v -E "$PARALLEL_PATTERN" all_tests.txt > serial_tests.txt
-
-              echo "Content of serial_tests.txt:"
-              cat serial_tests.txt
-
-              # Run pytest directly reading from the file
+              # Run pytest with the files
               cd ..
               source venv/bin/activate
-              python -m pytest --verbose --durations=0 $(cat udata/serial_tests.txt)
+              python -m pytest $(sed 's|^|udata/|' udata/serial_tests.txt)
             else
               echo "=== EXECUTOR 1: Running parallel tests ==="
-              # Prepend udata/ to each test path for correct resolution from project root
               PARALLEL_PATHS=()
               for test_path in "${PARALLEL_TESTS[@]}"; do
                 PARALLEL_PATHS+=("udata/$test_path")
               done
               PATHS_STRING="${PARALLEL_PATHS[*]}"
-              inv test --report --ci --paths "$PATHS_STRING" -- -v --durations=0
+              inv test --report --ci --paths "$PATHS_STRING"
             fi
       - store_test_results:
           path: reports/python

From 5e164a364cf9facab268ea739bc3e96ba5c7ef2e Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Mon, 23 Dec 2024 23:06:50 +0900
Subject: [PATCH 20/22] ci: add tests to be parallelized

---
 .circleci/config.yml | 20 ++++++++++++++++++--
 1 file changed, 18 insertions(+), 2 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index fde6779cc5..6b07066e7c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -56,16 +56,32 @@ jobs:
             mkdir -p reports/python
             source venv/bin/activate
 
-            # List of longer-running tests to execute in parallel (paths relative to udata/)
+            # Place here the tests to execute in parallel (paths relative to udata/)
+            # In order to gain some time when executing the tests in the CI, you should put here the tests files that:
+            # - takes a longer time to run than average
+            # - modify the global app configuration (so that they don't interfere with other tests)
+            # TODO: we could split the tests to be run by more than 2 executors
             PARALLEL_TESTS=(
               "harvest/tests/test_actions.py"
+              "harvest/tests/test_api.py"
               "tests/api/test_activities_api.py"
+              "tests/api/test_auth_api.py"
               "tests/api/test_datasets_api.py"
               "tests/api/test_organizations_api.py"
               "tests/api/test_reports_api.py"
-              "tests/apiv2/test_swagger.py"
+              "tests/api/test_reuses_api.py"
+              "tests/api/test_user_api.py"
               "tests/api/test_transfer_api.py"
+              "tests/apiv2/test_swagger.py"
+              "tests/dataset/test_dataset_model.py"
+              "tests/dataset/test_dataset_rdf.py"
               "tests/forms/test_form_field.py"
+              "tests/forms/test_model_field.py"
+              "tests/test_model.py"
+              "tests/test_routing.py"
+              "tests/test_uris.py"
+              # This test modifies the global app configuration
+              "tests/test_i18n.py"
             )
 
             # Convert array to string with | as separator for grep

From 5d66bf1ab5ccc90ff02b9ab73fbb50e6b56784fb Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Mon, 23 Dec 2024 23:34:31 +0900
Subject: [PATCH 21/22] ci: remove python sourcing and add more tests to be
 parallelized

---
 .circleci/config.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 6b07066e7c..86bf94f986 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -64,6 +64,8 @@ jobs:
             PARALLEL_TESTS=(
               "harvest/tests/test_actions.py"
               "harvest/tests/test_api.py"
+              "harvest/tests/test_base_backend.py"
+              "harvest/tests/test_dcat_backend.py"
               "tests/api/test_activities_api.py"
               "tests/api/test_auth_api.py"
               "tests/api/test_datasets_api.py"
@@ -77,9 +79,11 @@ jobs:
               "tests/dataset/test_dataset_rdf.py"
               "tests/forms/test_form_field.py"
               "tests/forms/test_model_field.py"
+              "tests/frontend/test_markdown.py"
               "tests/test_model.py"
               "tests/test_routing.py"
               "tests/test_uris.py"
+              "tests/test_utils.py"
               # This test modifies the global app configuration
               "tests/test_i18n.py"
             )
@@ -96,7 +100,6 @@ jobs:
 
               # Run pytest with the files
               cd ..
-              source venv/bin/activate
               python -m pytest $(sed 's|^|udata/|' udata/serial_tests.txt)
             else
               echo "=== EXECUTOR 1: Running parallel tests ==="

From a395ef081c7285b26b8bf6b1deaf67c994eff6ec Mon Sep 17 00:00:00 2001
From: Adrien Carpentier <me@adriencarpentier.com>
Date: Tue, 24 Dec 2024 00:09:51 +0900
Subject: [PATCH 22/22] ci: add more tests to be parallelized

---
 .circleci/config.yml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 86bf94f986..0e3c582b6a 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -60,7 +60,6 @@ jobs:
             # In order to gain some time when executing the tests in the CI, you should put here the tests files that:
             # - takes a longer time to run than average
             # - modify the global app configuration (so that they don't interfere with other tests)
-            # TODO: we could split the tests to be run by more than 2 executors
             PARALLEL_TESTS=(
               "harvest/tests/test_actions.py"
               "harvest/tests/test_api.py"
@@ -79,8 +78,10 @@ jobs:
               "tests/dataset/test_dataset_rdf.py"
               "tests/forms/test_form_field.py"
               "tests/forms/test_model_field.py"
+              "tests/forms/test_reference_field.py"
               "tests/frontend/test_markdown.py"
               "tests/test_model.py"
+              "tests/test_rdf.py"
               "tests/test_routing.py"
               "tests/test_uris.py"
               "tests/test_utils.py"
@@ -92,7 +93,7 @@ jobs:
             PARALLEL_PATTERN=$(printf "%s|" "${PARALLEL_TESTS[@]}" | sed 's/|$//')
 
             if [ "$CIRCLE_NODE_INDEX" = "0" ]; then
-              echo "=== EXECUTOR 0: Running all tests except parallel ones ==="
+              echo "=== EXECUTOR 0: Running all tests except isolated ones ==="
               cd ${CIRCLE_WORKING_DIRECTORY:-$(pwd)}/udata
 
               # Collect test files
@@ -102,7 +103,7 @@ jobs:
               cd ..
               python -m pytest $(sed 's|^|udata/|' udata/serial_tests.txt)
             else
-              echo "=== EXECUTOR 1: Running parallel tests ==="
+              echo "=== EXECUTOR 1: Running isolated tests ==="
               PARALLEL_PATHS=()
               for test_path in "${PARALLEL_TESTS[@]}"; do
                 PARALLEL_PATHS+=("udata/$test_path")