ci: update testrunner and use dd-octo-sts everywhere (#14152)

brettlangdon · web-flow · commit c5213ff50aa3 · 2025-07-31T18:19:22.000Z
Use the latest version of testrunner image in CI, including the new one which contains `dd-octo-sts` got GitHub token management. This PR: - Use the new image locally and in CI - We use the ghcr image still locally since external contributors need to be able to pull the image - Don't install riot when we already have it baked in - Replace aws ssm with dd-octo-sts to generate GitHub API tokens - Update scripts/ddtest for the new projects directory for the testrunner image The biggest change being introduced here is we are moving to a testrunner image which uses the user `bits` by default `/home/bits/` as the working directory. ## Checklist - [x] PR author has checked that all the criteria below are met - The PR description includes an overview of the change - The PR description articulates the motivation for the change - The change includes tests OR the PR description describes a testing strategy - The PR description notes risks associated with the change, if any - Newly-added code is easy to change - The change follows the [library release note guidelines](https://ddtrace.readthedocs.io/en/stable/releasenotes.html) - The change includes or references documentation updates if necessary - Backport labels are set (if [applicable](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)) ## Reviewer Checklist - [ ] Reviewer has checked that all the criteria below are met - Title is accurate - All changes are related to the pull request's stated goal - Avoids breaking [API](https://ddtrace.readthedocs.io/en/stable/versioning.html#interfaces) changes - Testing strategy adequately addresses listed risks - Newly-added code is easy to change - Release note makes sense to a user of the library - If necessary, author has acknowledged and discussed the performance implications of this PR as reported in the benchmarks PR comment - Backport labels are set in a manner that is consistent with the [release branch maintenance policy](https://ddtrace.readthedocs.io/en/latest/contributing.html#backporting)
diff --git a/.coveragerc b/.coveragerc
@@ -8,5 +8,5 @@ disable_warnings = couldnt-parse
 [paths]
 source =
     ./
-    /home/circleci/project/
     /root/project/
+    /home/bits/project
diff --git a/.github/workflows/requirements-locks.yml b/.github/workflows/requirements-locks.yml
@@ -11,7 +11,9 @@ jobs:
   validate:
     name: Check requirements lockfiles
     runs-on: ubuntu-latest
-    container: ghcr.io/datadog/dd-trace-py/testrunner:bca6869fffd715ea9a731f7b606807fa1b75cb71@sha256:9e3f53fa98ffc4b838b959d74d969aa2c384c4cbee7a3047a03d501be5f58760
+    container:
+      image: ghcr.io/datadog/dd-trace-py/testrunner:ae4c189ebf8e539f39905f21c7918cc19de69d13@sha256:9476c426c677d39a58c170ed3167e4d630ed4f067a8a2be2fb96d843795d2ac2
+      options: "--user 0"
     steps:
       - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -42,8 +42,16 @@ include:
 tests-gen:
   stage: tests
   extends: .testrunner
+  id_tokens:
+    DDOCTOSTS_ID_TOKEN:
+      aud: dd-octo-sts
   script:
-    - pip install riot==0.20.1
+    - |
+      if [ -z ${GH_TOKEN} ]
+      then
+        # Use dd-octo-sts to get GitHub token
+        export GH_TOKEN=$(dd-octo-sts token --scope DataDog/dd-trace-py --policy gitlab.github-access.read)
+      fi
     - riot -v run --pass-env -s gitlab-gen-config -v
   needs: []
   artifacts:
diff --git a/.gitlab/benchmarks/microbenchmarks.yml b/.gitlab/benchmarks/microbenchmarks.yml
@@ -8,7 +8,7 @@ variables:
   BENCHMARKING_IMAGE_REGISTRY: 486234852809.dkr.ecr.us-east-1.amazonaws.com
   MICROBENCHMARKS_CI_IMAGE: $BENCHMARKING_IMAGE_REGISTRY/ci/benchmarking-platform:dd-trace-py
   PACKAGE_IMAGE: registry.ddbuild.io/images/mirror/pypa/manylinux2014_x86_64:2024-08-12-7fde9b1
-  GITHUB_CLI_IMAGE: registry.ddbuild.io/github-cli:v27480869-eafb11d-2.43.0
+  GITHUB_CLI_IMAGE: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1
   BENCHMARKING_BRANCH: dd-trace-py
 
 .benchmarks:
@@ -64,6 +64,9 @@ baseline:detect:
   image: $GITHUB_CLI_IMAGE
   tags: [ "arch:amd64" ]
   stage: build
+  id_tokens:
+    DDOCTOSTS_ID_TOKEN:
+      aud: dd-octo-sts
   variables:
     UPSTREAM_BRANCH: $CI_COMMIT_REF_NAME
   script: |
@@ -74,7 +77,8 @@ baseline:detect:
 
     if [ -z ${GH_TOKEN} ]
     then
-      aws ssm get-parameter --region us-east-1 --name ci.$CI_PROJECT_NAME.gh_token --with-decryption --query "Parameter.Value" --out text > token
+      # Use dd-octo-sts to get GitHub token
+      dd-octo-sts token --scope DataDog/dd-trace-py --policy gitlab.github-access.read > token
       gh auth login --with-token < token
       rm token
     fi
diff --git a/.gitlab/templates/build-base-venvs.yml b/.gitlab/templates/build-base-venvs.yml
@@ -18,8 +18,6 @@ build_base_venvs:
     - when: always
   script: |
     set -e -o pipefail
-    apt update && apt install -y sccache
-    pip install riot==0.20.1
     riot -P -v generate --python=$PYTHON_VERSION
     echo "Running smoke tests"
     riot -v run -s --python=$PYTHON_VERSION smoke_test
diff --git a/.gitlab/templates/cached-testrunner.yml b/.gitlab/templates/cached-testrunner.yml
@@ -2,7 +2,6 @@
   extends: .testrunner
   variables:
     PIP_CACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/pip'
-    SCCACHE_DIR: '${{CI_PROJECT_DIR}}/.cache/sccache'
     EXT_CACHE_VENV: '${{CI_PROJECT_DIR}}/.cache/ext_cache_venv'
   before_script: |
     ulimit -c unlimited
@@ -26,13 +25,13 @@
     deactivate
     $SHELL scripts/save-ext-cache.sh
   cache:
-    # Share pip/sccache between jobs of the same Python version
-    - key: v1-build_base_venvs-${{PYTHON_VERSION}}-cache-{current_month}
+    # Share pip between jobs of the same Python version
+    - key: v2-build_base_venvs-${{PYTHON_VERSION}}-cache-{current_month}
       paths:
         - .cache
-    - key: v1-build_base_venvs-${{PYTHON_VERSION}}-ext-{current_month}
+    - key: v2-build_base_venvs-${{PYTHON_VERSION}}-ext-{current_month}
       paths:
         - .ext_cache
-    - key: v1-build_base_venvs-${{PYTHON_VERSION}}-download-cache-{current_month}
+    - key: v2-build_base_venvs-${{PYTHON_VERSION}}-download-cache-{current_month}
       paths:
         - .download_cache
diff --git a/.gitlab/testrunner.yml b/.gitlab/testrunner.yml
@@ -1,10 +1,17 @@
+variables:
+  TESTRUNNER_IMAGE: registry.ddbuild.io/dd-trace-py:v72171907-b7ecc7e-testrunner-2025.07.29@sha256:0951475f34dad2b2eed3f30f79895676dd6fb9c1ae2563c93729942ebbb269aa
+
 .testrunner:
-  image: ghcr.io/datadog/dd-trace-py/testrunner:8b69a2610342b333f8832422ffc4f3a9327bed13@sha256:c2d067947ffdb305fc7dc7ff1f8eb7035cfa110bd1199917dd2519eadd166402
+  image:
+    name: ${TESTRUNNER_IMAGE}
+    docker:
+      user: bits
   # DEV: we have a larger pool of amd64 runners, prefer that over arm64
   tags: [ "arch:amd64" ]
   timeout: 20m
   before_script:
     - ulimit -c unlimited
+    - git config --global --add safe.directory ${CI_PROJECT_DIR}
     - pyenv global 3.12 3.8 3.9 3.10 3.11 3.13
     - export _CI_DD_AGENT_URL=http://${HOST_IP}:8126/
   retry: 2
diff --git a/.gitlab/tests.yml b/.gitlab/tests.yml
@@ -98,7 +98,7 @@ include:
     # DEV: All job variables get shared with services, setting `DD_TRACE_AGENT_URL` on the testagent will tell it to forward all requests to the
     # agent at that host. Therefore setting this as a variable will cause recursive requests to the testagent
     - export DD_TRACE_AGENT_URL="http://testagent:9126"
-    - ln -s "${CI_PROJECT_DIR}" "/root/project"
+    - ln -s "${CI_PROJECT_DIR}" "/home/bits/project"
 
 
 # Required jobs will appear here
diff --git a/.riot/requirements/1336cbd.txt b/.riot/requirements/1336cbd.txt
@@ -22,7 +22,7 @@ pastedeploy==3.1.0
 plaster==1.1.2
 plaster-pastedeploy==1.0.1
 pluggy==1.5.0
-pserve-test-app @ file:///root/project/tests/contrib/pyramid/pserve_app
+pserve-test-app @ file:///home/bits/project/tests/contrib/pyramid/pserve_app
 pyramid==2.0.2
 pytest==8.3.1
 pytest-cov==5.0.0
diff --git a/.riot/requirements/192e4d0.txt b/.riot/requirements/192e4d0.txt
@@ -22,7 +22,7 @@ pastedeploy==3.1.0
 plaster==1.1.2
 plaster-pastedeploy==1.0.1
 pluggy==1.5.0
-pserve-test-app @ file:///root/project/tests/contrib/pyramid/pserve_app
+pserve-test-app @ file:///home/bits/project/tests/contrib/pyramid/pserve_app
 pyramid==2.0.2
 pytest==8.3.1
 pytest-cov==5.0.0
diff --git a/.riot/requirements/1e70094.txt b/.riot/requirements/1e70094.txt
@@ -20,7 +20,7 @@ pastedeploy==3.1.0
 plaster==1.1.2
 plaster-pastedeploy==1.0.1
 pluggy==1.5.0
-pserve-test-app @ file:///root/project/tests/contrib/pyramid/pserve_app
+pserve-test-app @ file:///home/bits/project/tests/contrib/pyramid/pserve_app
 pyramid==2.0.2
 pytest==8.3.3
 pytest-cov==5.0.0
diff --git a/.riot/requirements/26b7f73.txt b/.riot/requirements/26b7f73.txt
@@ -22,7 +22,7 @@ pastedeploy==3.1.0
 plaster==1.1.2
 plaster-pastedeploy==1.0.1
 pluggy==1.5.0
-pserve-test-app @ file:///root/project/tests/contrib/pyramid/pserve_app
+pserve-test-app @ file:///home/bits/project/tests/contrib/pyramid/pserve_app
 pyramid==1.10.8
 pytest==8.3.1
 pytest-cov==5.0.0
diff --git a/.riot/requirements/3348fe3.txt b/.riot/requirements/3348fe3.txt
@@ -22,7 +22,7 @@ pastedeploy==3.1.0
 plaster==1.1.2
 plaster-pastedeploy==1.0.1
 pluggy==1.5.0
-pserve-test-app @ file:///root/project/tests/contrib/pyramid/pserve_app
+pserve-test-app @ file:///home/bits/project/tests/contrib/pyramid/pserve_app
 pyramid==2.0.2
 pytest==8.3.1
 pytest-cov==5.0.0
diff --git a/.riot/requirements/95aa957.txt b/.riot/requirements/95aa957.txt
@@ -21,7 +21,7 @@ pastedeploy==3.1.0
 plaster==1.1.2
 plaster-pastedeploy==1.0.1
 pluggy==1.5.0
-pserve-test-app @ file:///root/project/tests/contrib/pyramid/pserve_app
+pserve-test-app @ file:///home/bits/project/tests/contrib/pyramid/pserve_app
 pyramid==2.0.2
 pytest==8.3.1
 pytest-cov==5.0.0
diff --git a/.riot/requirements/97d2271.txt b/.riot/requirements/97d2271.txt
@@ -22,7 +22,7 @@ pastedeploy==3.1.0
 plaster==1.1.2
 plaster-pastedeploy==1.0.1
 pluggy==1.5.0
-pserve-test-app @ file:///root/project/tests/contrib/pyramid/pserve_app
+pserve-test-app @ file:///home/bits/project/tests/contrib/pyramid/pserve_app
 pyramid==2.0.2
 pytest==8.3.1
 pytest-cov==5.0.0
diff --git a/.riot/requirements/b436a4c.txt b/.riot/requirements/b436a4c.txt
@@ -22,7 +22,7 @@ pastedeploy==3.1.0
 plaster==1.1.2
 plaster-pastedeploy==1.0.1
 pluggy==1.5.0
-pserve-test-app @ file:///root/project/tests/contrib/pyramid/pserve_app
+pserve-test-app @ file:///home/bits/project/tests/contrib/pyramid/pserve_app
 pyramid==1.10.8
 pytest==8.3.1
 pytest-cov==5.0.0
diff --git a/.riot/requirements/b56d9af.txt b/.riot/requirements/b56d9af.txt
@@ -20,7 +20,7 @@ pastedeploy==3.1.0
 plaster==1.1.2
 plaster-pastedeploy==1.0.1
 pluggy==1.5.0
-pserve-test-app @ file:///root/project/tests/contrib/pyramid/pserve_app
+pserve-test-app @ file:///home/bits/project/tests/contrib/pyramid/pserve_app
 pyramid==2.0.2
 pytest==8.3.1
 pytest-cov==5.0.0
diff --git a/.riot/requirements/d7f052d.txt b/.riot/requirements/d7f052d.txt
@@ -20,7 +20,7 @@ pastedeploy==3.1.0
 plaster==1.1.2
 plaster-pastedeploy==1.0.1
 pluggy==1.5.0
-pserve-test-app @ file:///root/project/tests/contrib/pyramid/pserve_app
+pserve-test-app @ file:///home/bits/project/tests/contrib/pyramid/pserve_app
 pyramid==2.0.2
 pytest==8.3.1
 pytest-cov==5.0.0
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -174,19 +174,19 @@ services:
         # build:
         #  context: ./docker
         #  dockerfile: Dockerfile
-        image: ghcr.io/datadog/dd-trace-py/testrunner:8b69a2610342b333f8832422ffc4f3a9327bed13@sha256:c2d067947ffdb305fc7dc7ff1f8eb7035cfa110bd1199917dd2519eadd166402
+        image: ghcr.io/datadog/dd-trace-py/testrunner:ae4c189ebf8e539f39905f21c7918cc19de69d13@sha256:9476c426c677d39a58c170ed3167e4d630ed4f067a8a2be2fb96d843795d2ac2
         command: bash
         environment:
           DD_FAST_BUILD: "1"
           CMAKE_BUILD_PARALLEL_LEVEL: "12"
           CARGO_BUILD_JOBS: "12"
         network_mode: host
         userns_mode: host
-        working_dir: /root/project/
+        working_dir: /home/bits/project/
         volumes:
           - ddagent:/tmp/ddagent
-          - ./:/root/project
-          - ./.riot:/root/project/.riot
+          - ./:/home/bits/project
+          - ./.riot:/home/bits/project/.riot
 
     localstack:
         image: localstack/localstack:1.4.0
diff --git a/scripts/ddtest b/scripts/ddtest
@@ -14,4 +14,4 @@ docker compose run \
        --rm \
        -i \
        testrunner \
-       bash -c "(git config --global --add safe.directory /root/project || true) && $CMD"
+       bash -c "(git config --global --add safe.directory /home/bits/project || true) && $CMD"
diff --git a/scripts/docs/install.sh b/scripts/docs/install.sh
@@ -8,12 +8,5 @@ if [[ "${READTHEDOCS}" = "True" ]]; then
 else
   if [[ "$(uname)" == "Darwin" ]]; then
     brew install enchant
-  else
-    export DEBIAN_FRONTEND="noninteractive"
-    export DEBCONF_NOWARNINGS="yes"
-
-    apt-get -qq update
-    apt-get -qy install --no-install-recommends libenchant-2-dev >/dev/null ||
-        apt-get -qy install --no-install-recommends libenchant-dev >/dev/null
   fi
 fi
diff --git a/scripts/gen_gitlab_config.py b/scripts/gen_gitlab_config.py
@@ -84,12 +84,12 @@ def __str__(self) -> str:
                 subprocess.check_output([".gitlab/scripts/get-riot-pip-cache-key.sh", suite_name]).decode().strip()
             )
             lines.append("  cache:")
-            lines.append("    key: v0-pip-${PIP_CACHE_KEY}-cache")
+            lines.append("    key: v1-pip-${PIP_CACHE_KEY}-cache")
             lines.append("    paths:")
             lines.append("      - .cache")
         else:
             lines.append("  cache:")
-            lines.append("    key: v0-${CI_JOB_NAME}-pip-cache")
+            lines.append("    key: v1-${CI_JOB_NAME}-pip-cache")
             lines.append("    paths:")
             lines.append("      - .cache")
 
@@ -212,7 +212,7 @@ def gen_build_docs() -> None:
             print("      hatch run docs:build", file=f)
             print("      mkdir -p /tmp/docs", file=f)
             print("  cache:", file=f)
-            print("    key: v1-build_docs-pip-cache", file=f)
+            print("    key: v2-build_docs-pip-cache", file=f)
             print("    paths:", file=f)
             print("      - .cache", file=f)
             print("  artifacts:", file=f)
@@ -296,7 +296,7 @@ def check(name: str, command: str, paths: t.Set[str]) -> None:
         f.write(
             """
   cache:
-    key: v1-precheck-pip-cache
+    key: v2-precheck-pip-cache
     paths:
       - .cache
 """
diff --git a/scripts/needs_testrun.py b/scripts/needs_testrun.py
@@ -84,29 +84,11 @@ def get_latest_commit_message() -> str:
     return ""
 
 
-GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN")
+GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", os.environ.get("GH_TOKEN"))
 if not GITHUB_TOKEN:
     try:
-        GITHUB_TOKEN = (
-            check_output(
-                [
-                    "aws",
-                    "ssm",
-                    "get-parameter",
-                    "--region",
-                    "us-east-1",
-                    "--name",
-                    f"ci.{os.environ['CI_PROJECT_NAME']}.gh_token",
-                    "--with-decryption",
-                    "--query",
-                    "Parameter.Value",
-                    "--output=text",
-                ]
-            )
-            .decode("utf-8")
-            .strip()
-        )
-        LOGGER.info("GitHub token retrieved from SSM")
+        GITHUB_TOKEN = check_output(["gh", "auth", "token"], text=True).strip()
+        LOGGER.info("GitHub token retrieved from gh auth token")
     except Exception:
         LOGGER.warning("No GitHub token available. Changes may not be detected accurately.", exc_info=True)
 else:
diff --git a/tests/ci_visibility/util.py b/tests/ci_visibility/util.py
@@ -177,6 +177,7 @@ def _get_default_os_env_vars():
         "DD_TRACE_AGENT_PORT",
         "DD_AGENT_HOST",
         "DD_TRACE_AGENT_HOSTNAME",
+        "HOME",
     }
 
     return {key: os.environ[key] for key in os_env_keys if key in os.environ}
diff --git a/tests/contrib/mariadb/test_mariadb.py b/tests/contrib/mariadb/test_mariadb.py
@@ -57,8 +57,8 @@ def test_connection_no_port_or_user_does_not_raise():
     try:
         mariadb.connect(**conf)
     except mariadb.OperationalError as exc:
-        # this error is expected because mariadb defaults user to root when not given
-        if "Access denied for user 'root'" not in str(exc):
+        # this error is expected because mariadb defaults user to root/bits when not given
+        if "Access denied for user '" not in str(exc):
             raise exc
 
 
