[BLD]: save OTEL traces in artifact during CI (#3378)

.github/actions/export-tilt-logs/action.yaml

@@ -0,0 +1,19 @@
+name: Export Tilt logs
+description: Exports logs and traces from services in Tilt to an artifact
+inputs:
+  artifact-name:
+    description: "The name of the artifact containing logs and traces from services in Tilt"
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Get logs of all services
+      id: get-logs
+      run: |
+        bin/get-logs.sh ${{ inputs.artifact-name }}.zip
+      shell: bash
+    - name: Upload logs as artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ inputs.artifact-name }}
+        path: "${{ inputs.artifact-name }}.zip"

.github/actions/tilt/action.yaml

@@ -31,3 +31,4 @@ runs:
         kubectl -n chroma port-forward svc/query-service 50053:50051 &
         kubectl -n chroma port-forward svc/frontend-service 8000:8000 &
         kubectl -n chroma port-forward svc/minio 9000:9000 &
+        kubectl -n chroma port-forward svc/jaeger 16686:16686 &

.github/workflows/_go-tests.yml

@@ -44,3 +44,8 @@ jobs:
         uses: ./.github/actions/go
       - uses: ./.github/actions/tilt
       - run: bin/cluster-test.sh bash -c 'cd go && go test -timeout 30s -run ^TestNodeWatcher$ github.com/chroma-core/chroma/go/pkg/memberlist_manager'
+      - name: Save service logs to artifact
+        if: always()
+        uses: ./.github/actions/export-tilt-logs
+        with:
+          artifact-name: "go-cluster-test"

.github/workflows/_python-tests.yml

@@ -118,21 +118,15 @@ jobs:
         shell: bash
         env:
           PROPERTY_TESTING_PRESET: ${{ inputs.property_testing_preset }}
-      - name: Get logs of all services
-        id: get-logs
-        if: success() || failure()
-        run: |
-          bin/get-logs.sh "${{ matrix.test-globs }}" "${{ matrix.python }}"
-          # Output the logs zip file path as a job output
-          echo "artifact_name=cluster_logs_$(basename "${{ matrix.test-globs }}" .py)_${{ matrix.python }}" >> $GITHUB_OUTPUT
-          echo "logs_zip_path=$(pwd)/$(basename "${{ matrix.test-globs }}" .py)_${{ matrix.python }}_logs.zip" >> $GITHUB_OUTPUT
-        shell: bash
-      - name: Upload logs as artifact
-        if: success() || failure()
-        uses: actions/upload-artifact@v4
+      - name: Compute artifact name
+        if: always()
+        id: compute-artifact-name
+        run: echo "artifact_name=cluster_logs_$(basename "${{ matrix.test-globs }}" .py)_${{ matrix.python }}" >> $GITHUB_OUTPUT
+      - name: Save service logs to artifact
+        if: always()
+        uses: ./.github/actions/export-tilt-logs
         with:
-          name: ${{ steps.get-logs.outputs.artifact_name }}
-          path: ${{ steps.get-logs.outputs.logs_zip_path }}
+          artifact-name: ${{ steps.compute-artifact-name.outputs.artifact_name }}
 
   merge-cluster-logs:
     runs-on: ubuntu-latest

.github/workflows/_rust-tests.yml

@@ -36,6 +36,12 @@ jobs:
         uses: ./.github/actions/tilt
       - name: Run tests
         run: cargo nextest run --profile k8s_integration
+      - name: Save service logs to artifact
+        if: always()
+        uses: ./.github/actions/export-tilt-logs
+        with:
+          artifact-name: "rust-integration-test"
+
   test-benches:
     strategy:
       matrix:

.github/workflows/nightly-tests.yml

@@ -27,21 +27,11 @@ jobs:
         shell: bash
         env:
           PROPERTY_TESTING_PRESET: slow
-      - name: Get logs of all services
-        id: get-logs
-        if: success() || failure()
-        run: |
-          bin/get-logs.sh "${{ matrix.test-globs }}" "3.12"
-          # Output the logs zip file path as a job output
-          echo "artifact_name=cluster_logs_$(basename "${{ matrix.test-globs }}" .py)_3.12" >> $GITHUB_OUTPUT
-          echo "logs_zip_path=$(pwd)/$(basename "${{ matrix.test-globs }}" .py)_3.12_logs.zip" >> $GITHUB_OUTPUT
-        shell: bash
-      - name: Upload logs as artifact
-        if: success() || failure()
-        uses: actions/upload-artifact@v4
+      - name: Save service logs to artifact
+        if: always()
+        uses: ./.github/actions/export-tilt-logs
         with:
-          name: ${{ steps.get-logs.outputs.artifact_name }}
-          path: ${{ steps.get-logs.outputs.logs_zip_path }}
+          artifact-name: "tilt-logs"
       - name: Send PagerDuty alert on failure
         if: ${{ failure() }}
         uses: Entle/action-pagerduty-alert@0.2.0

bin/get-logs.sh

@@ -6,24 +6,16 @@ NAMESPACE=chroma
 echo "Namespace: $NAMESPACE"
 
 # Check if the test-name and version-number are provided as arguments
-if [ -z "$1" ] || [ -z "$2" ]; then
-  echo "Usage: $0 <test-name> <version-number>"
+if [ -z "$1" ]; then
+  echo "Usage: $0 <output-file-path>"
   exit 1  # Exit with code 1 indicating an error
 fi
 
-TEST_FILE_PATH=$1
-TEST_VERSION=$2
+OUTPUT_FILE_PATH=$(readlink -m $1)
+TEMP_DIR=$(mktemp -d)
 
-# Extract the test file name from the path and remove the ".py" extension
-TEST_NAME=$(basename "$TEST_FILE_PATH" .py)
-echo "Test name: $TEST_NAME"
-echo "Test version: $TEST_VERSION"
-
-# Create a directory with the test name to store logs
-LOGS_DIR="./logs/${TEST_NAME}_${TEST_VERSION}"
-echo "Logs directory: $LOGS_DIR"
-mkdir -p "$LOGS_DIR"
-echo "Created logs dir: $LOGS_DIR"
+mkdir "$TEMP_DIR/logs"
+mkdir "$TEMP_DIR/traces"
 
 # Get the list of all pods in the namespace
 PODS=$(kubectl get pods -n $NAMESPACE -o jsonpath='{.items[*].metadata.name}')
@@ -33,14 +25,14 @@ echo "Got all the pods: $PODS"
 for POD in $PODS; do
   echo "Getting logs for pod: $POD"
   # Save the logs to a file named after the pod and test name
-  kubectl logs $POD -n $NAMESPACE --since=0s > "${LOGS_DIR}/${POD}_logs.txt"
+  kubectl logs $POD -n $NAMESPACE --since=0s > "${TEMP_DIR}/logs/${POD}.txt" || true
 done
 
+# Get traces from Jaeger for all services
+curl "http://localhost:16686/api/services" | jq -r '.data[]' | while read -r service; do curl "http://localhost:16686/api/traces?limit=100&lookback=1h&maxDuration&minDuration&service=$service" > "$TEMP_DIR/traces/$service.json" || true; done
+
 # Zip all log files
-zip -r "${TEST_NAME}_${TEST_VERSION}_logs.zip" "$LOGS_DIR"
+cd $TEMP_DIR &&zip -r "$OUTPUT_FILE_PATH" . && cd -
 
 # Print confirmation message
-echo "Logs have been zipped to ${TEST_NAME}_${TEST_VERSION}_logs.zip"
-
-# Output the path to the zip file
-echo "logs_zip_path=$(pwd)/${TEST_NAME}_${TEST_VERSION}_logs.zip" >> $GITHUB_OUTPUT
+echo "Logs have been zipped to $OUTPUT_FILE_PATH"