Fix probes when Qdrant is served over TLS

This also improves the integration test runtime and reliability by not installing a fresh cluster for every test but upgrading it with the new config.

Fixes #77

charts/qdrant/templates/statefulset.yaml

@@ -100,6 +100,9 @@ spec:
             httpGet:
               path: /livez
               port: {{ .targetPort }}
+              {{- if and $values.config.service $values.config.service.enable_tls }}
+              scheme: HTTPS
+              {{- end }}
             {{- end }}
             initialDelaySeconds: {{ $values.livenessProbe.initialDelaySeconds }}
             timeoutSeconds: {{ $values.livenessProbe.timeoutSeconds }}
@@ -117,6 +120,9 @@ spec:
             httpGet:
               path: /readyz
               port: {{ .targetPort }}
+              {{- if and $values.config.service $values.config.service.enable_tls }}
+              scheme: HTTPS
+              {{- end }}
             {{- end }}
             initialDelaySeconds: {{ $values.readinessProbe.initialDelaySeconds }}
             timeoutSeconds: {{ $values.readinessProbe.timeoutSeconds }}
@@ -134,6 +140,9 @@ spec:
             httpGet:
               path: /readyz
               port: {{ .targetPort }}
+              {{- if and $values.config.service $values.config.service.enable_tls }}
+              scheme: HTTPS
+              {{- end }}
             {{- end }}
             initialDelaySeconds: {{ $values.startupProbe.initialDelaySeconds }}
             timeoutSeconds: {{ $values.startupProbe.timeoutSeconds }}

charts/qdrant/templates/tests/test-db-interaction.yaml

@@ -11,15 +11,21 @@ metadata:
 spec:
   containers:
     - name: test-script
-      image: registry.suse.com/bci/python:3.11
+      image: registry.suse.com/bci/bci-base:latest
       args: ['bash', '/app/entrypoint.sh']
       volumeMounts:
         - mountPath: /app
           name: test-script
+        {{- if .Values.additionalVolumeMounts }}
+{{- toYaml .Values.additionalVolumeMounts | default "" | nindent 8 }}
+        {{- end}}
   volumes:
     - name: test-script
       configMap:
         name: "{{ include "qdrant.fullname" . }}-test-db-interaction"
+    {{- if .Values.additionalVolumes }}
+{{- toYaml .Values.additionalVolumes | default "" | nindent 4 }}
+    {{- end}}
   restartPolicy: Never
   serviceAccountName: {{ include "qdrant.fullname" . }}
 ---
@@ -35,43 +41,56 @@ data:
   entrypoint.sh: |
     #!/bin/bash
     set -xe
-    zypper install -y python311-dbm
-    pip3 install qdrant-client=={{ .Chart.AppVersion }}
-    python3 /app/test.py
-  test.py: |
-    from qdrant_client import QdrantClient
-    from qdrant_client.http.models import Distance, VectorParams
-    from qdrant_client.http.models import PointStruct
+    echo 'connect-timeout = 5' > ~/.curlrc
 
+    if [ -d /mnt/secrets/certs ]; then
+      cp /mnt/secrets/certs/ca.pem /usr/share/pki/trust/anchors/private-ca.pem
+      update-ca-certificates
+    fi
+
+    QDRANT_COLLECTION="test_collection"
     {{- range .Values.service.ports }}
     {{- if eq .name "http" }}
-    print("Connecting to {{ include "qdrant.fullname" $root }}.{{ $namespace }}:{{ .port }}")
-    client = QdrantClient("{{ include "qdrant.fullname" $root }}.{{ $namespace }}", port={{ .port }})
+    echo "Connecting to {{ include "qdrant.fullname" $root }}.{{ $namespace }}:{{ .port }}"
+    QDRANT_URL="http://{{ include "qdrant.fullname" $root }}.{{ $namespace }}:{{ .port }}"
+    {{- if and $root.Values.config.service $root.Values.config.service.enable_tls }}
+    echo "Using https"
+    QDRANT_URL="https://{{ include "qdrant.fullname" $root }}.{{ $namespace }}:{{ .port }}"
+    {{- end }}
     {{- end }}
     {{- end }}
+    API_KEY_HEADER=""
+    {{- if .Values.apiKey }}
+    API_KEY_HEADER="Api-key: {{ .Values.apiKey }}"
+    {{- end }}
+
+    # Delete collection if exists
+    curl -X DELETE -H "${API_KEY_HEADER}" $QDRANT_URL/collections/${QDRANT_COLLECTION}
 
-    client.recreate_collection(
-        collection_name="test_collection",
-        vectors_config=VectorParams(size=4, distance=Distance.DOT),
-    )
+    # Create collection
+    curl -X PUT \
+    -H 'Content-Type: application-json' \
+    -d '{"vectors":{"size":4,"distance":"Dot"}}' \
+    -H  "${API_KEY_HEADER}" \
+    $QDRANT_URL/collections/${QDRANT_COLLECTION} --fail-with-body
 
-    operation_info = client.upsert(
-      collection_name="test_collection",
-      wait=True,
-      points=[
-        PointStruct(id=1, vector=[0.05, 0.61, 0.76, 0.74], payload={"city": "Berlin"}),
-        PointStruct(id=2, vector=[0.19, 0.81, 0.75, 0.11], payload={"city": "London"}),
-        PointStruct(id=3, vector=[0.36, 0.55, 0.47, 0.94], payload={"city": "Moscow"}),
-        PointStruct(id=4, vector=[0.18, 0.01, 0.85, 0.80], payload={"city": "New York"}),
-        PointStruct(id=5, vector=[0.24, 0.18, 0.22, 0.44], payload={"city": "Beijing"}),
-        PointStruct(id=6, vector=[0.35, 0.08, 0.11, 0.44], payload={"city": "Mumbai"}),
-        ]
-    )
-    print(operation_info)
+    # Insert points
+    curl -X PUT \
+    -H 'Content-Type: application-json' \
+    -d '{"points":[
+      {"id":1,"vector":[0.05, 0.61, 0.76, 0.74],"payload":{"city":"Berlin"}},
+      {"id":2,"vector":[0.19, 0.81, 0.75, 0.11],"payload":{"city":"London"}},
+      {"id":3,"vector":[0.36, 0.55, 0.47, 0.94],"payload":{"city":"Moscow"}},
+      {"id":4,"vector":[0.18, 0.01, 0.85, 0.80],"payload":{"city":"New York"}},
+      {"id":5,"vector":[0.24, 0.18, 0.22, 0.44],"payload":{"city":"Beijing"}},
+      {"id":6,"vector":[0.35, 0.08, 0.11, 0.44],"payload":{"city":"Mumbai"}}
+    ]}' \
+    -H  "${API_KEY_HEADER}" \
+    $QDRANT_URL/collections/${QDRANT_COLLECTION}/points --fail-with-body
 
-    search_result = client.search(
-      collection_name="test_collection",
-      query_vector=[0.2, 0.1, 0.9, 0.7],
-      limit=3
-    )
-    print(search_result)
+    # Run query
+    curl -X POST \
+    -H 'Content-Type: application-json' \
+    -d '{"vector":[0.2, 0.1, 0.9, 0.7],"limit":3}' \
+    -H  "${API_KEY_HEADER}" \
+    $QDRANT_URL/collections/${QDRANT_COLLECTION}/points/search --fail-with-body

test/integration/api_key.bats

@@ -1,20 +1,12 @@
 setup_file() {
-    kubectl create namespace qdrant-helm-integration
-    kubectl create serviceaccount default -n qdrant-helm-integration || true
-    helm install qdrant charts/qdrant --set apiKey=foobar -n qdrant-helm-integration --wait
+    helm upgrade --install qdrant charts/qdrant --set apiKey=foobar -n qdrant-helm-integration --wait
     kubectl rollout status statefulset qdrant -n qdrant-helm-integration
 }
 
-teardown_file() {
-    helm uninstall qdrant -n qdrant-helm-integration
-    kubectl delete serviceaccount default -n qdrant-helm-integration
-    kubectl delete namespace qdrant-helm-integration
-}
-
 @test "api key authentication works" {
-    run kubectl exec -n default curl -- curl -s http://qdrant.qdrant-helm-integration:6333/collections -H 'api-key: foobar'
+    run kubectl exec -n default curl -- curl -s http://qdrant.qdrant-helm-integration:6333/collections -H 'api-key: foobar' --fail-with-body
     [ $status -eq 0 ]
-    [ "${output}" != "Invalid api-key" ]
+    [[ "${output}" =~ .*\"status\":\"ok\".* ]]
 }
 
 @test "api key authentication fails with key" {

test/integration/assets/.gitignore

@@ -0,0 +1,6 @@
+*.key
+*.crt
+*.csr
+*.pem
+*.ext
+*.srl

test/integration/assets/tls-values.yaml

@@ -0,0 +1,19 @@
+additionalVolumes:
+  - secret:
+      secretName: test-tls
+    name: tls
+
+additionalVolumeMounts:
+  - name: tls
+    mountPath: /mnt/secrets/certs
+
+config:
+  service:
+    enable_tls: true
+  tls:
+    cert: "/mnt/secrets/certs/tls.crt"
+    key: "/mnt/secrets/certs/tls.key"
+    ca_cert: "/mnt/secrets/certs/ca.pem"
+  cluster:
+    p2p:
+      enable_tls: true

test/integration/default_installation.bats

@@ -1,17 +1,6 @@
 setup_file() {
-    kubectl create namespace qdrant-helm-integration
-    kubectl create serviceaccount default -n qdrant-helm-integration || true
-    helm install qdrant charts/qdrant -n qdrant-helm-integration --wait
+    helm upgrade --install qdrant charts/qdrant -n qdrant-helm-integration --wait
     kubectl rollout status statefulset qdrant -n qdrant-helm-integration
-    # wait a bit to ensure that qdrant is really up, and we don't get connection refused
-    # errors in Github Actions
-    sleep 10
-}
-
-teardown_file() {
-    helm uninstall qdrant -n qdrant-helm-integration
-    kubectl delete serviceaccount default -n qdrant-helm-integration
-    kubectl delete namespace qdrant-helm-integration
 }
 
 @test "helm test - default values" {

test/integration/https.bats

@@ -0,0 +1,24 @@
+setup_file() {
+    rm test/integration/assets/ca.*
+    rm test/integration/assets/tls.*
+    openssl genrsa -des3 -passout pass:insecure -out test/integration/assets/ca.key 2048
+    openssl req -x509 -passin pass:insecure -new -nodes -key test/integration/assets/ca.key -sha256 -days 1825 -out test/integration/assets/ca.pem -subj "/C=DE/ST=Berlin/L=Berlin/O=Qdrant/OU=Cloud/CN=qdrant"
+    openssl genrsa -out test/integration/assets/tls.key 2048
+    openssl req -new -key test/integration/assets/tls.key -out test/integration/assets/tls.csr -subj "/C=DE/ST=Berlin/L=Berlin/O=Qdrant/OU=Cloud/CN=qdrant.qdrant-helm-integration"
+    openssl x509 -req -passin pass:insecure -in test/integration/assets/tls.csr -CA test/integration/assets/ca.pem -CAkey test/integration/assets/ca.key -CAcreateserial -out test/integration/assets/tls.crt -days 825 -sha256
+
+    kubectl -n qdrant-helm-integration create secret generic test-tls --from-file=tls.key="test/integration/assets/tls.key" --from-file=tls.crt="test/integration/assets/tls.crt" --from-file=ca.pem="test/integration/assets/ca.pem"
+    helm upgrade --install qdrant charts/qdrant -n qdrant-helm-integration --wait -f test/integration/assets/tls-values.yaml
+    kubectl rollout status statefulset qdrant -n qdrant-helm-integration
+}
+
+@test "Connection with https" {
+    run kubectl exec -n default curl -- curl -k -s https://qdrant.qdrant-helm-integration:6333/collections --fail-with-body
+    [ $status -eq 0 ]
+    [[ "${output}" =~ .*\"status\":\"ok\".* ]]
+}
+
+@test "helm test - with https" {
+    run helm test qdrant -n qdrant-helm-integration --logs
+    [ $status -eq 0 ]
+}

test/integration/random_api_key.bats

@@ -1,22 +1,14 @@
 setup_file() {
-    kubectl create namespace qdrant-helm-integration
-    kubectl create serviceaccount default -n qdrant-helm-integration || true
-    helm install qdrant charts/qdrant --set apiKey=true -n qdrant-helm-integration --wait
+    helm upgrade --install qdrant charts/qdrant --set apiKey=true -n qdrant-helm-integration --wait
     kubectl rollout status statefulset qdrant -n qdrant-helm-integration
 }
 
-teardown_file() {
-    helm uninstall qdrant -n qdrant-helm-integration
-    kubectl delete serviceaccount default -n qdrant-helm-integration
-    kubectl delete namespace qdrant-helm-integration
-}
-
 @test "random api key works" {
     apiKey=$(kubectl -n qdrant-helm-integration get secret qdrant-apikey -o jsonpath="{.data.api-key}" | base64 --decode)
     [ ${#apiKey} -eq 32 ]
-    run kubectl exec -n default curl -- curl -s http://qdrant.qdrant-helm-integration:6333/collections -H "api-key: ${apiKey}"
+    run kubectl exec -n default curl -- curl -s http://qdrant.qdrant-helm-integration:6333/collections -H "api-key: ${apiKey}" --fail-with-body
     [ $status -eq 0 ]
-    [ "${output}" != "Invalid api-key" ]
+    [[ "${output}" =~ .*\"status\":\"ok\".* ]]
 }
 
 @test "random api key stays the same after upgrade" {

test/integration/security_context.bats

@@ -1,16 +1,5 @@
-setup_file() {
-    kubectl create namespace qdrant-helm-integration
-    kubectl create serviceaccount default -n qdrant-helm-integration || true
-}
-
-teardown_file() {
-    helm uninstall qdrant -n qdrant-helm-integration
-    kubectl delete serviceaccount default -n qdrant-helm-integration
-    kubectl delete namespace qdrant-helm-integration
-}
-
 @test "update without security context to security context corrects file permissions" {
-    helm install qdrant charts/qdrant --set-json 'podSecurityContext=false,containerSecurityContext=false' -n qdrant-helm-integration --wait
+    helm upgrade --install qdrant charts/qdrant --set-json 'podSecurityContext=false,containerSecurityContext=false' -n qdrant-helm-integration --wait
     kubectl rollout status statefulset qdrant -n qdrant-helm-integration
     user=$(kubectl exec qdrant-0 -n qdrant-helm-integration -- id -u)
     [ "${user}" = "0" ]

test/integration/setup_suite.bash

@@ -3,6 +3,8 @@ setup_suite() {
     kubectl create serviceaccount default -n default
     kubectl -n default run curl --image=curlimages/curl --command -- sh -c "sleep 3600"
     kubectl wait --for=condition=Ready pod/curl -n default --timeout=300s
+    kubectl create namespace qdrant-helm-integration
+    kubectl create serviceaccount default -n qdrant-helm-integration || true
 }
 
 teardown_suite() {

test/integration/unprivileged_image.bats

@@ -1,17 +1,6 @@
 setup_file() {
-    kubectl create namespace qdrant-helm-integration
-    kubectl create serviceaccount default -n qdrant-helm-integration || true
-    helm install qdrant charts/qdrant -n qdrant-helm-integration --wait --set image.useUnprivilegedImage=true
+    helm upgrade --install qdrant charts/qdrant -n qdrant-helm-integration --wait --set image.useUnprivilegedImage=true
     kubectl rollout status statefulset qdrant -n qdrant-helm-integration
-    # wait a bit to ensure that qdrant is really up, and we don't get connection refused
-    # errors in Github Actions
-    sleep 10
-}
-
-teardown_file() {
-    helm uninstall qdrant -n qdrant-helm-integration
-    kubectl delete serviceaccount default -n qdrant-helm-integration
-    kubectl delete namespace qdrant-helm-integration
 }
 
 @test "helm test - with unprivileged image" {