Test: Added test for index replication

.github/workflows/tests-cluster-chainsaw.yaml

@@ -59,6 +59,29 @@ jobs:
           echo "Chart.yaml:"
           cat Chart.yaml
 
+      # Added by ParadeDB: Authenticate to Docker Hub to avoid rate limits
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_ACCESS_TOKEN }}
+
+      # Added by ParadeDB: Always pull the latest version of paradedb/paradedb
+      - name: Set ParadeDB Version to Latest
+        working-directory: charts/paradedb/
+        env:
+          GH_TOKEN: ${{ secrets.GHA_CREATE_RELEASE_PAT }}
+        run: |
+          LATEST_TAG=$(curl -s https://api.github.com/repos/paradedb/paradedb/tags | jq -r '.[0].name')
+          APP_VERSION=${LATEST_TAG#v}
+          sed -i "s/^[[:space:]]*paradedb: .*/  paradedb: \"$APP_VERSION\"/" values.yaml
+          sed -i "s/^version: .*/version: ${{ vars.CHART_VERSION_MAJOR }}.${{ vars.CHART_VERSION_MINOR }}.${{ vars.CHART_VERSION_PATCH }}/" Chart.yaml
+          echo "values.yaml:"
+          cat values.yaml
+          echo "----------------------------------------"
+          echo "Chart.yaml:"
+          cat Chart.yaml
+
       - name: Setup kind
         uses: ./.github/actions/setup-kind
 

charts/paradedb/README.md

@@ -211,15 +211,15 @@ refer to  the [CloudNativePG Documentation](https://cloudnative-pg.io/documentat
 | cluster.monitoring.customQueries | list | `[]` | Custom Prometheus metrics Will be stored in the ConfigMap |
 | cluster.monitoring.customQueriesSecret | list | `[]` | The list of secrets containing the custom queries |
 | cluster.monitoring.disableDefaultQueries | bool | `false` | Whether the default queries should be injected. Set it to true if you don't want to inject default queries into the cluster. |
-| cluster.monitoring.enabled | bool | `false` | Whether to enable monitoring |
+| cluster.monitoring.enabled | bool | `true` | Whether to enable monitoring |
 | cluster.monitoring.podMonitor.enabled | bool | `true` | Whether to enable the PodMonitor |
 | cluster.monitoring.podMonitor.metricRelabelings | list | `[]` | The list of metric relabelings for the PodMonitor. Applied to samples before ingestion. |
 | cluster.monitoring.podMonitor.relabelings | list | `[]` | The list of relabelings for the PodMonitor. Applied to samples before scraping. |
 | cluster.monitoring.prometheusRule.enabled | bool | `true` | Whether to enable the PrometheusRule automated alerts |
 | cluster.monitoring.prometheusRule.excludeRules | list | `[]` | Exclude specified rules |
 | cluster.postgresGID | int | `-1` | The GID of the postgres user inside the image, defaults to 26 |
 | cluster.postgresUID | int | `-1` | The UID of the postgres user inside the image, defaults to 26 |
-| cluster.postgresql.parameters | object | `{}` | PostgreSQL configuration options (postgresql.conf) |
+| cluster.postgresql.parameters | object | `{"cron.database_name":"postgres"}` | PostgreSQL configuration options (postgresql.conf) |
 | cluster.postgresql.pg_hba | list | `[]` | PostgreSQL Host Based Authentication rules (lines to be appended to the pg_hba.conf file) |
 | cluster.postgresql.pg_ident | list | `[]` | PostgreSQL User Name Maps rules (lines to be appended to the pg_ident.conf file) |
 | cluster.postgresql.shared_preload_libraries | list | `[]` | Lists of shared preload libraries to add to the default ones |

charts/paradedb/test/paradedb-enterprise/01-paradedb-NCC-1701-D_cluster-assert.yaml

@@ -3,4 +3,4 @@ kind: Cluster
 metadata:
   name: paradedb-ncc-1701-d
 status:
-  readyInstances: 1
+  readyInstances: 2

charts/paradedb/test/paradedb-enterprise/01-paradedb-NCC-1701-D_cluster.yaml

@@ -4,7 +4,7 @@ version:
   major: "17"
   paradedb: "0.13.2"
 cluster:
-  instances: 1
+  instances: 2
   storage:
     size: 256Mi
   imagePullSecrets:

charts/paradedb/test/paradedb-enterprise/02-paradedb_test.yaml

@@ -24,7 +24,7 @@ spec:
                 schema_name => 'public',
                 table_name => 'mock_items_paradedb_enterprise'
               );
-              CREATE INDEX search_idx_paradedb_enterprise_ ON mock_items_paradedb_enterprise
+              CREATE INDEX search_idx ON mock_items_paradedb_enterprise
               USING bm25 (id, description, category, rating, in_stock, created_at, metadata, weight_range)
               WITH (key_field='id');
             EOSQL
@@ -35,4 +35,9 @@ spec:
               LIMIT 1;
             EOSQL
             echo -$RESULT-
-            test "$RESULT" = " Bluetooth-enabled speaker"
+            if [ "$RESULT" = " Bluetooth-enabled speaker" ]; then
+              echo "Test for description search passed."
+            else
+              echo "Test for description search failed."
+              exit 1
+            fi

charts/paradedb/test/paradedb-enterprise/03-paradedb_test-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: paradedb-enterprise-index-test
+status:
+  succeeded: 1

charts/paradedb/test/paradedb-enterprise/03-paradedb_test.yaml

@@ -0,0 +1,46 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: paradedb-enterprise-index-test
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: data-test
+        env:
+          - name: DB_URI
+            valueFrom:
+              secretKeyRef:
+                name: paradedb-ncc-1701-d-app
+                key: uri
+        image: alpine:3.19
+        command: ['sh', '-c']
+        args:
+          - |
+            apk --no-cache add postgresql-client
+            DB_URI="${DB_URI/paradedb-ncc-1701-d-rw/paradedb-ncc-1701-d-ro}"
+            RESULT=$(psql "$DB_URI" -t) <<-EOSQL
+              SELECT description
+              FROM mock_items_paradedb_enterprise
+              WHERE description @@@ '"bluetooth speaker"~1'
+              LIMIT 1;
+            EOSQL
+            echo -$RESULT-
+            if [ "$RESULT" = " Bluetooth-enabled speaker" ]; then
+              echo "Test for description search on replicas passed."
+            else
+              echo "Test for description search on replicas failed."
+              exit 1
+            fi
+
+            SIZE_RESULT=$(psql "$DB_URI" -t) <<-EOSQL
+              SELECT pg_size_pretty(pg_relation_size('search_idx'));
+            EOSQL
+            echo -$SIZE_RESULT-
+            if [ "$SIZE_RESULT" != " 0 bytes" ]; then
+              echo "Test for index size on replicas passed."
+            else
+              echo "Test for index size on replicas failed."
+              exit 1
+            fi

charts/paradedb/test/paradedb-enterprise/chainsaw-test.yaml

@@ -50,6 +50,25 @@ spec:
             kind: Job
         - podLogs:
             selector: cnpg.io/cluster=paradedb-ncc-1701-d
+        - podLogs:
+            selector: batch.kubernetes.io/job-name=paradedb-enterprise-test
+    - name: Verify index replication
+      timeouts:
+        apply: 1s
+        assert: 30s
+      try:
+        - apply:
+            file: 03-paradedb_test.yaml
+        - assert:
+            file: 03-paradedb_test-assert.yaml
+      catch:
+        - describe:
+            apiVersion: batch/v1
+            kind: Job
+        - podLogs:
+            selector: cnpg.io/cluster=paradedb-ncc-1701-d
+        - podLogs:
+            selector: batch.kubernetes.io/job-name=paradedb-enterprise-index-test
     - name: Cleanup
       try:
         - script:

charts/paradedb/test/postgresql-minio-backup-restore/00-minio_cleanup-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: data-test
+status:
+  succeeded: 1

charts/paradedb/test/postgresql-minio-backup-restore/00-minio_cleanup.yaml

@@ -0,0 +1,16 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: minio-cleanup
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: minio-cleanup
+        image: minio/mc
+        command: ['sh', '-c']
+        args:
+         - |
+           mc alias set myminio https://minio.minio.svc.cluster.local minio minio123
+           mc rm --recursive --force myminio/mybucket/paradedb

charts/paradedb/test/postgresql-minio-backup-restore/01-standalone_cluster-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: import-paradedb
+status:
+  readyInstances: 2

charts/paradedb/test/postgresql-minio-backup-restore/01-standalone_cluster.yaml

@@ -0,0 +1,29 @@
+type: paradedb
+mode: standalone
+version:
+  major: "17"
+  paradedb: "0.13.2"
+cluster:
+  instances: 2
+  storage:
+    size: 256Mi
+
+backups:
+  enabled: true
+  provider: s3
+  endpointURL: "https://minio.minio.svc.cluster.local"
+  endpointCA:
+    name: kube-root-ca.crt
+    key: ca.crt
+  wal:
+    encryption: ""
+  data:
+    encryption: ""
+  s3:
+    bucket: "mybucket"
+    path: "/paradedb/v1"
+    accessKey: "minio"
+    secretKey: "minio123"
+    region: "local"
+  scheduledBackups: []
+  retentionPolicy: "30d"

charts/paradedb/test/postgresql-minio-backup-restore/02-data_write-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: data-write
+status:
+  succeeded: 1

charts/paradedb/test/postgresql-minio-backup-restore/02-data_write.yaml

@@ -0,0 +1,23 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: data-write
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: data-write
+        env:
+          - name: DB_URI
+            valueFrom:
+              secretKeyRef:
+                name: standalone-paradedb-superuser
+                key: uri
+        image: alpine:3.19
+        command: ['sh', '-c']
+        args:
+         - |
+           apk --no-cache add postgresql-client
+           DB_URI=$(echo $DB_URI | sed "s|/\*|/|" )
+           psql "$DB_URI" -c "CREATE TABLE mygoodtable (id serial PRIMARY KEY);"

charts/paradedb/test/postgresql-minio-backup-restore/03-backup.yaml

@@ -0,0 +1,8 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Backup
+metadata:
+  name: post-init-backup
+spec:
+  method: barmanObjectStore
+  cluster:
+    name: standalone-paradedb

charts/paradedb/test/postgresql-minio-backup-restore/03-backup_completed-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Backup
+metadata:
+  name: post-init-backup
+spec:
+  cluster:
+    name: standalone-paradedb
+  method: barmanObjectStore
+status:
+  phase: completed

charts/paradedb/test/postgresql-minio-backup-restore/03-backup_running-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Backup
+metadata:
+  name: post-init-backup
+spec:
+  cluster:
+    name: standalone-paradedb
+  method: barmanObjectStore
+status:
+  phase: running

charts/paradedb/test/postgresql-minio-backup-restore/03-checkpoint.yaml

@@ -0,0 +1,27 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: backup-checkpoint
+spec:
+  template:
+    spec:
+      restartPolicy: OnFailure
+      containers:
+      - name: create-checkpoint
+        env:
+          - name: DB_URI
+            valueFrom:
+              secretKeyRef:
+                name: standalone-paradedb-superuser
+                key: uri
+        image: alpine:3.19
+        command: ['sh', '-c']
+        args:
+         - |
+           apk --no-cache add postgresql-client
+           DB_URI=$(echo $DB_URI | sed "s|/\*|/|" )
+           END_TIME=$(( $(date +%s) + 30 ))
+           while [ $(date +%s) -lt $END_TIME ]; do
+             psql "$DB_URI" -c "SELECT pg_switch_wal();CHECKPOINT;"
+             sleep 5
+           done

charts/paradedb/test/postgresql-minio-backup-restore/04-post_backup_data_write-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: data-write-post-backup
+status:
+  succeeded: 1

charts/paradedb/test/postgresql-minio-backup-restore/04-post_backup_data_write.yaml

@@ -0,0 +1,57 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: configmap-creator-sa
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: configmap-creator
+rules:
+- apiGroups: [""]
+  resources: ["configmaps"]
+  verbs: ["create"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: configmap-creator-binding
+subjects:
+- kind: ServiceAccount
+  name: configmap-creator-sa
+roleRef:
+  kind: Role
+  name: configmap-creator
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: data-write-post-backup
+spec:
+  template:
+    spec:
+      serviceAccountName: configmap-creator-sa
+      restartPolicy: OnFailure
+      containers:
+      - name: data-write
+        env:
+          - name: DB_URI
+            valueFrom:
+              secretKeyRef:
+                name: standalone-paradedb-superuser
+                key: uri
+          - name: NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        image: alpine:3.19
+        command: ['sh', '-c']
+        args:
+         - |
+           apk --no-cache add postgresql-client kubectl coreutils
+           DB_URI=$(echo $DB_URI | sed "s|/\*|/|" )
+           DATE_NO_BAD_TABLE=$(date --rfc-3339=ns)
+           sleep 30
+           psql "$DB_URI" -c "CREATE TABLE mybadtable (id serial PRIMARY KEY);"
+           kubectl create configmap date-no-bad-table --from-literal=date="$DATE_NO_BAD_TABLE"

charts/paradedb/test/postgresql-minio-backup-restore/05-recovery_backup_cluster-assert.yaml

@@ -0,0 +1,6 @@
+apiVersion: postgresql.cnpg.io/v1
+kind: Cluster
+metadata:
+  name: import-schemaonly-paradedb
+status:
+  readyInstances: 2