diff --git a/tests/templates/kuttl/ldap/30-test-opensearch.yaml b/tests/templates/kuttl/ldap/30-test-opensearch.yaml index a428007..7b27c60 100644 --- a/tests/templates/kuttl/ldap/30-test-opensearch.yaml +++ b/tests/templates/kuttl/ldap/30-test-opensearch.yaml @@ -22,6 +22,10 @@ spec: # required for pip install - name: HOME value: /stackable + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace volumeMounts: - name: script mountPath: /stackable/scripts @@ -68,9 +72,12 @@ metadata: name: test-opensearch data: test.py: | + import os from opensearchpy import OpenSearch - host = 'opensearch' + namespace = os.environ['NAMESPACE'] + + host = f'opensearch.{namespace}.svc.cluster.local' port = 9200 auth = ('integrationtest', 'integrationtest') ca_certs_path = '/stackable/tls/ca.crt' @@ -81,8 +88,6 @@ data: http_auth = auth, use_ssl = True, verify_certs = True, - ssl_assert_hostname = False, - ssl_show_warn = False, ca_certs = ca_certs_path ) diff --git a/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 b/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 index cf5f195..ad50b21 100644 --- a/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/metrics/20-install-opensearch.yaml.j2 @@ -27,7 +27,7 @@ spec: storage: data: capacity: 100Mi - listenerClass: external-stable + listenerClass: external-unstable replicas: 3 configOverrides: opensearch.yml: diff --git a/tests/templates/kuttl/opensearch-dashboards/10-install-opensearch.yaml.j2 b/tests/templates/kuttl/opensearch-dashboards/10-install-opensearch.yaml.j2 index 5d0871a..0d620e1 100644 --- a/tests/templates/kuttl/opensearch-dashboards/10-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/opensearch-dashboards/10-install-opensearch.yaml.j2 @@ -23,7 +23,7 @@ spec: roleGroups: default: config: - listenerClass: external-stable + listenerClass: external-unstable replicas: 1 podOverrides: spec: diff --git a/tests/templates/kuttl/smoke/10-assert.yaml.j2 b/tests/templates/kuttl/smoke/10-assert.yaml.j2 index 60c58ed..43b8aa5 100644 --- a/tests/templates/kuttl/smoke/10-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-assert.yaml.j2 @@ -869,7 +869,7 @@ metadata: kind: OpenSearchCluster name: opensearch spec: - className: external-stable + className: external-unstable extraPodSelectorLabels: {} ports: - name: http diff --git a/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 b/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 index ee70e27..c717b98 100644 --- a/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 +++ b/tests/templates/kuttl/smoke/10-install-opensearch.yaml.j2 @@ -29,7 +29,7 @@ spec: storage: data: capacity: 100Mi - listenerClass: external-stable + listenerClass: external-unstable replicas: 3 podOverrides: spec: diff --git a/tests/templates/kuttl/smoke/20-test-opensearch.yaml b/tests/templates/kuttl/smoke/20-test-opensearch.yaml index b7cfe4c..fcc3fcc 100644 --- a/tests/templates/kuttl/smoke/20-test-opensearch.yaml +++ b/tests/templates/kuttl/smoke/20-test-opensearch.yaml @@ -22,6 +22,10 @@ spec: # required for pip install - name: HOME value: /stackable + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace volumeMounts: - name: script mountPath: /stackable/scripts @@ -70,9 +74,12 @@ data: test.py: | # https://docs.opensearch.org/docs/latest/clients/python-low-level/#sample-program + import os from opensearchpy import OpenSearch - host = 'opensearch' + namespace = os.environ['NAMESPACE'] + + host = f'opensearch.{namespace}.svc.cluster.local' port = 9200 auth = ('admin', 'AJVFsGJBbpT6mChn') # For testing only. Don't store credentials in code. ca_certs_path = '/stackable/tls/ca.crt' @@ -84,8 +91,6 @@ data: http_auth = auth, use_ssl = True, verify_certs = True, - ssl_assert_hostname = False, - ssl_show_warn = False, ca_certs = ca_certs_path ) diff --git a/tests/templates/kuttl/snapshot-s3/00-patch-ns.yaml b/tests/templates/kuttl/snapshot-s3/00-patch-ns.yaml new file mode 100644 index 0000000..d4f91fa --- /dev/null +++ b/tests/templates/kuttl/snapshot-s3/00-patch-ns.yaml @@ -0,0 +1,15 @@ +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + kubectl patch namespace $NAMESPACE --patch=' + { + "metadata": { + "labels": { + "pod-security.kubernetes.io/enforce": "privileged" + } + } + }' + timeout: 120 diff --git a/tests/templates/kuttl/snapshot-s3/01-rbac.yaml b/tests/templates/kuttl/snapshot-s3/01-rbac.yaml new file mode 100644 index 0000000..64eced8 --- /dev/null +++ b/tests/templates/kuttl/snapshot-s3/01-rbac.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-service-account +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: + - apiGroups: + - security.openshift.io + resources: + - securitycontextconstraints + resourceNames: + - privileged + verbs: + - use +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role-binding +subjects: + - kind: ServiceAccount + name: test-service-account +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: test-role diff --git a/tests/templates/kuttl/snapshot-s3/02-assert.yaml.j2 b/tests/templates/kuttl/snapshot-s3/02-assert.yaml.j2 new file mode 100644 index 0000000..50b1d4c --- /dev/null +++ b/tests/templates/kuttl/snapshot-s3/02-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/snapshot-s3/02-install-vector-aggregator-discovery-config-map.yaml.j2 b/tests/templates/kuttl/snapshot-s3/02-install-vector-aggregator-discovery-config-map.yaml.j2 new file mode 100644 index 0000000..2d6a0df --- /dev/null +++ b/tests/templates/kuttl/snapshot-s3/02-install-vector-aggregator-discovery-config-map.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/snapshot-s3/10-assert.yaml b/tests/templates/kuttl/snapshot-s3/10-assert.yaml new file mode 100644 index 0000000..009a4ce --- /dev/null +++ b/tests/templates/kuttl/snapshot-s3/10-assert.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +--- +apiVersion: v1 +kind: Secret +metadata: + name: s3-credentials diff --git a/tests/templates/kuttl/snapshot-s3/10-install-s3-credentials-secret.yaml b/tests/templates/kuttl/snapshot-s3/10-install-s3-credentials-secret.yaml new file mode 100644 index 0000000..e5b552f --- /dev/null +++ b/tests/templates/kuttl/snapshot-s3/10-install-s3-credentials-secret.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: s3-credentials +stringData: + s3.client.default.access_key: opensearchAccessKey + s3.client.default.secret_key: opensearchSecretKey diff --git a/tests/templates/kuttl/snapshot-s3/11-assert.yaml b/tests/templates/kuttl/snapshot-s3/11-assert.yaml new file mode 100644 index 0000000..477bdd0 --- /dev/null +++ b/tests/templates/kuttl/snapshot-s3/11-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: minio +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/snapshot-s3/11-install-minio.yaml.j2 b/tests/templates/kuttl/snapshot-s3/11-install-minio.yaml.j2 new file mode 100644 index 0000000..3b5fe37 --- /dev/null +++ b/tests/templates/kuttl/snapshot-s3/11-install-minio.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install minio oci://registry-1.docker.io/bitnamicharts/minio + --version 17.0.21 + --values 11_minio-values.yaml + --namespace $NAMESPACE + timeout: 240 diff --git a/tests/templates/kuttl/snapshot-s3/11_minio-values.yaml.j2 b/tests/templates/kuttl/snapshot-s3/11_minio-values.yaml.j2 new file mode 100644 index 0000000..6b163ca --- /dev/null +++ b/tests/templates/kuttl/snapshot-s3/11_minio-values.yaml.j2 @@ -0,0 +1,60 @@ +--- +global: + security: + allowInsecureImages: true + +image: + repository: bitnamilegacy/minio + +defaultInitContainers: + volumePermissions: + image: + repository: bitnamilegacy/os-shell + +console: + image: + repository: bitnamilegacy/minio-object-browser + +persistence: + enabled: false # "false" means, that an emptyDir is used instead of a persistentVolumeClaim + size: 64Mi + +provisioning: + enabled: true + buckets: + - name: opensearch + users: + - username: integrationtest + password: integrationtest + policies: + - readwrite + - consoleAdmin + - diagnostics + extraCommands: + - >- + mc admin user svcacct add + --access-key "$(cat /tmp/s3-credentials/s3.client.default.access_key)" + --secret-key "$(cat /tmp/s3-credentials/s3.client.default.secret_key)" + provisioning integrationtest + extraVolumeMounts: + - name: s3-credentials + mountPath: /tmp/s3-credentials + extraVolumes: + - name: s3-credentials + secret: + secretName: s3-credentials + resources: + requests: + memory: 1Gi + cpu: 512m + limits: + memory: 1Gi + cpu: "1" + +resources: + requests: + memory: 1Gi + cpu: 512m + limits: + memory: 1Gi + cpu: "1" diff --git a/tests/templates/kuttl/snapshot-s3/20-assert.yaml.j2 b/tests/templates/kuttl/snapshot-s3/20-assert.yaml.j2 new file mode 100644 index 0000000..06c64a1 --- /dev/null +++ b/tests/templates/kuttl/snapshot-s3/20-assert.yaml.j2 @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: opensearch-nodes-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/snapshot-s3/20-install-opensearch.yaml.j2 b/tests/templates/kuttl/snapshot-s3/20-install-opensearch.yaml.j2 new file mode 100644 index 0000000..19c88bc --- /dev/null +++ b/tests/templates/kuttl/snapshot-s3/20-install-opensearch.yaml.j2 @@ -0,0 +1,211 @@ +--- +apiVersion: opensearch.stackable.tech/v1alpha1 +kind: OpenSearchCluster +metadata: + name: opensearch +spec: + image: +{% if test_scenario['values']['opensearch'].find(",") > 0 %} + custom: "{{ test_scenario['values']['opensearch'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['opensearch'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['opensearch'] }}" +{% endif %} + pullPolicy: IfNotPresent +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nodes: + config: + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-unstable + roleGroups: + default: + replicas: 1 + configOverrides: + opensearch.yml: + # Disable memory mapping in this test; If memory mapping were activated, the kernel setting + # vm.max_map_count would have to be increased to 262144 on the node. + node.store.allow_mmap: "false" + # Disable the disk allocation decider in this test; Otherwise the test depends on the disk + # usage of the node and if the relative watermark set in + # `cluster.routing.allocation.disk.watermark.high` is reached then the security index could + # not be created even if enough disk space would be available. + cluster.routing.allocation.disk.threshold_enabled: "false" + plugins.security.allow_default_init_securityindex: "true" + plugins.security.ssl.transport.enabled: "true" + plugins.security.ssl.transport.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.transport.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.transport.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt + plugins.security.ssl.http.enabled: "true" + plugins.security.ssl.http.pemcert_filepath: /stackable/opensearch/config/tls/tls.crt + plugins.security.ssl.http.pemkey_filepath: /stackable/opensearch/config/tls/tls.key + plugins.security.ssl.http.pemtrustedcas_filepath: /stackable/opensearch/config/tls/ca.crt + s3.client.default.endpoint: http://minio:9000/ + s3.client.default.protocol: http + s3.client.default.region: unused # but required + s3.client.default.path_style_access: "true" + podOverrides: + spec: + initContainers: + - name: init-keystore + image: oci.stackable.tech/sdp/opensearch:3.1.0-stackable0.0.0-dev + command: + - /bin/bash + - -euxo + - pipefail + - -c + args: + - | + bin/opensearch-keystore create + + for i in keystore-secrets/*; do + key=$(basename $i) + bin/opensearch-keystore add-file "$key" "$i" + done + + cp --archive config/opensearch.keystore initialized-keystore + volumeMounts: + - name: keystore + mountPath: /stackable/opensearch/initialized-keystore + readOnly: false + - name: keystore-secrets + mountPath: /stackable/opensearch/keystore-secrets + readOnly: true + containers: + - name: opensearch + volumeMounts: + - name: security-config + mountPath: /stackable/opensearch/config/opensearch-security + readOnly: true + - name: tls + mountPath: /stackable/opensearch/config/tls + readOnly: true + - name: keystore + mountPath: /stackable/opensearch/config/opensearch.keystore + subPath: opensearch.keystore + readOnly: true + volumes: + - name: keystore + emptyDir: + sizeLimit: 1Mi + - name: keystore-secrets + secret: + secretName: s3-credentials + defaultMode: 0o660 + - name: security-config + secret: + secretName: opensearch-security-config + defaultMode: 0o660 + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/scope: node,pod,service=opensearch,service=opensearch-nodes-default-headless + secrets.stackable.tech/class: tls + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" +--- +apiVersion: v1 +kind: Secret +metadata: + name: opensearch-security-config +stringData: + action_groups.yml: | + --- + _meta: + type: actiongroups + config_version: 2 + allowlist.yml: | + --- + _meta: + type: allowlist + config_version: 2 + + config: + enabled: false + audit.yml: | + --- + _meta: + type: audit + config_version: 2 + + config: + enabled: false + config.yml: | + --- + _meta: + type: config + config_version: 2 + + config: + dynamic: + authc: + basic_internal_auth_domain: + description: Authenticate via HTTP Basic against internal users database + http_enabled: true + transport_enabled: true + order: 1 + http_authenticator: + type: basic + challenge: true + authentication_backend: + type: intern + authz: {} + internal_users.yml: | + --- + # The hash value is a bcrypt hash and can be generated with plugin/tools/hash.sh + + _meta: + type: internalusers + config_version: 2 + + admin: + hash: $2y$10$xRtHZFJ9QhG9GcYhRpAGpufCZYsk//nxsuel5URh0GWEBgmiI4Q/e + reserved: true + backend_roles: + - admin + description: OpenSearch admin user + + kibanaserver: + hash: $2y$10$vPgQ/6ilKDM5utawBqxoR.7euhVQ0qeGl8mPTeKhmFT475WUDrfQS + reserved: true + description: OpenSearch Dashboards user + nodes_dn.yml: | + --- + _meta: + type: nodesdn + config_version: 2 + roles.yml: | + --- + _meta: + type: roles + config_version: 2 + roles_mapping.yml: | + --- + _meta: + type: rolesmapping + config_version: 2 + + all_access: + reserved: false + backend_roles: + - admin + + kibana_server: + reserved: true + users: + - kibanaserver + tenants.yml: | + --- + _meta: + type: tenants + config_version: 2 diff --git a/tests/templates/kuttl/snapshot-s3/30-assert.yaml b/tests/templates/kuttl/snapshot-s3/30-assert.yaml new file mode 100644 index 0000000..9e330a0 --- /dev/null +++ b/tests/templates/kuttl/snapshot-s3/30-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-snapshot +status: + succeeded: 1 diff --git a/tests/templates/kuttl/snapshot-s3/30-test-snapshot.yaml b/tests/templates/kuttl/snapshot-s3/30-test-snapshot.yaml new file mode 100644 index 0000000..ad95ecd --- /dev/null +++ b/tests/templates/kuttl/snapshot-s3/30-test-snapshot.yaml @@ -0,0 +1,154 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: test-snapshot +spec: + template: + spec: + containers: + - name: test-snapshot + image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev + command: + - /bin/bash + - -euxo + - pipefail + - -c + args: + - | + pip install opensearch-py==3.0.0 + python scripts/test.py + env: + # required for pip install + - name: HOME + value: /stackable + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: script + mountPath: /stackable/scripts + - name: tls + mountPath: /stackable/tls + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + resources: + requests: + memory: 128Mi + cpu: 100m + limits: + memory: 128Mi + cpu: 400m + volumes: + - name: script + configMap: + name: test-opensearch + - name: tls + ephemeral: + volumeClaimTemplate: + metadata: + annotations: + secrets.stackable.tech/class: tls + spec: + storageClassName: secrets.stackable.tech + accessModes: + - ReadWriteOnce + resources: + requests: + storage: "1" + serviceAccountName: test-service-account + securityContext: + fsGroup: 1000 + restartPolicy: OnFailure +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: test-opensearch +data: + test.py: | + import os + from opensearchpy import OpenSearch + from opensearchpy.exceptions import RequestError + + namespace = os.environ['NAMESPACE'] + + host = f'opensearch.{namespace}.svc.cluster.local' + port = 9200 + auth = ('admin', 'AJVFsGJBbpT6mChn') # For testing only. Don't store credentials in code. + ca_certs_path = '/stackable/tls/ca.crt' + + client = OpenSearch( + hosts = [{'host': host, 'port': port}], + http_compress = True, + http_auth = auth, + use_ssl = True, + verify_certs = True, + ca_certs = ca_certs_path + ) + + # Create an index + index_name = 'test-index' + + try: + response = client.indices.create(index=index_name) + print(f'Creating index; {response=}') + except RequestError as err: + if err.error == 'resource_already_exists_exception': + print(f'The index was already created in a prior test run; {index_name=}') + else: + raise + + # Add a document to the index + response = client.index( + index = index_name, + body = { + 'name': 'Stackable' + }, + id = 1, + ) + + print(f'Adding document; {response=}') + + # Create snapshot repository + snapshot_repository = "snapshot_repository" + + response = client.snapshot.create_repository( + repository=snapshot_repository, + body={ + "type": "s3", + "settings": { + "bucket": "opensearch" + } + } + ) + + print(f'Creating snapshot repository; {response=}') + + # Create snapshot + snapshot = "test_snapshot" + + try: + response = client.snapshot.create( + repository=snapshot_repository, + snapshot=snapshot, + body={ + "indices": index_name + } + ) + print(f'Creating snapshot; {response=}') + except RequestError as err: + if err.error == 'invalid_snapshot_name_exception': + print(f'The snapshot was already created in a prior test run; {snapshot=}') + else: + raise + + # Delete the index. + response = client.indices.delete(index=index_name) + + print(f'Deleting index; {response=}') diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index de7d2b3..40792a1 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -34,6 +34,10 @@ tests: dimensions: - opensearch - opensearch_home + # requires the repository-s3 plugin + - name: snapshot-s3 + dimensions: + - opensearch suites: - name: nightly patch: