diff --git a/Makefile b/Makefile index ac1a460fb6..9057efeceb 100644 --- a/Makefile +++ b/Makefile @@ -199,6 +199,11 @@ test-cli-flow: @echo "✅ test-cli-flow" .PHONY: test-cli-flow +test-cli-flow-dev: + $(TIME) ./test/cli/test_cli_flow.sh --dev + @echo "✅ test-cli-flow-dev" +.PHONY: test-cli-flow-dev + test-core-config-map-flow: $(TIME) ./test/cli/test_cli_flow.sh --check_core_config_map @echo "✅ test-core-config-map-flow" diff --git a/pkg/backingstore/backingstore.go b/pkg/backingstore/backingstore.go index 31524c8597..88018d27e7 100644 --- a/pkg/backingstore/backingstore.go +++ b/pkg/backingstore/backingstore.go @@ -242,6 +242,10 @@ const ( // Test ENV minimal resources testEnvMinCPUString string = "50m" testEnvMinMemoryString string = "200Mi" + + // Dev ENV minimal resources + devEnvMinCPUString string = "500m" + devEnvMinMemoryString string = "500Mi" ) // CmdCreatePVPool returns a CLI command diff --git a/pkg/backingstore/reconciler.go b/pkg/backingstore/reconciler.go index dd170eeab2..0a9f8b0e90 100644 --- a/pkg/backingstore/reconciler.go +++ b/pkg/backingstore/reconciler.go @@ -1397,6 +1397,7 @@ func (r *Reconciler) upgradeBackingStore(sts *appsv1.StatefulSet) error { } func (r *Reconciler) reconcileResources(src, dst *corev1.ResourceList, minCPU, minMem resource.Quantity) error { + log := r.Logger cpu := minCPU mem := minMem @@ -1416,6 +1417,7 @@ func (r *Reconciler) reconcileResources(src, dst *corev1.ResourceList, minCPU, m mem = qty } } + log.Infof("BackingStore %q was created with resurce cpu:%v mem:%v.", r.BackingStore.Name, cpu, mem) (*dst)[corev1.ResourceCPU] = cpu (*dst)[corev1.ResourceMemory] = mem @@ -1431,5 +1433,9 @@ func getMinimalResourcesByEnv() (string, string) { minCPUStringByEnv = testEnvMinCPUString minMemoryStringByEnv = testEnvMinMemoryString } + if util.IsDevEnv() { + minCPUStringByEnv = devEnvMinCPUString + minMemoryStringByEnv = devEnvMinMemoryString + } return minCPUStringByEnv, minMemoryStringByEnv } diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index 5542d73eac..e1bb2eef80 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -122,6 +122,15 @@ func RunInstall(cmd *cobra.Command, args []string) { }) c.Deployment.Spec.Template.Spec.Containers[0].Env = operatorContainer.Env } + devEnv, _ := cmd.Flags().GetBool("dev") + if devEnv { + operatorContainer := c.Deployment.Spec.Template.Spec.Containers[0] + operatorContainer.Env = append(operatorContainer.Env, corev1.EnvVar{ + Name: "DEV_ENV", + Value: "true", + }) + c.Deployment.Spec.Template.Spec.Containers[0].Env = operatorContainer.Env + } admission, _ := cmd.Flags().GetBool("admission") if admission { diff --git a/pkg/util/util.go b/pkg/util/util.go index 26b52fb0ac..ecf0e9a82e 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -434,7 +434,7 @@ func KubeDelete(obj client.Object, opts ...client.DeleteOption) bool { } time.Sleep(10 * time.Millisecond) - + err = wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { err := klient.Delete(ctx, obj, opts...) if err == nil { @@ -2157,3 +2157,13 @@ func IsTestEnv() bool { } return false } + +// IsDevEnv checks for DEV_ENV env var existance and equality +// to true and returns true or false accordingly +func IsDevEnv() bool { + devEnv, ok := os.LookupEnv("DEV_ENV") + if ok && devEnv == "true" { + return true + } + return false +} diff --git a/test/cli/resources/external-db-ssl.yaml b/test/cli/resources/external-db-ssl.yaml new file mode 100644 index 0000000000..f79771f5d9 --- /dev/null +++ b/test/cli/resources/external-db-ssl.yaml @@ -0,0 +1,107 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres-external + namespace: test +spec: + type: ClusterIP + selector: + app: external-db + ports: + - port: 5432 +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ssl-postgres-conf-sh + namespace: test + labels: + app: noobaa +data: + sslconf.sh: | + #!/bin/bash + # echo ssl setting into pg_hba.conf configuration file + echo 'local all all trust' > /var/lib/postgresql/data/pg_hba.conf + echo 'hostssl all all all cert clientcert=verify-full' >> /var/lib/postgresql/data/pg_hba.conf +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + app: external-db + name: postgres-external + namespace: test +spec: + initContainers: + - name: volume-mount-hack + image: postgres:15 + command: ["sh", "-c", "cp /etc/ssl/server.key /etc/private/server.key && chown root:ssl-cert /etc/private/server.key"] + volumeMounts: + - name: postgres-ssl + mountPath: /etc/ssl + - name: key-volume + mountPath: /etc/private + containers: + - name: external-db-ssl + image: postgres:15 + args: + - -c + - ssl=on + - -c + - ssl_cert_file=/etc/ssl/server.crt + - -c + - ssl_key_file=/etc/private/server.key + - -c + - ssl_ca_file=/etc/ssl/ca.crt + imagePullPolicy: IfNotPresent + resources: + # https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/ + # requests: + # cpu: 100m + # memory: 500Mi + # limits: + # cpu: 100m + # memory: 500Mi + requests: + cpu: 1000m + memory: 2Gi + limits: + cpu: 1000m + memory: 2Gi + env: + - name: POSTGRES_PASSWORD + value: noobaa + - name: LC_COLLATE + value: C + ports: + - containerPort: 5432 + volumeMounts: + - name: postgres-ssl + mountPath: /etc/ssl + - name: key-volume + mountPath: /etc/private + - name: ssl-postgres-conf-sh-volume + mountPath: /docker-entrypoint-initdb.d + volumes: + - name: key-volume + emptyDir: {} + - name: ssl-postgres-conf-sh-volume + configMap: + name: ssl-postgres-conf-sh + items: + - key: sslconf.sh + path: sslconf.sh + - name: postgres-ssl + secret: + secretName: postgres-ssl + defaultMode: 0600 + items: + - key: server.key + path: server.key + mode: 0640 + - key: server.crt + path: server.crt + mode: 0777 + - key: ca.crt + path: ca.crt + mode: 0777 diff --git a/test/cli/resources/external-db.yaml b/test/cli/resources/external-db.yaml new file mode 100644 index 0000000000..ab8a32a43c --- /dev/null +++ b/test/cli/resources/external-db.yaml @@ -0,0 +1,39 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres-external + namespace: test +spec: + type: ClusterIP + selector: + app: external-db + ports: + - port: 5432 +--- +apiVersion: v1 +kind: Pod +metadata: + labels: + app: external-db + name: postgres-external + namespace: test +spec: + containers: + - name: external-db + image: postgres:15 + imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 500Mi + limits: + cpu: 100m + memory: 500Mi + env: + - name: POSTGRES_PASSWORD + value: noobaa + - name: LC_COLLATE + value: C + ports: + - containerPort: 5432 + diff --git a/test/cli/test_cli_flow.sh b/test/cli/test_cli_flow.sh index d076acbbfb..5d579acff2 100755 --- a/test/cli/test_cli_flow.sh +++ b/test/cli/test_cli_flow.sh @@ -7,6 +7,7 @@ export PS4='\e[36m+ ${FUNCNAME:-main}\e[0m@\e[32m${BASH_SOURCE}:\e[35m${LINENO} NAMESPACE='test' CM=false +RESOURCE='mini' function post_install_tests { aws_credentials @@ -31,12 +32,18 @@ function post_install_tests { } function main { - local install_external=$((RANDOM%2)) + local install_external=$((RANDOM%3)) + install_external=2 if [ ${install_external} -eq 0 ] then noobaa_install_external else - noobaa_install + if [ ${install_external} -eq 1 ] + then + noobaa_install_external_ssl + else + noobaa_install + fi fi if [ "${CM}" == "true" ] then @@ -47,7 +54,12 @@ function main { if [ ${install_external} -eq 0 ] then delete_external_postgres - fi + else + if [ ${install_external} -eq 1 ] + then + delete_external_postgres_ssl + fi + fi } function usage { @@ -95,7 +107,11 @@ do -n|--namespace) NAMESPACE=${2} shift 2;; --check_core_config_map) CM=true - shift;; + shift;; + --dev) RESOURCE='dev' + shift;; + --mini) RESOURCE='mini' + shift;; -h|--help) usage;; *) usage;; esac diff --git a/test/cli/test_cli_functions.sh b/test/cli/test_cli_functions.sh index f67c36a5f1..713b6835bf 100644 --- a/test/cli/test_cli_functions.sh +++ b/test/cli/test_cli_functions.sh @@ -176,31 +176,52 @@ function install { local use_obc_cleanup_policy [ $((RANDOM%2)) -gt 0 ] && use_obc_cleanup_policy="--use-obc-cleanup-policy" - test_noobaa install --mini --admission ${use_obc_cleanup_policy} + test_noobaa install --${RESOURCE} --admission ${use_obc_cleanup_policy} - local status=$(kuberun silence get noobaa noobaa -o 'jsonpath={.status.phase}') - while [ "${status}" != "Ready" ] - do - echo_time "💬 Waiting for status Ready, Status is ${status}" - sleep 10 - status=$(kuberun silence get noobaa noobaa -o 'jsonpath={.status.phase}') - done + wait_for_noobaa_ready + wait_for_backingstore_ready noobaa-default-backing-store } function run_external_postgres { - kubectl run postgres-external --image=postgres:15 --env POSTGRES_PASSWORD=password --port 5432 --expose + # kubectl run postgres-external --image=postgres:15 --env POSTGRES_PASSWORD=password --port 5432 --expose + echo_time "Creating an external postgres DB for test (NO SSL)" + kuberun create -f $(dirname ${0})/resources/external-db.yaml +} + +function run_external_postgres_ssl { + echo_time "Creating an external postgres DB for test (SSL)" + kuberun create secret generic postgres-ssl --from-file=certs/server.crt --from-file=certs/server.key --from-file=certs/ca.crt + kuberun create -f $(dirname ${0})/resources/external-db-ssl.yaml } function delete_external_postgres { - kubectl delete pod postgres-external - kubectl delete service postgres-external + kuberun delete -f $(dirname ${0})/resources/external-db.yaml +} + +function delete_external_postgres_ssl { + kuberun delete -f $(dirname ${0})/resources/external-db-ssl.yaml + kuberun delete secret postgres-ssl } function install_external { local postgres_url="postgresql://postgres:password@postgres-external.${NAMESPACE}.svc:5432/postgres" echo_time "Installing NooBaa in external postgres mode postgres-url=${postgres_url}" - test_noobaa install --mini --postgres-url=${postgres_url}" + test_noobaa install --${RESOURCE} --postgres-url=${postgres_url} + wait_for_noobaa_ready + wait_for_backingstore_ready noobaa-default-backing-store +} + +function install_external_ssl { + local postgres_url="postgresql://postgres:password@postgres-external.${NAMESPACE}.svc:5432/postgres" + echo_time "Installing NooBaa in external postgres mode postgres-url=${postgres_url} with SSL" + test_noobaa install --${RESOURCE} --postgres-url=${postgres_url} --pg-ssl-required --pg-ssl-unauthorized --pg-ssl-key certs/client.key --pg-ssl-cert certs/client.crt + + wait_for_noobaa_ready + wait_for_backingstore_ready noobaa-default-backing-store +} + +function wait_for_noobaa_ready { local status=$(kuberun silence get noobaa noobaa -o 'jsonpath={.status.phase}') while [ "${status}" != "Ready" ] do @@ -210,8 +231,26 @@ function install_external { done } +function wait_for_backingstore_ready { + local status=$(kuberun silence get backingstore noobaa-default-backing-store -o 'jsonpath={.status.phase}') + local status=$(kuberun silence get backingstore ${1} -o 'jsonpath={.status.phase}') + while [ "${status}" != "Ready" ] + do + echo_time "💬 Waiting for status Ready, Status is ${status}" + sleep 10 + status=$(kuberun silence get noobaa noobaa -o 'jsonpath={.status.phase}') + done +} + +function clean_leftovers { + test_noobaa --timeout uninstall + kuberun delete deploy,sts,service,job,po,pv,pvc,cm,secret --all + ${kubectl} delete sc nsfs-local +} + function noobaa_install { #noobaa timeout install # Maybe when creating server we can use local PV + clean_leftovers install test_noobaa status kuberun get noobaa @@ -221,6 +260,7 @@ function noobaa_install { function noobaa_install_external { #noobaa timeout install # Maybe when creating server we can use local PV + clean_leftovers run_external_postgres install_external test_noobaa status @@ -228,6 +268,25 @@ function noobaa_install_external { kuberun describe noobaa } +function noobaa_install_external_ssl { + #noobaa timeout install # Maybe when creating server we can use local PV + mkdir -p -m 755 certs + openssl ecparam -name prime256v1 -genkey -noout -out certs/ca.key + openssl req -new -x509 -sha256 -key certs/ca.key -out certs/ca.crt -subj "/CN=ca.noobaa.com" + openssl genrsa -out certs/server.key 2048 + openssl req -new -sha256 -key certs/server.key -out certs/server.csr -subj "/CN=postgres-external.${NAMESPACE}.svc" + openssl x509 -req -in certs/server.csr -CA certs/ca.crt -CAkey certs/ca.key -CAcreateserial -out certs/server.crt -days 365 -sha256 + openssl ecparam -name prime256v1 -genkey -noout -out certs/client.key + openssl req -new -sha256 -key certs/client.key -out certs/client.csr -subj "/CN=postgres" + openssl x509 -req -in certs/client.csr -CA certs/ca.crt -CAkey certs/ca.key -CAcreateserial -out certs/client.crt -days 365 -sha256 + clean_leftovers + run_external_postgres_ssl + install_external_ssl + test_noobaa status + kuberun get noobaa + kuberun describe noobaa +} + function test_admission_deployment { kuberun get Secret "admission-webhook-secret" kuberun get ValidatingWebhookConfiguration "admission-validation-webhook" @@ -429,13 +488,20 @@ function check_pv_pool_resources { --request-cpu 300m \ --limit-cpu 200m + local mem=400 + local cpu=100 + if [ "$RESOURCE" == "dev" ] + then + mem=500 + cpu=500 + fi test_noobaa backingstore create pv-pool minimum-request-limit \ --num-volumes 1 \ --pv-size-gb 16 \ - --request-cpu 100m \ - --request-memory 400Mi \ - --limit-cpu 100m \ - --limit-memory 400Mi + --request-cpu $(cpu)m \ + --request-memory $(mem)Mi \ + --limit-cpu $(cpu)m \ + --limit-memory $(mem)Mi #TOD see why it fails, currently disabling as it takes 10 mins. # time="2022-04-11T14:18:17Z" level=error msg="❌ BackingStore \"large-request-limit\" Phase is \"Rejected\": Failed connecting all pods in backingstore for more than 10 minutes Current failing: 1 from requested: 1" # NAME TYPE TARGET-BUCKET PHASE AGE @@ -476,7 +542,7 @@ function check_S3_compatible { --target-bucket ${buckets[cycle]} \ --endpoint s3.${NAMESPACE}.svc.cluster.local:443 \ --secret-name ${SECRET_NAME} - test_noobaa backingstore status ${backingstore[cycle]} + wait_for_backingstore_ready ${backingstore[cycle]} done test_noobaa backingstore list test_noobaa status