From 78d9609db30ba30d69d4ff5f376fb4e7ee0f7ad0 Mon Sep 17 00:00:00 2001 From: harayuanwang Date: Tue, 28 Jun 2022 16:32:32 +0000 Subject: [PATCH 01/37] Updated README.md --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d8d63691..b5ec6729 100644 --- a/README.md +++ b/README.md @@ -108,14 +108,16 @@ YAML file templates are available under [`/config/samples`](./config/samples/). To delete all the CRD instances deployed to cluster by the operator, run the following commands, where is the namespace of the cluster object: ```sh - kubectl delete oraclerestdataservices.database.oracle.com --all -n + kubectl delete oraclerestdataservice.database.oracle.com --all -n kubectl delete singleinstancedatabase.database.oracle.com --all -n kubectl delete shardingdatabase.database.oracle.com --all -n + kubectl delete dbcssystem.database.oracle.com --all -n kubectl delete autonomousdatabase.database.oracle.com --all -n kubectl delete autonomousdatabasebackup.database.oracle.com --all -n kubectl delete autonomousdatabaserestore.database.oracle.com --all -n kubectl delete autonomouscontainerdatabase.database.oracle.com --all -n - kubectl delete dbcssystem.database.oracle.com --all -n + kubectl delete cdb.database.oracle.com --all -n + kubectl delete pdb.database.oracle.com --all -n ``` After all CRD instances are deleted, it is safe to remove the CRDs, APISerivces and operator deployment. From 1f20f432a94597b3b0a7b59f00ba302001ca40c6 Mon Sep 17 00:00:00 2001 From: Mohammed Yunus Date: Wed, 29 Jun 2022 08:42:36 +0000 Subject: [PATCH 02/37] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..1c5aaf93 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,28 @@ +build-operator: + stage: build + variables: + IMAGE: "$DOCKER_REPO:$CI_COMMIT_BRANCH" + OP_YAML: oracle-database-operator.yaml + script: + - make docker-build IMG="$IMAGE" + - docker push "$IMAGE" + - newimage=$(docker inspect $IMAGE | python -c 'import json,sys; print json.load(sys.stdin)[0]["RepoDigests"][0]') + - echo $newimage + - docker rmi "$IMAGE" && docker system prune -f + - make operator-yaml IMG=$newimage + - if [ "$CI_COMMIT_BRANCH" != "master" ]; then sed -i "s/\(replicas.\) 3/\1 1/g" ./$OP_YAML; fi + - curl -s -n $ARTIFACTORY_REPO/$CI_COMMIT_BRANCH/$OP_YAML -T ./$OP_YAML + only: + variables: + - $CI_COMMIT_MESSAGE =~ /\#run-pipeline/ + - $CI_COMMIT_BRANCH =~ /master/ + - $CI_MERGE_REQUEST_ID != "" + except: + variables: + - $CI_COMMIT_MESSAGE =~ /\#skip-pipeline/ + +cleanup: + stage: .post + script: + - echo "Clean up downloaded binaries" + - rm -rf bin/ From 50589c5dba5d25d271aed91bcb96ba20119066ba Mon Sep 17 00:00:00 2001 From: harayuanwang Date: Fri, 1 Jul 2022 18:03:09 +0000 Subject: [PATCH 03/37] Update .gitlab-ci.yml file --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c5aaf93..9c978f45 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,6 +20,7 @@ build-operator: except: variables: - $CI_COMMIT_MESSAGE =~ /\#skip-pipeline/ + - $CI_COMMIT_TAG != "" cleanup: stage: .post From 22f1f1d3251159a633621ccc85cbd0bc3b730da7 Mon Sep 17 00:00:00 2001 From: harayuanwang Date: Fri, 1 Jul 2022 18:07:52 +0000 Subject: [PATCH 04/37] Revert "Update .gitlab-ci.yml file" This reverts commit 50589c5dba5d25d271aed91bcb96ba20119066ba --- .gitlab-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9c978f45..1c5aaf93 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -20,7 +20,6 @@ build-operator: except: variables: - $CI_COMMIT_MESSAGE =~ /\#skip-pipeline/ - - $CI_COMMIT_TAG != "" cleanup: stage: .post From ee9bb97461ed7c53adbd10b286cee7a97009950a Mon Sep 17 00:00:00 2001 From: harayuanwang Date: Fri, 1 Jul 2022 18:31:14 +0000 Subject: [PATCH 05/37] Tinglwan create tag skip ci --- .gitlab-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1c5aaf93..cbef6241 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,6 +4,7 @@ build-operator: IMAGE: "$DOCKER_REPO:$CI_COMMIT_BRANCH" OP_YAML: oracle-database-operator.yaml script: + - echo $CI_COMMIT_TAG - make docker-build IMG="$IMAGE" - docker push "$IMAGE" - newimage=$(docker inspect $IMAGE | python -c 'import json,sys; print json.load(sys.stdin)[0]["RepoDigests"][0]') @@ -20,6 +21,7 @@ build-operator: except: variables: - $CI_COMMIT_MESSAGE =~ /\#skip-pipeline/ + - $CI_COMMIT_TAG != null cleanup: stage: .post From b67e4347bf3e9baa16841972038fb91e8802857e Mon Sep 17 00:00:00 2001 From: abhisbyk Date: Mon, 11 Jul 2022 14:27:25 +0530 Subject: [PATCH 06/37] Added env vars for tcps support Signed-off-by: abhisbyk --- .../v1alpha1/singleinstancedatabase_types.go | 2 + ...se.oracle.com_singleinstancedatabases.yaml | 4 ++ .../samples/sidb/singleinstancedatabase.yaml | 6 +++ .../singleinstancedatabase_controller.go | 46 +++++++++++++++++-- 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/apis/database/v1alpha1/singleinstancedatabase_types.go b/apis/database/v1alpha1/singleinstancedatabase_types.go index 11ee82db..12b8ee9d 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_types.go +++ b/apis/database/v1alpha1/singleinstancedatabase_types.go @@ -65,6 +65,8 @@ type SingleInstanceDatabaseSpec struct { FlashBack bool `json:"flashBack,omitempty"` ArchiveLog bool `json:"archiveLog,omitempty"` ForceLogging bool `json:"forceLog,omitempty"` + EnableTCPS bool `json:"enableTCPS,omitempty"` + TcpsPort int `json:"tcpsPort,omitempty"` CloneFrom string `json:"cloneFrom,omitempty"` ReadinessCheckPeriod int `json:"readinessCheckPeriod,omitempty"` diff --git a/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml b/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml index a4124691..6e7c3ec4 100644 --- a/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml +++ b/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml @@ -87,6 +87,8 @@ spec: - enterprise - express type: string + enableTCPS: + type: boolean flashBack: type: boolean forceLog: @@ -160,6 +162,8 @@ spec: maxLength: 12 pattern: ^[a-zA-Z0-9]+$ type: string + tcpsPort: + type: integer required: - image type: object diff --git a/config/samples/sidb/singleinstancedatabase.yaml b/config/samples/sidb/singleinstancedatabase.yaml index f0502697..0d099255 100644 --- a/config/samples/sidb/singleinstancedatabase.yaml +++ b/config/samples/sidb/singleinstancedatabase.yaml @@ -44,6 +44,12 @@ spec: ## Enable/Disable ForceLogging forceLog: false + ## Enable TCPS + enableTCPS: false + + ## TCPS custom port + tcpsPort: 1522 + ## NA if cloning from a SourceDB (cloneFrom is set) ## Specify both sgaSize and pgaSize (in MB) or dont specify both ## Specify Non-Zero value to use diff --git a/controllers/database/singleinstancedatabase_controller.go b/controllers/database/singleinstancedatabase_controller.go index 714c07c0..82351e2d 100644 --- a/controllers/database/singleinstancedatabase_controller.go +++ b/controllers/database/singleinstancedatabase_controller.go @@ -78,6 +78,9 @@ type SingleInstanceDatabaseReconciler struct { var requeueY ctrl.Result = ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second} var requeueN ctrl.Result = ctrl.Result{} +// Service Port Declaration +var svc_port string + const singleInstanceDatabaseFinalizer = "database.oracle.com/singleinstancedatabasefinalizer" //+kubebuilder:rbac:groups=database.oracle.com,resources=singleinstancedatabases,verbs=get;list;watch;create;update;patch;delete @@ -134,6 +137,14 @@ func (r *SingleInstanceDatabaseReconciler) Reconcile(ctx context.Context, req ct r.Status().Update(ctx, singleInstanceDatabase) } + // Service Port Initialization + svc_port = func() string { + if singleInstanceDatabase.Spec.EnableTCPS { + return strconv.Itoa(singleInstanceDatabase.Spec.TcpsPort) + } + return "1521" + }() + // Manage SingleInstanceDatabase Deletion result, err = r.manageSingleInstanceDatabaseDeletion(req, ctx, singleInstanceDatabase) if result.Requeue { @@ -639,7 +650,16 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns }, }, }, - Ports: []corev1.ContainerPort{{ContainerPort: 1521}, {ContainerPort: 5500}}, + Ports: []corev1.ContainerPort{ + {ContainerPort: func() int32 { + if m.Spec.EnableTCPS { + return int32(m.Spec.TcpsPort) + } + return int32(1521) + }(), + }, + {ContainerPort: 5500}, + }, ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ @@ -687,7 +707,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns }, { Name: "SVC_PORT", - Value: "1521", + Value: svc_port, }, { Name: "ORACLE_CHARACTERSET", @@ -697,6 +717,14 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns Name: "ORACLE_EDITION", Value: m.Spec.Edition, }, + { + Name: "ENABLE_TCPS", + Value: strconv.FormatBool(m.Spec.EnableTCPS), + }, + { + Name: "TCPS_PORT", + Value: strconv.Itoa(m.Spec.TcpsPort), + }, } } if m.Spec.CloneFrom == "" { @@ -708,7 +736,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns }, { Name: "SVC_PORT", - Value: "1521", + Value: svc_port, }, { Name: "CREATE_PDB", @@ -766,6 +794,14 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns Name: "SKIP_DATAPATCH", Value: "true", }, + { + Name: "ENABLE_TCPS", + Value: strconv.FormatBool(m.Spec.EnableTCPS), + }, + { + Name: "TCPS_PORT", + Value: strconv.Itoa(m.Spec.TcpsPort), + }, } } // For clone DB use case @@ -776,7 +812,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns }, { Name: "SVC_PORT", - Value: "1521", + Value: svc_port, }, { Name: "ORACLE_SID", @@ -790,7 +826,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns Name: "PRIMARY_DB_CONN_STR", Value: func() string { if dbcommons.IsSourceDatabaseOnCluster(m.Spec.CloneFrom) { - return n.Name + ":1521/" + n.Spec.Sid + return n.Name + ":" + svc_port + "/" + n.Spec.Sid } return m.Spec.CloneFrom }(), From 254487312053df46a56e7c71b8cf58e554d55773 Mon Sep 17 00:00:00 2001 From: abhisbyk Date: Mon, 11 Jul 2022 15:05:00 +0530 Subject: [PATCH 07/37] Correcting port for the db service Signed-off-by: abhisbyk --- .../database/singleinstancedatabase_controller.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/controllers/database/singleinstancedatabase_controller.go b/controllers/database/singleinstancedatabase_controller.go index 82351e2d..c852e9ae 100644 --- a/controllers/database/singleinstancedatabase_controller.go +++ b/controllers/database/singleinstancedatabase_controller.go @@ -914,8 +914,13 @@ func (r *SingleInstanceDatabaseReconciler) instantiateSVCSpec(m *dbapi.SingleIns Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ { - Name: "listener", - Port: 1521, + Name: "listener", + Port: func() int32 { + if m.Spec.EnableTCPS { + return int32(m.Spec.TcpsPort) + } + return int32(1521) + }(), Protocol: corev1.ProtocolTCP, }, { From 5085f826baf45b31da84b35551f1122f4c140570 Mon Sep 17 00:00:00 2001 From: abhisbyk Date: Fri, 29 Jul 2022 10:50:33 +0530 Subject: [PATCH 08/37] Intermediate commit-1 Signed-off-by: abhisbyk --- apis/database/v1alpha1/singleinstancedatabase_types.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apis/database/v1alpha1/singleinstancedatabase_types.go b/apis/database/v1alpha1/singleinstancedatabase_types.go index 12b8ee9d..ab20ae2a 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_types.go +++ b/apis/database/v1alpha1/singleinstancedatabase_types.go @@ -148,6 +148,8 @@ type SingleInstanceDatabaseStatus struct { PdbConnectString string `json:"pdbConnectString,omitempty"` ApexInstalled bool `json:"apexInstalled,omitempty"` PrebuiltDB bool `json:"prebuiltDB,omitempty"` + // +kubebuilder:default:=false + IsTcpsEnabled bool `json:"isTcpsEnabled"` // +patchMergeKey=type // +patchStrategy=merge From a42d805a2eb5097c69bfe475698578ca65416951 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 29 Jul 2022 05:34:31 +0000 Subject: [PATCH 09/37] Showing FQDN of LB if it's ingress IP is not available --- .../database/oraclerestdataservice_controller.go | 13 +++++++++---- .../database/singleinstancedatabase_controller.go | 11 ++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/controllers/database/oraclerestdataservice_controller.go b/controllers/database/oraclerestdataservice_controller.go index 9021a6ec..319b0d1e 100644 --- a/controllers/database/oraclerestdataservice_controller.go +++ b/controllers/database/oraclerestdataservice_controller.go @@ -855,13 +855,18 @@ func (r *OracleRestDataServiceReconciler) createSVC(ctx context.Context, req ctr m.Status.ServiceIP = "" if m.Spec.LoadBalancer { if len(svc.Status.LoadBalancer.Ingress) > 0 { - m.Status.DatabaseApiUrl = "https://" + svc.Status.LoadBalancer.Ingress[0].IP + ":" + + // 'lbAddress' will contain the Fully Qualified Hostname of the LB. If the hostname is not available it will contain the IP address of the LB + lbAddress := svc.Status.LoadBalancer.Ingress[0].Hostname + if lbAddress == "" { + lbAddress = svc.Status.LoadBalancer.Ingress[0].IP + } + m.Status.DatabaseApiUrl = "https://" + lbAddress + ":" + fmt.Sprint(svc.Spec.Ports[0].Port) + "/ords/" + n.Status.Pdbname + "/_/db-api/stable/" - m.Status.ServiceIP = svc.Status.LoadBalancer.Ingress[0].IP - m.Status.DatabaseActionsUrl = "https://" + svc.Status.LoadBalancer.Ingress[0].IP + ":" + + m.Status.ServiceIP = lbAddress + m.Status.DatabaseActionsUrl = "https://" + lbAddress + ":" + fmt.Sprint(svc.Spec.Ports[0].Port) + "/ords/sql-developer" if m.Status.ApexConfigured { - m.Status.ApxeUrl = "https://" + svc.Status.LoadBalancer.Ingress[0].IP + ":" + + m.Status.ApxeUrl = "https://" + lbAddress + ":" + fmt.Sprint(svc.Spec.Ports[0].Port) + "/ords/" + n.Status.Pdbname + "/apex" } } diff --git a/controllers/database/singleinstancedatabase_controller.go b/controllers/database/singleinstancedatabase_controller.go index 714c07c0..a97e3446 100644 --- a/controllers/database/singleinstancedatabase_controller.go +++ b/controllers/database/singleinstancedatabase_controller.go @@ -1090,9 +1090,14 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex if m.Spec.LoadBalancer { m.Status.ClusterConnectString = svc.Name + "." + svc.Namespace + ":" + fmt.Sprint(svc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) if len(svc.Status.LoadBalancer.Ingress) > 0 { - m.Status.ConnectString = svc.Status.LoadBalancer.Ingress[0].IP + ":" + fmt.Sprint(svc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) - m.Status.PdbConnectString = svc.Status.LoadBalancer.Ingress[0].IP + ":" + fmt.Sprint(svc.Spec.Ports[0].Port) + "/" + strings.ToUpper(pdbName) - m.Status.OemExpressUrl = "https://" + svc.Status.LoadBalancer.Ingress[0].IP + ":" + fmt.Sprint(svc.Spec.Ports[1].Port) + "/em" + // 'lbAddress' will contain the Fully Qualified Hostname of the LB. If the hostname is not available it will contain the IP address of the LB + lbAddress := svc.Status.LoadBalancer.Ingress[0].Hostname + if lbAddress == "" { + lbAddress = svc.Status.LoadBalancer.Ingress[0].IP + } + m.Status.ConnectString = lbAddress + ":" + fmt.Sprint(svc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) + m.Status.PdbConnectString = lbAddress + ":" + fmt.Sprint(svc.Spec.Ports[0].Port) + "/" + strings.ToUpper(pdbName) + m.Status.OemExpressUrl = "https://" + lbAddress + ":" + fmt.Sprint(svc.Spec.Ports[1].Port) + "/em" } return requeueN, nil } From b48aadde5edb5d6ab36e5e12d4110813248fa725 Mon Sep 17 00:00:00 2001 From: harayuanwang Date: Fri, 29 Jul 2022 19:14:58 +0000 Subject: [PATCH 10/37] Tinglwan fix makefile --- Makefile | 56 +++++++++++++++++++++++++++++++------------------------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 78cf72ac..0d1163df 100644 --- a/Makefile +++ b/Makefile @@ -104,32 +104,38 @@ operator-yaml: manifests kustomize undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. $(KUSTOMIZE) build config/default | kubectl delete -f - +##@ Build Dependencies + +## Location to install dependencies to +LOCALBIN ?= $(shell pwd)/bin +$(LOCALBIN): + mkdir -p $(LOCALBIN) + +## Tool Binaries +KUSTOMIZE ?= $(LOCALBIN)/kustomize +CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen +ENVTEST ?= $(LOCALBIN)/setup-envtest + +## Tool Versions +KUSTOMIZE_VERSION ?= v3.8.7 +CONTROLLER_TOOLS_VERSION ?= v0.6.1 + +KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" +.PHONY: kustomize +kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. +$(KUSTOMIZE): $(LOCALBIN) + curl -s $(KUSTOMIZE_INSTALL_SCRIPT) | bash -s -- $(subst v,,$(KUSTOMIZE_VERSION)) $(LOCALBIN) + +.PHONY: controller-gen +controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary. +$(CONTROLLER_GEN): $(LOCALBIN) + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION) + +.PHONY: envtest +envtest: $(ENVTEST) ## Download envtest-setup locally if necessary. +$(ENVTEST): $(LOCALBIN) + GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest -CONTROLLER_GEN = $(shell pwd)/bin/controller-gen -controller-gen: ## Download controller-gen locally if necessary. - $(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.6.1) - -KUSTOMIZE = $(shell pwd)/bin/kustomize -kustomize: ## Download kustomize locally if necessary. - $(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v3@v3.8.7) - -ENVTEST = $(shell pwd)/bin/setup-envtest -envtest: ## Download envtest-setup locally if necessary. - $(call go-get-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest@latest) - -# go-get-tool will 'go get' any package $2 and install it to $1. -PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST)))) -define go-get-tool -@[ -f $(1) ] || { \ -set -e ;\ -TMP_DIR=$$(mktemp -d) ;\ -cd $$TMP_DIR ;\ -go mod init tmp ;\ -echo "Downloading $(2)" ;\ -GOBIN=$(PROJECT_DIR)/bin go get $(2) ;\ -rm -rf $$TMP_DIR ;\ -} -endef .PHONY: bundle bundle: manifests kustomize ## Generate bundle manifests and metadata, then validate generated files. From 4e130468ecd9fd2a666f5fc658867d28af9eeb66 Mon Sep 17 00:00:00 2001 From: abhisbyk Date: Sun, 7 Aug 2022 23:48:46 +0530 Subject: [PATCH 11/37] Added cert renewal logic, enable/disable TCPs, creation of clusterIP service for ORDS/APEX installation Signed-off-by: abhisbyk --- .../v1alpha1/oraclerestdataservice_webhook.go | 1 - .../v1alpha1/singleinstancedatabase_types.go | 4 +- commons/database/constants.go | 26 +- commons/database/utils.go | 16 + ...se.oracle.com_singleinstancedatabases.yaml | 8 + .../singleinstancedatabase_controller.go | 324 ++++++++++++------ 6 files changed, 269 insertions(+), 110 deletions(-) diff --git a/apis/database/v1alpha1/oraclerestdataservice_webhook.go b/apis/database/v1alpha1/oraclerestdataservice_webhook.go index 0676c7b3..9e63eee2 100644 --- a/apis/database/v1alpha1/oraclerestdataservice_webhook.go +++ b/apis/database/v1alpha1/oraclerestdataservice_webhook.go @@ -39,7 +39,6 @@ package v1alpha1 import ( - apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" diff --git a/apis/database/v1alpha1/singleinstancedatabase_types.go b/apis/database/v1alpha1/singleinstancedatabase_types.go index ab20ae2a..5f8efbc4 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_types.go +++ b/apis/database/v1alpha1/singleinstancedatabase_types.go @@ -149,7 +149,9 @@ type SingleInstanceDatabaseStatus struct { ApexInstalled bool `json:"apexInstalled,omitempty"` PrebuiltDB bool `json:"prebuiltDB,omitempty"` // +kubebuilder:default:=false - IsTcpsEnabled bool `json:"isTcpsEnabled"` + IsTcpsEnabled bool `json:"isTcpsEnabled"` + TcpsPort int `json:"tcpsPort,omitempty"` + CertCreationTimestamp string `json:"certCreationTimestamp,omitempty"` // +patchMergeKey=type // +patchStrategy=merge diff --git a/commons/database/constants.go b/commons/database/constants.go index 3c899e51..0e71742d 100644 --- a/commons/database/constants.go +++ b/commons/database/constants.go @@ -38,6 +38,8 @@ package commons +const DEFAULT_LISTENER_PORT int32 = 1521 + const ORACLE_UID int64 = 54321 const ORACLE_GUID int64 = 54321 @@ -318,12 +320,12 @@ const InitORDSCMD string = "if [ -f $ORDS_HOME/config/ords/defaults.xml ]; then "\numask 022" const GetSessionInfoSQL string = "select s.sid || ',' || s.serial# as Info FROM v\\$session s, v\\$process p " + - "WHERE (s.username = 'ORDS_PUBLIC_USER' or "+ - "s.username = 'APEX_PUBLIC_USER' or "+ - "s.username = 'APEX_REST_PUBLIC_USER' or "+ - "s.username = 'APEX_LISTENER' or "+ - "s.username = 'C##_DBAPI_CDB_ADMIN' or "+ - "s.username = 'C##_DBAPI_PDB_ADMIN' ) AND p.addr(+) = s.paddr;" + "WHERE (s.username = 'ORDS_PUBLIC_USER' or " + + "s.username = 'APEX_PUBLIC_USER' or " + + "s.username = 'APEX_REST_PUBLIC_USER' or " + + "s.username = 'APEX_LISTENER' or " + + "s.username = 'C##_DBAPI_CDB_ADMIN' or " + + "s.username = 'C##_DBAPI_PDB_ADMIN' ) AND p.addr(+) = s.paddr;" const KillSessionSQL string = "alter system kill session '%[1]s';" @@ -426,13 +428,12 @@ const ConfigureApexRest string = "if [ -f ${ORDS_HOME}/config/apex/apex_rest_con "echo -e \"%[1]s\n%[1]s\" | %[2]s ; else echo \"Apex Folder doesn't exist\" ; fi ;" const AlterApexUsers string = "\nALTER SESSION SET CONTAINER=%[2]s;" + - "\n ALTER USER APEX_PUBLIC_USER IDENTIFIED BY \\\"%[1]s\\\" ACCOUNT UNLOCK; "+ + "\n ALTER USER APEX_PUBLIC_USER IDENTIFIED BY \\\"%[1]s\\\" ACCOUNT UNLOCK; " + "\n ALTER USER APEX_REST_PUBLIC_USER IDENTIFIED BY \\\"%[1]s\\\" ACCOUNT UNLOCK;" + "\n ALTER USER APEX_LISTENER IDENTIFIED BY \\\"%[1]s\\\" ACCOUNT UNLOCK;" + "\nexec APEX_UTIL.set_workspace(p_workspace => 'INTERNAL');" + "\nexec APEX_UTIL.EDIT_USER(p_user_id => APEX_UTIL.GET_USER_ID('ADMIN'), p_user_name => 'ADMIN', p_web_password => '%[1]s', p_new_password => '%[1]s');\n" - const CopyApexImages string = " ( while true; do sleep 60; echo \"Copying Apex Images...\" ; done ) & mkdir -p /opt/oracle/oradata/${ORACLE_SID^^}_ORDS/apex/images && " + " cp -R /opt/oracle/oradata/${ORACLE_SID^^}/apex/images/* /opt/oracle/oradata/${ORACLE_SID^^}_ORDS/apex/images; chown -R oracle:oinstall /opt/oracle/oradata/${ORACLE_SID^^}_ORDS/apex; kill -9 $!;" @@ -480,3 +481,12 @@ const SetApexUsers string = "\numask 177" + // Get Sid, Pdbname, Edition for prebuilt db const GetSidPdbEditionCMD string = "echo $ORACLE_SID,$ORACLE_PDB,$ORACLE_EDITION,Edition;" + +// Command to enable TCPS as a formatted string. The parameter would be the port at which TCPS is enabled. +const EnableTcpsCMD string = "$ORACLE_BASE/$CONFIG_TCPS_FILE %d" + +// Command for TCPS certs renewal to prevent their expiry. It is same as the EnableTcpsCMD +const RenewCertsCMD string = EnableTcpsCMD + +// Command to disable TCPS +const DisableTcpsCMD string = "$ORACLE_BASE/$CONFIG_TCPS_FILE disable" diff --git a/commons/database/utils.go b/commons/database/utils.go index c5ebbefd..100b532c 100644 --- a/commons/database/utils.go +++ b/commons/database/utils.go @@ -50,6 +50,7 @@ import ( "unicode" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" @@ -673,3 +674,18 @@ func ApexPasswordValidator(pwd string) bool { return hasMinLen && hasUpper && hasLower && hasNumber && hasSpecial } + +// Function for patching the K8s service with the payload. +// Patch strategy used: Strategic Merge Patch +func PatchService(config *rest.Config, namespace string, ctx context.Context, req ctrl.Request, svcName string, payload string) error { + log := ctrllog.FromContext(ctx).WithValues("patchService", req.NamespacedName) + client, err := kubernetes.NewForConfig(config) + if err != nil { + log.Error(err, "config error") + } + + // Trying to patch the service resource using Strategic Merge strategy + log.Info("Patching the service", "Service", svcName) + _, err = client.CoreV1().Services(namespace).Patch(ctx, svcName, types.StrategicMergePatchType, []byte(payload), metav1.PatchOptions{}) + return err +} diff --git a/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml b/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml index 6e7c3ec4..b6aa6352 100644 --- a/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml +++ b/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml @@ -175,6 +175,8 @@ spec: type: boolean archiveLog: type: string + certCreationTimestamp: + type: string charset: type: string cloneFrom: @@ -283,6 +285,9 @@ spec: type: integer initSgaSize: type: integer + isTcpsEnabled: + default: false + type: boolean nodes: items: type: string @@ -329,7 +334,10 @@ spec: type: object status: type: string + tcpsPort: + type: integer required: + - isTcpsEnabled - persistence type: object type: object diff --git a/controllers/database/singleinstancedatabase_controller.go b/controllers/database/singleinstancedatabase_controller.go index c852e9ae..d58aa76c 100644 --- a/controllers/database/singleinstancedatabase_controller.go +++ b/controllers/database/singleinstancedatabase_controller.go @@ -78,8 +78,9 @@ type SingleInstanceDatabaseReconciler struct { var requeueY ctrl.Result = ctrl.Result{Requeue: true, RequeueAfter: 15 * time.Second} var requeueN ctrl.Result = ctrl.Result{} -// Service Port Declaration -var svc_port string +// For scheduling reconcile to renew certs if TCPS is enabled +// Default value is requeueN (No reconcile) +var futureRequeue ctrl.Result = requeueN const singleInstanceDatabaseFinalizer = "database.oracle.com/singleinstancedatabasefinalizer" @@ -137,14 +138,6 @@ func (r *SingleInstanceDatabaseReconciler) Reconcile(ctx context.Context, req ct r.Status().Update(ctx, singleInstanceDatabase) } - // Service Port Initialization - svc_port = func() string { - if singleInstanceDatabase.Spec.EnableTCPS { - return strconv.Itoa(singleInstanceDatabase.Spec.TcpsPort) - } - return "1521" - }() - // Manage SingleInstanceDatabase Deletion result, err = r.manageSingleInstanceDatabaseDeletion(req, ctx, singleInstanceDatabase) if result.Requeue { @@ -203,6 +196,13 @@ func (r *SingleInstanceDatabaseReconciler) Reconcile(ctx context.Context, req ct } } + // Configure TCPS + result, err = r.configTcps(singleInstanceDatabase, readyPod, ctx, req) + if result.Requeue { + r.Log.Info("Reconcile queued") + return result, nil + } + // Update DB config result, err = r.updateDBConfig(singleInstanceDatabase, readyPod, ctx, req) if result.Requeue { @@ -241,6 +241,15 @@ func (r *SingleInstanceDatabaseReconciler) Reconcile(ctx context.Context, req ct completed = true r.Log.Info("Reconcile completed") + + // Scheduling a reconcile for certificate renewal, if TCPS is enabled + if futureRequeue != requeueN { + r.Log.Info("Scheduling Reconcile for cert renewal", "Duration(Hours)", futureRequeue.RequeueAfter.Hours()) + copyFutureRequeue := futureRequeue + futureRequeue = requeueN + return copyFutureRequeue, nil + } + return requeueN, nil } @@ -650,16 +659,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns }, }, }, - Ports: []corev1.ContainerPort{ - {ContainerPort: func() int32 { - if m.Spec.EnableTCPS { - return int32(m.Spec.TcpsPort) - } - return int32(1521) - }(), - }, - {ContainerPort: 5500}, - }, + Ports: []corev1.ContainerPort{{ContainerPort: dbcommons.DEFAULT_LISTENER_PORT}, {ContainerPort: 5500}}, ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ @@ -707,7 +707,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns }, { Name: "SVC_PORT", - Value: svc_port, + Value: strconv.Itoa(int(dbcommons.DEFAULT_LISTENER_PORT)), }, { Name: "ORACLE_CHARACTERSET", @@ -717,14 +717,6 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns Name: "ORACLE_EDITION", Value: m.Spec.Edition, }, - { - Name: "ENABLE_TCPS", - Value: strconv.FormatBool(m.Spec.EnableTCPS), - }, - { - Name: "TCPS_PORT", - Value: strconv.Itoa(m.Spec.TcpsPort), - }, } } if m.Spec.CloneFrom == "" { @@ -736,7 +728,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns }, { Name: "SVC_PORT", - Value: svc_port, + Value: strconv.Itoa(int(dbcommons.DEFAULT_LISTENER_PORT)), }, { Name: "CREATE_PDB", @@ -794,14 +786,6 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns Name: "SKIP_DATAPATCH", Value: "true", }, - { - Name: "ENABLE_TCPS", - Value: strconv.FormatBool(m.Spec.EnableTCPS), - }, - { - Name: "TCPS_PORT", - Value: strconv.Itoa(m.Spec.TcpsPort), - }, } } // For clone DB use case @@ -812,7 +796,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns }, { Name: "SVC_PORT", - Value: svc_port, + Value: strconv.Itoa(int(dbcommons.DEFAULT_LISTENER_PORT)), }, { Name: "ORACLE_SID", @@ -826,7 +810,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns Name: "PRIMARY_DB_CONN_STR", Value: func() string { if dbcommons.IsSourceDatabaseOnCluster(m.Spec.CloneFrom) { - return n.Name + ":" + svc_port + "/" + n.Spec.Sid + return n.Name + ":" + strconv.Itoa(int(dbcommons.DEFAULT_LISTENER_PORT)) + "/" + n.Spec.Sid } return m.Spec.CloneFrom }(), @@ -890,13 +874,14 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns //############################################################################# // Instantiate Service spec from SingleInstanceDatabase spec //############################################################################# -func (r *SingleInstanceDatabaseReconciler) instantiateSVCSpec(m *dbapi.SingleInstanceDatabase) *corev1.Service { +func (r *SingleInstanceDatabaseReconciler) instantiateSVCSpec(m *dbapi.SingleInstanceDatabase, + svcName string, listenerPort int32, svcType corev1.ServiceType) *corev1.Service { svc := &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", }, ObjectMeta: metav1.ObjectMeta{ - Name: m.Name, + Name: svcName, Namespace: m.Namespace, Labels: map[string]string{ "app": m.Name, @@ -914,13 +899,8 @@ func (r *SingleInstanceDatabaseReconciler) instantiateSVCSpec(m *dbapi.SingleIns Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{ { - Name: "listener", - Port: func() int32 { - if m.Spec.EnableTCPS { - return int32(m.Spec.TcpsPort) - } - return int32(1521) - }(), + Name: "listener", + Port: listenerPort, Protocol: corev1.ProtocolTCP, }, { @@ -932,12 +912,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiateSVCSpec(m *dbapi.SingleIns Selector: map[string]string{ "app": m.Name, }, - Type: corev1.ServiceType(func() string { - if m.Spec.LoadBalancer { - return "LoadBalancer" - } - return "NodePort" - }()), + Type: svcType, }, } // Set SingleInstanceDatabase instance as the owner and controller @@ -1073,49 +1048,121 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex log := r.Log.WithValues("createOrReplaceSVC", req.NamespacedName) - svcDeleted := false - // Check if the Service already exists, if not create a new one - svc := &corev1.Service{} - // Get retrieves an obj ( a struct pointer ) for the given object key from the Kubernetes Cluster. - err := r.Get(ctx, types.NamespacedName{Name: m.Name, Namespace: m.Namespace}, svc) - if err == nil { - svcType := corev1.ServiceType("NodePort") - if m.Spec.LoadBalancer { - svcType = corev1.ServiceType("LoadBalancer") + /** If TCPS is enabled, there will be two k8s services: + 1. One service exposing the TCPS port for the user to connect, + 2. One service exposing default listerner port inside the cluster only (for ORDS, APEX installation etc) + **/ + + defaultSvc := &corev1.Service{} + tcpsSvc := &corev1.Service{} + // userSvc would indicate the service which is used to connect to the Database by the database user + var userSvc *corev1.Service + + defaultSvcName := m.Name + tcpsSvcName := m.Name + "-tcps" + + // Querying for the K8s service resources + getDefaultSvcErr := r.Get(ctx, types.NamespacedName{Name: defaultSvcName, Namespace: m.Namespace}, defaultSvc) + getTcpsSvcErr := r.Get(ctx, types.NamespacedName{Name: tcpsSvcName, Namespace: m.Namespace}, tcpsSvc) + + // svcType defines the type of the service as specified in the singleinstancedatabase.yaml file + svcType := corev1.ServiceType("NodePort") + if m.Spec.LoadBalancer { + svcType = corev1.ServiceType("LoadBalancer") + } + + if m.Spec.EnableTCPS { + if getDefaultSvcErr != nil && apierrors.IsNotFound(getDefaultSvcErr) { + // Create a new cluster service + svc := r.instantiateSVCSpec(m, defaultSvcName, dbcommons.DEFAULT_LISTENER_PORT, corev1.ServiceType("ClusterIP")) + log.Info("Creating a new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + err := r.Create(ctx, svc) + if err != nil { + log.Error(err, "Failed to create new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + return requeueY, err + } + } else if defaultSvc.Spec.Type != corev1.ServiceType("ClusterIP") { + // Patch the service type + payload := "{\"spec\": {\"type\": \"ClusterIP\"}}" + //Attempt Service patching + log.Info("Patching the service", "Service.Name", defaultSvc.Name, "Target Type", "ClusterIP") + err := dbcommons.PatchService(r.Config, m.Namespace, ctx, req, defaultSvcName, payload) + if err != nil { + log.Error(err, "Failed to patch Service") + return requeueY, err + } + } - if svc.Spec.Type != svcType { - log.Info("Deleting service", "name", svc.Name) - err = r.Delete(ctx, svc) + // When TCPS is enabled userSvc would point to tcpsSvc + userSvc = tcpsSvc + if getTcpsSvcErr != nil && apierrors.IsNotFound(getTcpsSvcErr) { + // Reset connect strings whenever service is recreated /* + m.Status.ConnectString = dbcommons.ValueUnavailable + m.Status.PdbConnectString = dbcommons.ValueUnavailable + m.Status.OemExpressUrl = dbcommons.ValueUnavailable + // Create a new service + svc := r.instantiateSVCSpec(m, tcpsSvcName, int32(m.Spec.TcpsPort), svcType) + log.Info("Creating a new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + err := r.Create(ctx, svc) if err != nil { - r.Log.Error(err, "Failed to delete service", "name", svc.Name) - return requeueN, err + log.Error(err, "Failed to create new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + return requeueY, err + } + userSvc = svc + } else if tcpsSvc.Spec.Type != svcType { + // Patch the service type + payload := fmt.Sprintf("{\"spec\": {\"type\": \"%s\"}}", svcType) + //Attempt Service patching + log.Info("Patching the service", "Service.Name", defaultSvc.Name, "Target Type", svcType) + err := dbcommons.PatchService(r.Config, m.Namespace, ctx, req, tcpsSvcName, payload) + if err != nil { + log.Error(err, "Failed to patch Service") + return requeueY, err } - svcDeleted = true } - } - if svcDeleted || err != nil && apierrors.IsNotFound(err) { - // Define a new Service - svc = r.instantiateSVCSpec(m) - log.Info("Creating a new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - err = r.Create(ctx, svc) - if err != nil { - log.Error(err, "Failed to create new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - return requeueY, err + + } else { + // Only one service is required if TCPS is not enabled + // userSvc would point to the defaultSvc + userSvc = defaultSvc + if getDefaultSvcErr != nil && apierrors.IsNotFound(getDefaultSvcErr) { + // Reset connect strings whenever service is recreated /* + m.Status.ConnectString = dbcommons.ValueUnavailable + m.Status.PdbConnectString = dbcommons.ValueUnavailable + m.Status.OemExpressUrl = dbcommons.ValueUnavailable + // Create a new service with + svc := r.instantiateSVCSpec(m, defaultSvcName, dbcommons.DEFAULT_LISTENER_PORT, svcType) + log.Info("Creating a new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + err := r.Create(ctx, svc) + if err != nil { + log.Error(err, "Failed to create new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + return requeueY, err + } + userSvc = svc + } else if defaultSvc.Spec.Type != svcType { + // Patch the service type + payload := fmt.Sprintf("{\"spec\": {\"type\": \"%s\"}}", svcType) + //Attempt Service patching + log.Info("Patching the service", "Service.Name", defaultSvc.Name, "Target Type", svcType) + err := dbcommons.PatchService(r.Config, m.Namespace, ctx, req, defaultSvcName, payload) + if err != nil { + log.Error(err, "Failed to patch Service") + return requeueY, err + } } - eventReason := "Service creation" - eventMsg := "successfully created service type " + string(svc.Spec.Type) - r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) - log.Info(eventMsg) - // Reset connect strings whenever service is recreated /* - m.Status.ConnectString = dbcommons.ValueUnavailable - m.Status.PdbConnectString = dbcommons.ValueUnavailable - m.Status.OemExpressUrl = dbcommons.ValueUnavailable - } else if err != nil { - log.Error(err, "Failed to get Service") - return requeueY, err + + if getTcpsSvcErr == nil { + // Delete this tcps service + log.Info("Deleting service", "name", tcpsSvcName) + err := r.Delete(ctx, tcpsSvc) + if err != nil { + r.Log.Error(err, "Failed to delete service", "name", tcpsSvc.Name) + return requeueN, err + } + } + } - log.Info("Found Existing Service ", "Service Name ", svc.Name) pdbName := strings.ToUpper(m.Spec.Pdbname) sid := m.Spec.Sid @@ -1129,21 +1176,21 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex } if m.Spec.LoadBalancer { - m.Status.ClusterConnectString = svc.Name + "." + svc.Namespace + ":" + fmt.Sprint(svc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) - if len(svc.Status.LoadBalancer.Ingress) > 0 { - m.Status.ConnectString = svc.Status.LoadBalancer.Ingress[0].IP + ":" + fmt.Sprint(svc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) - m.Status.PdbConnectString = svc.Status.LoadBalancer.Ingress[0].IP + ":" + fmt.Sprint(svc.Spec.Ports[0].Port) + "/" + strings.ToUpper(pdbName) - m.Status.OemExpressUrl = "https://" + svc.Status.LoadBalancer.Ingress[0].IP + ":" + fmt.Sprint(svc.Spec.Ports[1].Port) + "/em" + m.Status.ClusterConnectString = userSvc.Name + "." + userSvc.Namespace + ":" + fmt.Sprint(userSvc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) + if len(userSvc.Status.LoadBalancer.Ingress) > 0 { + m.Status.ConnectString = userSvc.Status.LoadBalancer.Ingress[0].IP + ":" + fmt.Sprint(userSvc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) + m.Status.PdbConnectString = userSvc.Status.LoadBalancer.Ingress[0].IP + ":" + fmt.Sprint(userSvc.Spec.Ports[0].Port) + "/" + strings.ToUpper(pdbName) + m.Status.OemExpressUrl = "https://" + userSvc.Status.LoadBalancer.Ingress[0].IP + ":" + fmt.Sprint(userSvc.Spec.Ports[1].Port) + "/em" } return requeueN, nil } - m.Status.ClusterConnectString = svc.Name + "." + svc.Namespace + ":" + fmt.Sprint(svc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) + m.Status.ClusterConnectString = userSvc.Name + "." + userSvc.Namespace + ":" + fmt.Sprint(userSvc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) nodeip := dbcommons.GetNodeIp(r, ctx, req) if nodeip != "" { - m.Status.ConnectString = nodeip + ":" + fmt.Sprint(svc.Spec.Ports[0].NodePort) + "/" + strings.ToUpper(sid) - m.Status.PdbConnectString = nodeip + ":" + fmt.Sprint(svc.Spec.Ports[0].NodePort) + "/" + strings.ToUpper(pdbName) - m.Status.OemExpressUrl = "https://" + nodeip + ":" + fmt.Sprint(svc.Spec.Ports[1].NodePort) + "/em" + m.Status.ConnectString = nodeip + ":" + fmt.Sprint(userSvc.Spec.Ports[0].NodePort) + "/" + strings.ToUpper(sid) + m.Status.PdbConnectString = nodeip + ":" + fmt.Sprint(userSvc.Spec.Ports[0].NodePort) + "/" + strings.ToUpper(pdbName) + m.Status.OemExpressUrl = "https://" + nodeip + ":" + fmt.Sprint(userSvc.Spec.Ports[1].NodePort) + "/em" } return requeueN, nil @@ -1708,6 +1755,83 @@ func (r *SingleInstanceDatabaseReconciler) deleteWallet(m *dbapi.SingleInstanceD return requeueN, nil } +//############################################################################# +// Configuring TCPS +//############################################################################# +func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDatabase, + readyPod corev1.Pod, ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + eventReason := "Configuring TCPS" + if m.Spec.EnableTCPS && !m.Status.IsTcpsEnabled { + // Enable TCPS + eventMsg := "Enabling TCPS in the database..." + r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) + + _, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", + ctx, req, false, "bash", "-c", fmt.Sprintf(dbcommons.EnableTcpsCMD, m.Spec.TcpsPort)) + if err != nil { + r.Log.Error(err, err.Error()) + eventMsg = "Error encountered in enabling TCPS!" + r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) + return requeueY, nil + } + // Updating the Status and publishing the event + m.Status.CertCreationTimestamp = time.Now().Format(time.RFC3339) + m.Status.IsTcpsEnabled = true + r.Status().Update(ctx, m) + + eventMsg = "TCPS Enabled." + r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) + + // 26040h = 1085 days + futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: func() time.Duration { requeueDuration, _ := time.ParseDuration("26040h"); return requeueDuration }()} + + } else if !m.Spec.EnableTCPS && m.Status.IsTcpsEnabled { + // Disable TCPS + eventMsg := "Disabling TCPS in the database..." + r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) + + _, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", + ctx, req, false, "bash", "-c", dbcommons.DisableTcpsCMD) + if err != nil { + r.Log.Error(err, err.Error()) + return requeueY, nil + } + + // Updating the Status and publishing the event + m.Status.CertCreationTimestamp = "" + m.Status.IsTcpsEnabled = false + r.Status().Update(ctx, m) + + eventMsg = "TCPS Disabled." + r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) + + } else if m.Spec.EnableTCPS && m.Status.IsTcpsEnabled { + // Cert Renewal Logic + // Certificates are renewed when 10 days remain for certs expiry + certCreationTimestamp, _ := time.Parse(time.RFC3339, m.Status.CertCreationTimestamp) + duration := time.Since(certCreationTimestamp) + allowdDuration, _ := time.ParseDuration("26000h") + if duration > allowdDuration { + _, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", + ctx, req, false, "bash", "-c", fmt.Sprintf(dbcommons.EnableTcpsCMD, m.Spec.TcpsPort)) + if err != nil { + r.Log.Error(err, err.Error()) + return requeueY, nil + } + // Updating the Status and publishing the event + m.Status.CertCreationTimestamp = time.Now().Format(time.RFC3339) + r.Status().Update(ctx, m) + + eventMsg := "TCPS Certificates Renewed at time %s," + r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg, time.Now().Format(time.RFC3339)) + + // 26040h = 1085 days + futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: func() time.Duration { requeueDuration, _ := time.ParseDuration("26040h"); return requeueDuration }()} + } + } + return requeueN, nil +} + //############################################################################# // Execute Datapatch //############################################################################# From a22970dc64f4ffa4bfacbefa3aa5695e23a6e7ec Mon Sep 17 00:00:00 2001 From: abhisbyk Date: Mon, 8 Aug 2022 12:32:18 +0530 Subject: [PATCH 12/37] Added port change validation, status to Updating while enabling/disabling tcps Signed-off-by: abhisbyk --- .../v1alpha1/singleinstancedatabase_webhook.go | 4 ++++ .../database/singleinstancedatabase_controller.go | 11 +++++++++++ 2 files changed, 15 insertions(+) diff --git a/apis/database/v1alpha1/singleinstancedatabase_webhook.go b/apis/database/v1alpha1/singleinstancedatabase_webhook.go index 2b527083..9dc171e4 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_webhook.go +++ b/apis/database/v1alpha1/singleinstancedatabase_webhook.go @@ -297,6 +297,10 @@ func (r *SingleInstanceDatabase) ValidateUpdate(oldRuntimeObject runtime.Object) allErrs = append(allErrs, field.Forbidden(field.NewPath("spec").Child("persistence"), "uninstall ORDS to change Persistence")) } + if old.Status.IsTcpsEnabled && old.Status.TcpsPort != r.Spec.TcpsPort { + allErrs = append(allErrs, + field.Forbidden(field.NewPath("spec").Child("tcpsPort"), "cannot change TCPS port, please disable TCPS first then enable it with newly desired port")) + } if len(allErrs) == 0 { return nil } diff --git a/controllers/database/singleinstancedatabase_controller.go b/controllers/database/singleinstancedatabase_controller.go index d58aa76c..d232d189 100644 --- a/controllers/database/singleinstancedatabase_controller.go +++ b/controllers/database/singleinstancedatabase_controller.go @@ -1763,6 +1763,9 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat eventReason := "Configuring TCPS" if m.Spec.EnableTCPS && !m.Status.IsTcpsEnabled { // Enable TCPS + m.Status.Status = dbcommons.StatusUpdating + r.Status().Update(ctx, m) + eventMsg := "Enabling TCPS in the database..." r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) @@ -1777,6 +1780,7 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat // Updating the Status and publishing the event m.Status.CertCreationTimestamp = time.Now().Format(time.RFC3339) m.Status.IsTcpsEnabled = true + m.Status.TcpsPort = m.Spec.TcpsPort r.Status().Update(ctx, m) eventMsg = "TCPS Enabled." @@ -1787,6 +1791,9 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat } else if !m.Spec.EnableTCPS && m.Status.IsTcpsEnabled { // Disable TCPS + m.Status.Status = dbcommons.StatusUpdating + r.Status().Update(ctx, m) + eventMsg := "Disabling TCPS in the database..." r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) @@ -1799,6 +1806,7 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat // Updating the Status and publishing the event m.Status.CertCreationTimestamp = "" + m.Status.TcpsPort = 0 m.Status.IsTcpsEnabled = false r.Status().Update(ctx, m) @@ -1812,6 +1820,9 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat duration := time.Since(certCreationTimestamp) allowdDuration, _ := time.ParseDuration("26000h") if duration > allowdDuration { + m.Status.Status = dbcommons.StatusUpdating + r.Status().Update(ctx, m) + _, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", ctx, req, false, "bash", "-c", fmt.Sprintf(dbcommons.EnableTcpsCMD, m.Spec.TcpsPort)) if err != nil { From 9886cb9e8cf7fadd6f4076f43aeb0ec25b2332b0 Mon Sep 17 00:00:00 2001 From: abhisbyk Date: Mon, 8 Aug 2022 21:40:28 +0530 Subject: [PATCH 13/37] Updating the status properly for the LB Signed-off-by: abhisbyk --- .../singleinstancedatabase_controller.go | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/controllers/database/singleinstancedatabase_controller.go b/controllers/database/singleinstancedatabase_controller.go index 267d36ce..128ae2ff 100644 --- a/controllers/database/singleinstancedatabase_controller.go +++ b/controllers/database/singleinstancedatabase_controller.go @@ -1124,13 +1124,15 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex } else { // Only one service is required if TCPS is not enabled + + // Reset connect strings whenever service is recreated /* + m.Status.ConnectString = dbcommons.ValueUnavailable + m.Status.PdbConnectString = dbcommons.ValueUnavailable + m.Status.OemExpressUrl = dbcommons.ValueUnavailable + // userSvc would point to the defaultSvc userSvc = defaultSvc if getDefaultSvcErr != nil && apierrors.IsNotFound(getDefaultSvcErr) { - // Reset connect strings whenever service is recreated /* - m.Status.ConnectString = dbcommons.ValueUnavailable - m.Status.PdbConnectString = dbcommons.ValueUnavailable - m.Status.OemExpressUrl = dbcommons.ValueUnavailable // Create a new service with svc := r.instantiateSVCSpec(m, defaultSvcName, dbcommons.DEFAULT_LISTENER_PORT, svcType) log.Info("Creating a new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) @@ -1187,15 +1189,14 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex m.Status.PdbConnectString = lbAddress + ":" + fmt.Sprint(userSvc.Spec.Ports[0].Port) + "/" + strings.ToUpper(pdbName) m.Status.OemExpressUrl = "https://" + lbAddress + ":" + fmt.Sprint(userSvc.Spec.Ports[1].Port) + "/em" } - return requeueN, nil - } - - m.Status.ClusterConnectString = userSvc.Name + "." + userSvc.Namespace + ":" + fmt.Sprint(userSvc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) - nodeip := dbcommons.GetNodeIp(r, ctx, req) - if nodeip != "" { - m.Status.ConnectString = nodeip + ":" + fmt.Sprint(userSvc.Spec.Ports[0].NodePort) + "/" + strings.ToUpper(sid) - m.Status.PdbConnectString = nodeip + ":" + fmt.Sprint(userSvc.Spec.Ports[0].NodePort) + "/" + strings.ToUpper(pdbName) - m.Status.OemExpressUrl = "https://" + nodeip + ":" + fmt.Sprint(userSvc.Spec.Ports[1].NodePort) + "/em" + } else { + m.Status.ClusterConnectString = userSvc.Name + "." + userSvc.Namespace + ":" + fmt.Sprint(userSvc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) + nodeip := dbcommons.GetNodeIp(r, ctx, req) + if nodeip != "" { + m.Status.ConnectString = nodeip + ":" + fmt.Sprint(userSvc.Spec.Ports[0].NodePort) + "/" + strings.ToUpper(sid) + m.Status.PdbConnectString = nodeip + ":" + fmt.Sprint(userSvc.Spec.Ports[0].NodePort) + "/" + strings.ToUpper(pdbName) + m.Status.OemExpressUrl = "https://" + nodeip + ":" + fmt.Sprint(userSvc.Spec.Ports[1].NodePort) + "/em" + } } return requeueN, nil From 522ab6b2ad68523c39f9434068aa7ea8bd212abc Mon Sep 17 00:00:00 2001 From: Ruggero Citton Date: Tue, 16 Aug 2022 10:46:48 +0200 Subject: [PATCH 14/37] bug:33822886 --- apis/database/v1alpha1/pdb_webhook.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/apis/database/v1alpha1/pdb_webhook.go b/apis/database/v1alpha1/pdb_webhook.go index b02b797d..573a8729 100644 --- a/apis/database/v1alpha1/pdb_webhook.go +++ b/apis/database/v1alpha1/pdb_webhook.go @@ -36,6 +36,11 @@ ** SOFTWARE. */ +/* MODIFIED (MM/DD/YY) +** rcitton 07/14/22 - 33822886 +*/ + + package v1alpha1 import ( @@ -157,6 +162,14 @@ func (r *PDB) validateAction(allErrs *field.ErrorList) { *allErrs = append(*allErrs, field.Required(field.NewPath("spec").Child("fileNameConversions"), "Please specify a value for fileNameConversions. Values can be a filename convert pattern or NONE")) } + if r.Spec.TotalSize == "" { + *allErrs = append(*allErrs, + field.Required(field.NewPath("spec").Child("totalSize"), "When the storage is not UNLIMITED the Total Size must be specified")) + } + if r.Spec.TempSize == "" { + *allErrs = append(*allErrs, + field.Required(field.NewPath("spec").Child("tempSize"), "When the storage is not UNLIMITED the Temp Size must be specified")) + } if *(r.Spec.TDEImport) { r.validateTDEInfo(allErrs) } @@ -168,11 +181,11 @@ func (r *PDB) validateAction(allErrs *field.ErrorList) { } if r.Spec.TotalSize == "" { *allErrs = append(*allErrs, - field.Required(field.NewPath("spec").Child("totalSize"), "Please specify size of the tablespace")) + field.Required(field.NewPath("spec").Child("totalSize"), "When the storage is not UNLIMITED the Total Size must be specified")) } if r.Spec.TempSize == "" { *allErrs = append(*allErrs, - field.Required(field.NewPath("spec").Child("tempSize"), "Please specify size of the temporary tablespace")) + field.Required(field.NewPath("spec").Child("tempSize"), "When the storage is not UNLIMITED the Temp Size must be specified")) } case "PLUG": if r.Spec.XMLFileName == "" { From ea48a4ec09c7302a1895e779a04b0536033e3267 Mon Sep 17 00:00:00 2001 From: abhisbyk Date: Mon, 22 Aug 2022 12:58:48 +0530 Subject: [PATCH 15/37] Two services: clusterIP for internal communication and external LB/NP for users Signed-off-by: abhisbyk --- .../v1alpha1/singleinstancedatabase_types.go | 5 +- .../singleinstancedatabase_webhook.go | 14 +- commons/database/constants.go | 9 +- ...se.oracle.com_singleinstancedatabases.yaml | 10 +- .../samples/sidb/singleinstancedatabase.yaml | 8 +- .../singleinstancedatabase_controller.go | 324 +++++++++++------- 6 files changed, 224 insertions(+), 146 deletions(-) diff --git a/apis/database/v1alpha1/singleinstancedatabase_types.go b/apis/database/v1alpha1/singleinstancedatabase_types.go index 5f8efbc4..d5b26e78 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_types.go +++ b/apis/database/v1alpha1/singleinstancedatabase_types.go @@ -61,12 +61,12 @@ type SingleInstanceDatabaseSpec struct { Charset string `json:"charset,omitempty"` Pdbname string `json:"pdbName,omitempty"` LoadBalancer bool `json:"loadBalancer,omitempty"` + ServicePort int `json:"servicePort,omitempty"` ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` FlashBack bool `json:"flashBack,omitempty"` ArchiveLog bool `json:"archiveLog,omitempty"` ForceLogging bool `json:"forceLog,omitempty"` EnableTCPS bool `json:"enableTCPS,omitempty"` - TcpsPort int `json:"tcpsPort,omitempty"` CloneFrom string `json:"cloneFrom,omitempty"` ReadinessCheckPeriod int `json:"readinessCheckPeriod,omitempty"` @@ -150,8 +150,9 @@ type SingleInstanceDatabaseStatus struct { PrebuiltDB bool `json:"prebuiltDB,omitempty"` // +kubebuilder:default:=false IsTcpsEnabled bool `json:"isTcpsEnabled"` - TcpsPort int `json:"tcpsPort,omitempty"` CertCreationTimestamp string `json:"certCreationTimestamp,omitempty"` + DbHostname string `json:"dbHostname,omitempty"` + DbPort int `json:"dbPort,omitempty"` // +patchMergeKey=type // +patchStrategy=merge diff --git a/apis/database/v1alpha1/singleinstancedatabase_webhook.go b/apis/database/v1alpha1/singleinstancedatabase_webhook.go index 9dc171e4..0d450871 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_webhook.go +++ b/apis/database/v1alpha1/singleinstancedatabase_webhook.go @@ -234,6 +234,16 @@ func (r *SingleInstanceDatabase) ValidateCreate() error { } } + // servicePort validation + if !r.Spec.LoadBalancer { + // NodePort service is expected. In this case servicePort should be in range 30000-32767 + if r.Spec.ServicePort != 0 && (r.Spec.ServicePort < 30000 || r.Spec.ServicePort > 32767) { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec").Child("servicePort"), r.Spec.ServicePort, + "servicePort should be in 30000-32767 range.")) + } + } + if len(allErrs) == 0 { return nil } @@ -297,10 +307,6 @@ func (r *SingleInstanceDatabase) ValidateUpdate(oldRuntimeObject runtime.Object) allErrs = append(allErrs, field.Forbidden(field.NewPath("spec").Child("persistence"), "uninstall ORDS to change Persistence")) } - if old.Status.IsTcpsEnabled && old.Status.TcpsPort != r.Spec.TcpsPort { - allErrs = append(allErrs, - field.Forbidden(field.NewPath("spec").Child("tcpsPort"), "cannot change TCPS port, please disable TCPS first then enable it with newly desired port")) - } if len(allErrs) == 0 { return nil } diff --git a/commons/database/constants.go b/commons/database/constants.go index 0e71742d..3aeb66f7 100644 --- a/commons/database/constants.go +++ b/commons/database/constants.go @@ -38,7 +38,9 @@ package commons -const DEFAULT_LISTENER_PORT int32 = 1521 +const CONTAINER_LISTENER_PORT int32 = 1521 + +const CONTAINER_TCPS_PORT int32 = 1522 const ORACLE_UID int64 = 54321 @@ -483,10 +485,13 @@ const SetApexUsers string = "\numask 177" + const GetSidPdbEditionCMD string = "echo $ORACLE_SID,$ORACLE_PDB,$ORACLE_EDITION,Edition;" // Command to enable TCPS as a formatted string. The parameter would be the port at which TCPS is enabled. -const EnableTcpsCMD string = "$ORACLE_BASE/$CONFIG_TCPS_FILE %d" +const EnableTcpsCMD string = "$ORACLE_BASE/$CONFIG_TCPS_FILE" // Command for TCPS certs renewal to prevent their expiry. It is same as the EnableTcpsCMD const RenewCertsCMD string = EnableTcpsCMD // Command to disable TCPS const DisableTcpsCMD string = "$ORACLE_BASE/$CONFIG_TCPS_FILE disable" + +// TCPS clientWallet update command +const ClientWalletUpdate string = "sed -i -e 's/HOST.*$/HOST=%s)/g' -e 's/PORT.*$/PORT=%d)/g' ${ORACLE_BASE}/oradata/clientWallet/${ORACLE_SID}/tnsnames.ora" diff --git a/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml b/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml index b6aa6352..b8383ef8 100644 --- a/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml +++ b/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml @@ -156,14 +156,14 @@ spec: additionalProperties: type: string type: object + servicePort: + type: integer sid: description: SID must be alphanumeric (no special characters, only a-z, A-Z, 0-9), and no longer than 12 characters. maxLength: 12 pattern: ^[a-zA-Z0-9]+$ type: string - tcpsPort: - type: integer required: - image type: object @@ -263,6 +263,10 @@ spec: datafilesPatched: default: "false" type: string + dbHostname: + type: string + dbPort: + type: integer edition: type: string flashBack: @@ -334,8 +338,6 @@ spec: type: object status: type: string - tcpsPort: - type: integer required: - isTcpsEnabled - persistence diff --git a/config/samples/sidb/singleinstancedatabase.yaml b/config/samples/sidb/singleinstancedatabase.yaml index 0d099255..9adc9601 100644 --- a/config/samples/sidb/singleinstancedatabase.yaml +++ b/config/samples/sidb/singleinstancedatabase.yaml @@ -47,9 +47,6 @@ spec: ## Enable TCPS enableTCPS: false - ## TCPS custom port - tcpsPort: 1522 - ## NA if cloning from a SourceDB (cloneFrom is set) ## Specify both sgaSize and pgaSize (in MB) or dont specify both ## Specify Non-Zero value to use @@ -85,6 +82,11 @@ spec: ## Type of service . Applicable on cloud enviroments only ## if loadBalService : false, service type = "NodePort" else "LoadBalancer" loadBalancer: false + + ## If loadBalancer is enabled, the servicePort is the load balancer port number + ## If loadBalancer is disabled, the servicePort is the NodePort(should be in range 30000-32767) + #servicePort: 30001 + ## Service Annotations (Cloud provider specific), for configuring the service (e.g. private LoadBalancer service) #serviceAnnotations: # service.beta.kubernetes.io/oci-load-balancer-internal: "true" diff --git a/controllers/database/singleinstancedatabase_controller.go b/controllers/database/singleinstancedatabase_controller.go index 128ae2ff..09b54fdb 100644 --- a/controllers/database/singleinstancedatabase_controller.go +++ b/controllers/database/singleinstancedatabase_controller.go @@ -57,6 +57,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/rest" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" @@ -659,7 +660,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns }, }, }, - Ports: []corev1.ContainerPort{{ContainerPort: dbcommons.DEFAULT_LISTENER_PORT}, {ContainerPort: 5500}}, + Ports: []corev1.ContainerPort{{ContainerPort: dbcommons.CONTAINER_LISTENER_PORT}, {ContainerPort: 5500}}, ReadinessProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ @@ -707,7 +708,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns }, { Name: "SVC_PORT", - Value: strconv.Itoa(int(dbcommons.DEFAULT_LISTENER_PORT)), + Value: strconv.Itoa(int(dbcommons.CONTAINER_LISTENER_PORT)), }, { Name: "ORACLE_CHARACTERSET", @@ -728,7 +729,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns }, { Name: "SVC_PORT", - Value: strconv.Itoa(int(dbcommons.DEFAULT_LISTENER_PORT)), + Value: strconv.Itoa(int(dbcommons.CONTAINER_LISTENER_PORT)), }, { Name: "CREATE_PDB", @@ -796,7 +797,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns }, { Name: "SVC_PORT", - Value: strconv.Itoa(int(dbcommons.DEFAULT_LISTENER_PORT)), + Value: strconv.Itoa(int(dbcommons.CONTAINER_LISTENER_PORT)), }, { Name: "ORACLE_SID", @@ -810,7 +811,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns Name: "PRIMARY_DB_CONN_STR", Value: func() string { if dbcommons.IsSourceDatabaseOnCluster(m.Spec.CloneFrom) { - return n.Name + ":" + strconv.Itoa(int(dbcommons.DEFAULT_LISTENER_PORT)) + "/" + n.Spec.Sid + return n.Name + ":" + strconv.Itoa(int(dbcommons.CONTAINER_LISTENER_PORT)) + "/" + n.Spec.Sid } return m.Spec.CloneFrom }(), @@ -875,7 +876,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns // Instantiate Service spec from SingleInstanceDatabase spec //############################################################################# func (r *SingleInstanceDatabaseReconciler) instantiateSVCSpec(m *dbapi.SingleInstanceDatabase, - svcName string, listenerPort int32, svcType corev1.ServiceType) *corev1.Service { + svcName string, ports []corev1.ServicePort, svcType corev1.ServiceType) *corev1.Service { svc := &corev1.Service{ TypeMeta: metav1.TypeMeta{ Kind: "Service", @@ -897,24 +898,14 @@ func (r *SingleInstanceDatabaseReconciler) instantiateSVCSpec(m *dbapi.SingleIns }(), }, Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - { - Name: "listener", - Port: listenerPort, - Protocol: corev1.ProtocolTCP, - }, - { - Name: "xmldb", - Port: 5500, - Protocol: corev1.ProtocolTCP, - }, - }, + Ports: []corev1.ServicePort{}, Selector: map[string]string{ "app": m.Name, }, Type: svcType, }, } + svc.Spec.Ports = ports // Set SingleInstanceDatabase instance as the owner and controller ctrl.SetControllerReference(m, svc, r.Scheme) return svc @@ -1041,129 +1032,144 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplacePVC(ctx context.Contex } //############################################################################# -// Create a Service for SingleInstanceDatabase +// Create Services for SingleInstanceDatabase //############################################################################# func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Context, req ctrl.Request, m *dbapi.SingleInstanceDatabase) (ctrl.Result, error) { log := r.Log.WithValues("createOrReplaceSVC", req.NamespacedName) - /** If TCPS is enabled, there will be two k8s services: - 1. One service exposing the TCPS port for the user to connect, - 2. One service exposing default listerner port inside the cluster only (for ORDS, APEX installation etc) + /** Two k8s services gets created: + 1. One service is ClusterIP service for cluster only communications on the listener port, + 2. One service is NodePort/LoadBalancer (according to the YAML specs) for users to connect **/ - defaultSvc := &corev1.Service{} - tcpsSvc := &corev1.Service{} - // userSvc would indicate the service which is used to connect to the Database by the database user - var userSvc *corev1.Service + // clusterSvc is the cluster-wide service and extSvc is the external service for the users to connect + clusterSvc := &corev1.Service{} + extSvc := &corev1.Service{} - defaultSvcName := m.Name - tcpsSvcName := m.Name + "-tcps" + clusterSvcName := m.Name + extSvcName := m.Name + "-ext" + + // svcPort is the intended port for extSvc taken from singleinstancedatabase YAML file + // If loadBalancer is true, it would be the listener port otherwise it would be node port + svcPort := func() int32 { + if m.Spec.ServicePort != 0 { + return int32(m.Spec.ServicePort) + } else { + if m.Spec.EnableTCPS { + return dbcommons.CONTAINER_TCPS_PORT + } else { + return dbcommons.CONTAINER_LISTENER_PORT + } + } + }() + + // extSvcTargetPort is used to check the target port of the extSvc when TCPS is enabled/disabled + extSvcTargetPort := dbcommons.CONTAINER_LISTENER_PORT + if m.Spec.EnableTCPS { + extSvcTargetPort = dbcommons.CONTAINER_TCPS_PORT + } // Querying for the K8s service resources - getDefaultSvcErr := r.Get(ctx, types.NamespacedName{Name: defaultSvcName, Namespace: m.Namespace}, defaultSvc) - getTcpsSvcErr := r.Get(ctx, types.NamespacedName{Name: tcpsSvcName, Namespace: m.Namespace}, tcpsSvc) + getClusterSvcErr := r.Get(ctx, types.NamespacedName{Name: clusterSvcName, Namespace: m.Namespace}, clusterSvc) + getExtSvcErr := r.Get(ctx, types.NamespacedName{Name: extSvcName, Namespace: m.Namespace}, extSvc) + + if getClusterSvcErr != nil && apierrors.IsNotFound(getClusterSvcErr) { + // Create a new ClusterIP service + ports := []corev1.ServicePort{{Name: "listener", Port: dbcommons.CONTAINER_LISTENER_PORT, Protocol: corev1.ProtocolTCP}} + svc := r.instantiateSVCSpec(m, clusterSvcName, ports, corev1.ServiceType("ClusterIP")) + log.Info("Creating a new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + err := r.Create(ctx, svc) + if err != nil { + log.Error(err, "Failed to create new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + return requeueY, err + } + } else if getClusterSvcErr != nil { + // Error encountered in obtaining the clusterSvc service resource + log.Error(getClusterSvcErr, "Error encountered in obtaining the service", clusterSvcName) + return requeueY, getClusterSvcErr + } - // svcType defines the type of the service as specified in the singleinstancedatabase.yaml file - svcType := corev1.ServiceType("NodePort") + // extSvcType defines the type of the service (LoadBalancer/NodePort) for extSvc as specified in the singleinstancedatabase.yaml file + extSvcType := corev1.ServiceType("NodePort") if m.Spec.LoadBalancer { - svcType = corev1.ServiceType("LoadBalancer") + extSvcType = corev1.ServiceType("LoadBalancer") } - if m.Spec.EnableTCPS { - if getDefaultSvcErr != nil && apierrors.IsNotFound(getDefaultSvcErr) { - // Create a new cluster service - svc := r.instantiateSVCSpec(m, defaultSvcName, dbcommons.DEFAULT_LISTENER_PORT, corev1.ServiceType("ClusterIP")) - log.Info("Creating a new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - err := r.Create(ctx, svc) - if err != nil { - log.Error(err, "Failed to create new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - return requeueY, err - } - } else if defaultSvc.Spec.Type != corev1.ServiceType("ClusterIP") { - // Patch the service type - payload := "{\"spec\": {\"type\": \"ClusterIP\"}}" - //Attempt Service patching - log.Info("Patching the service", "Service.Name", defaultSvc.Name, "Target Type", "ClusterIP") - err := dbcommons.PatchService(r.Config, m.Namespace, ctx, req, defaultSvcName, payload) - if err != nil { - log.Error(err, "Failed to patch Service") - return requeueY, err - } - - } + isExtSvcFound := true - // When TCPS is enabled userSvc would point to tcpsSvc - userSvc = tcpsSvc - if getTcpsSvcErr != nil && apierrors.IsNotFound(getTcpsSvcErr) { - // Reset connect strings whenever service is recreated /* - m.Status.ConnectString = dbcommons.ValueUnavailable - m.Status.PdbConnectString = dbcommons.ValueUnavailable - m.Status.OemExpressUrl = dbcommons.ValueUnavailable - // Create a new service - svc := r.instantiateSVCSpec(m, tcpsSvcName, int32(m.Spec.TcpsPort), svcType) - log.Info("Creating a new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - err := r.Create(ctx, svc) - if err != nil { - log.Error(err, "Failed to create new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - return requeueY, err - } - userSvc = svc - } else if tcpsSvc.Spec.Type != svcType { - // Patch the service type - payload := fmt.Sprintf("{\"spec\": {\"type\": \"%s\"}}", svcType) - //Attempt Service patching - log.Info("Patching the service", "Service.Name", defaultSvc.Name, "Target Type", svcType) - err := dbcommons.PatchService(r.Config, m.Namespace, ctx, req, tcpsSvcName, payload) + if getExtSvcErr != nil && apierrors.IsNotFound(getExtSvcErr) { + isExtSvcFound = false + } else if getExtSvcErr != nil { + // Error encountered in obtaining the extSvc service resource + log.Error(getExtSvcErr, "Error encountered in obtaining the service", extSvcName) + return requeueY, getExtSvcErr + } else { + // extSvc service found + var extSvcPort int32 + if extSvc.Spec.Type == corev1.ServiceType("LoadBalancer") { + extSvcPort = extSvc.Spec.Ports[1].Port + } else if extSvc.Spec.Type == corev1.ServiceType("NodePort") { + extSvcPort = extSvc.Spec.Ports[1].NodePort + } + + if extSvc.Spec.Type != extSvcType || extSvcPort != svcPort || extSvc.Spec.Ports[1].TargetPort.IntVal != extSvcTargetPort { + // Deleting th service + log.Info("Deleting service", "name", extSvcName) + err := r.Delete(ctx, extSvc) if err != nil { - log.Error(err, "Failed to patch Service") - return requeueY, err + r.Log.Error(err, "Failed to delete service", "name", extSvcName) + return requeueN, err } + isExtSvcFound = false } + } - } else { - // Only one service is required if TCPS is not enabled - - // Reset connect strings whenever service is recreated /* + if !isExtSvcFound { + // Reset connect strings whenever extSvc is recreated + m.Status.Status = dbcommons.StatusUpdating m.Status.ConnectString = dbcommons.ValueUnavailable m.Status.PdbConnectString = dbcommons.ValueUnavailable m.Status.OemExpressUrl = dbcommons.ValueUnavailable - // userSvc would point to the defaultSvc - userSvc = defaultSvc - if getDefaultSvcErr != nil && apierrors.IsNotFound(getDefaultSvcErr) { - // Create a new service with - svc := r.instantiateSVCSpec(m, defaultSvcName, dbcommons.DEFAULT_LISTENER_PORT, svcType) - log.Info("Creating a new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - err := r.Create(ctx, svc) - if err != nil { - log.Error(err, "Failed to create new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) - return requeueY, err - } - userSvc = svc - } else if defaultSvc.Spec.Type != svcType { - // Patch the service type - payload := fmt.Sprintf("{\"spec\": {\"type\": \"%s\"}}", svcType) - //Attempt Service patching - log.Info("Patching the service", "Service.Name", defaultSvc.Name, "Target Type", svcType) - err := dbcommons.PatchService(r.Config, m.Namespace, ctx, req, defaultSvcName, payload) - if err != nil { - log.Error(err, "Failed to patch Service") - return requeueY, err - } + // New service has to be created + ports := []corev1.ServicePort{ + { + Name: "xmldb", + Port: 5500, + Protocol: corev1.ProtocolTCP, + }, } - if getTcpsSvcErr == nil { - // Delete this tcps service - log.Info("Deleting service", "name", tcpsSvcName) - err := r.Delete(ctx, tcpsSvc) - if err != nil { - r.Log.Error(err, "Failed to delete service", "name", tcpsSvc.Name) - return requeueN, err + if m.Spec.LoadBalancer { + ports = append(ports, corev1.ServicePort{ + Name: "listener", + Protocol: corev1.ProtocolTCP, + Port: svcPort, + TargetPort: intstr.FromInt(int(extSvcTargetPort)), + }) + } else { + ports = append(ports, corev1.ServicePort{ + Name: "listener", + Protocol: corev1.ProtocolTCP, + Port: extSvcTargetPort, + TargetPort: intstr.FromInt(int(extSvcTargetPort)), + }) + if m.Spec.ServicePort != 0 { + ports[1].NodePort = svcPort } } + // Create the service + svc := r.instantiateSVCSpec(m, extSvcName, ports, extSvcType) + log.Info("Creating a new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + err := r.Create(ctx, svc) + if err != nil { + log.Error(err, "Failed to create new service", "Service.Namespace", svc.Namespace, "Service.Name", svc.Name) + return requeueY, err + } + extSvc = svc } pdbName := strings.ToUpper(m.Spec.Pdbname) @@ -1178,24 +1184,24 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex } if m.Spec.LoadBalancer { - m.Status.ClusterConnectString = userSvc.Name + "." + userSvc.Namespace + ":" + fmt.Sprint(userSvc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) - if len(userSvc.Status.LoadBalancer.Ingress) > 0 { + m.Status.ClusterConnectString = extSvc.Name + "." + extSvc.Namespace + ":" + fmt.Sprint(extSvc.Spec.Ports[1].Port) + "/" + strings.ToUpper(sid) + if len(extSvc.Status.LoadBalancer.Ingress) > 0 { // 'lbAddress' will contain the Fully Qualified Hostname of the LB. If the hostname is not available it will contain the IP address of the LB - lbAddress := userSvc.Status.LoadBalancer.Ingress[0].Hostname + lbAddress := extSvc.Status.LoadBalancer.Ingress[0].Hostname if lbAddress == "" { - lbAddress = userSvc.Status.LoadBalancer.Ingress[0].IP + lbAddress = extSvc.Status.LoadBalancer.Ingress[0].IP } - m.Status.ConnectString = lbAddress + ":" + fmt.Sprint(userSvc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) - m.Status.PdbConnectString = lbAddress + ":" + fmt.Sprint(userSvc.Spec.Ports[0].Port) + "/" + strings.ToUpper(pdbName) - m.Status.OemExpressUrl = "https://" + lbAddress + ":" + fmt.Sprint(userSvc.Spec.Ports[1].Port) + "/em" + m.Status.ConnectString = lbAddress + ":" + fmt.Sprint(extSvc.Spec.Ports[1].Port) + "/" + strings.ToUpper(sid) + m.Status.PdbConnectString = lbAddress + ":" + fmt.Sprint(extSvc.Spec.Ports[1].Port) + "/" + strings.ToUpper(pdbName) + m.Status.OemExpressUrl = "https://" + lbAddress + ":" + fmt.Sprint(extSvc.Spec.Ports[0].Port) + "/em" } } else { - m.Status.ClusterConnectString = userSvc.Name + "." + userSvc.Namespace + ":" + fmt.Sprint(userSvc.Spec.Ports[0].Port) + "/" + strings.ToUpper(sid) + m.Status.ClusterConnectString = extSvc.Name + "." + extSvc.Namespace + ":" + fmt.Sprint(extSvc.Spec.Ports[1].Port) + "/" + strings.ToUpper(sid) nodeip := dbcommons.GetNodeIp(r, ctx, req) if nodeip != "" { - m.Status.ConnectString = nodeip + ":" + fmt.Sprint(userSvc.Spec.Ports[0].NodePort) + "/" + strings.ToUpper(sid) - m.Status.PdbConnectString = nodeip + ":" + fmt.Sprint(userSvc.Spec.Ports[0].NodePort) + "/" + strings.ToUpper(pdbName) - m.Status.OemExpressUrl = "https://" + nodeip + ":" + fmt.Sprint(userSvc.Spec.Ports[1].NodePort) + "/em" + m.Status.ConnectString = nodeip + ":" + fmt.Sprint(extSvc.Spec.Ports[1].NodePort) + "/" + strings.ToUpper(sid) + m.Status.PdbConnectString = nodeip + ":" + fmt.Sprint(extSvc.Spec.Ports[1].NodePort) + "/" + strings.ToUpper(pdbName) + m.Status.OemExpressUrl = "https://" + nodeip + ":" + fmt.Sprint(extSvc.Spec.Ports[0].NodePort) + "/em" } } @@ -1761,6 +1767,50 @@ func (r *SingleInstanceDatabaseReconciler) deleteWallet(m *dbapi.SingleInstanceD return requeueN, nil } +//############################################################################# +// Updating clientWallet when TCPS is enabled +//############################################################################# +func (r *SingleInstanceDatabaseReconciler) updateClientWallet(m *dbapi.SingleInstanceDatabase, + readyPod corev1.Pod, ctx context.Context, req ctrl.Request) error { + // Updation of tnsnames.ora in clientWallet for HOST and PORT fields + extSvc := &corev1.Service{} + extSvcName := m.Name + "-ext" + getExtSvcErr := r.Get(ctx, types.NamespacedName{Name: extSvcName, Namespace: m.Namespace}, extSvc) + + if getExtSvcErr == nil { + var host string + var port int32 + if m.Spec.LoadBalancer { + if len(extSvc.Status.LoadBalancer.Ingress) > 0 { + host = extSvc.Status.LoadBalancer.Ingress[0].Hostname + if host == "" { + host = extSvc.Status.LoadBalancer.Ingress[0].IP + } + port = extSvc.Spec.Ports[1].Port + } + } else { + host = dbcommons.GetNodeIp(r, ctx, req) + if host != "" { + port = extSvc.Spec.Ports[1].NodePort + } + } + if host != "" && host != m.Status.DbHostname && port != int32(m.Status.DbPort) { + _, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", + ctx, req, false, "bash", "-c", fmt.Sprintf(dbcommons.ClientWalletUpdate, host, port)) + if err != nil { + r.Log.Error(err, err.Error()) + return err + } + m.Status.DbHostname = host + m.Status.DbPort = int(port) + } + } else { + r.Log.Info("Unable to get the service while updating the clientWallet", "Service.Namespace", extSvc.Namespace, "Service.Name", extSvcName) + return getExtSvcErr + } + return nil +} + //############################################################################# // Configuring TCPS //############################################################################# @@ -1775,18 +1825,18 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat eventMsg := "Enabling TCPS in the database..." r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) - _, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", - ctx, req, false, "bash", "-c", fmt.Sprintf(dbcommons.EnableTcpsCMD, m.Spec.TcpsPort)) + out, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", + ctx, req, false, "bash", "-c", dbcommons.EnableTcpsCMD) if err != nil { r.Log.Error(err, err.Error()) eventMsg = "Error encountered in enabling TCPS!" r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) return requeueY, nil } + r.Log.Info("enableTcps Output : \n" + out) // Updating the Status and publishing the event m.Status.CertCreationTimestamp = time.Now().Format(time.RFC3339) m.Status.IsTcpsEnabled = true - m.Status.TcpsPort = m.Spec.TcpsPort r.Status().Update(ctx, m) eventMsg = "TCPS Enabled." @@ -1795,6 +1845,12 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat // 26040h = 1085 days futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: func() time.Duration { requeueDuration, _ := time.ParseDuration("26040h"); return requeueDuration }()} + // update clientWallet + err = r.updateClientWallet(m, readyPod, ctx, req) + if err != nil { + r.Log.Error(err, "Error in updating tnsnames.ora in clientWallet...") + return requeueY, nil + } } else if !m.Spec.EnableTCPS && m.Status.IsTcpsEnabled { // Disable TCPS m.Status.Status = dbcommons.StatusUpdating @@ -1803,16 +1859,15 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat eventMsg := "Disabling TCPS in the database..." r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) - _, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", + out, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", ctx, req, false, "bash", "-c", dbcommons.DisableTcpsCMD) if err != nil { r.Log.Error(err, err.Error()) return requeueY, nil } - + r.Log.Info("disable TCPS Output : \n" + out) // Updating the Status and publishing the event m.Status.CertCreationTimestamp = "" - m.Status.TcpsPort = 0 m.Status.IsTcpsEnabled = false r.Status().Update(ctx, m) @@ -1829,12 +1884,13 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat m.Status.Status = dbcommons.StatusUpdating r.Status().Update(ctx, m) - _, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", - ctx, req, false, "bash", "-c", fmt.Sprintf(dbcommons.EnableTcpsCMD, m.Spec.TcpsPort)) + out, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", + ctx, req, false, "bash", "-c", fmt.Sprintf(dbcommons.EnableTcpsCMD)) if err != nil { r.Log.Error(err, err.Error()) return requeueY, nil } + r.Log.Info("Cert Renewal Output : \n" + out) // Updating the Status and publishing the event m.Status.CertCreationTimestamp = time.Now().Format(time.RFC3339) r.Status().Update(ctx, m) @@ -1845,6 +1901,12 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat // 26040h = 1085 days futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: func() time.Duration { requeueDuration, _ := time.ParseDuration("26040h"); return requeueDuration }()} } + // update clientWallet + err := r.updateClientWallet(m, readyPod, ctx, req) + if err != nil { + r.Log.Error(err, "Error in updating tnsnames.ora clientWallet...") + return requeueY, nil + } } return requeueN, nil } From 3a49ff3bef970eec64fd25423a1e6bf16435faa3 Mon Sep 17 00:00:00 2001 From: abhisbyk Date: Tue, 23 Aug 2022 13:59:43 +0530 Subject: [PATCH 16/37] Readme changes for TCPS and custom ports Signed-off-by: abhisbyk --- docs/sidb/README.md | 58 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/docs/sidb/README.md b/docs/sidb/README.md index 6d960194..eb951bf6 100644 --- a/docs/sidb/README.md +++ b/docs/sidb/README.md @@ -21,6 +21,8 @@ Oracle Database Operator for Kubernetes (`OraOperator`) includes the Single Inst * [Advanced Database Configurations](#advanced-database-configurations) * [Run Database with Multiple Replicas](#run-database-with-multiple-replicas) * [Setup Database with LoadBalancer](#setup-database-with-loadbalancer) + * [Enabling TCPS Connections](#enabling-tcps-connections) + * [Specifying Custom Ports](#specifying-custom-ports) * [OracleRestDataService Resource](#oraclerestdataservice-resource) * [REST Enable a Database](#rest-enable-a-database) * [Provision ORDS](#provision-ords) @@ -504,6 +506,62 @@ $ kubectl --type=merge -p '{"spec":{"loadBalancer": true}}' patch singleinstance singleinstancedatabase.database.oracle.com/sidb-sample patched ``` +### Enabling TCPS Connections +You can enable TCPS connections in the database by setting the `enableTCPS` field to `true` in the [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file, and applying it using `kubectl apply` command. + +Alternatively, you can use the following command: +```bash +kubectl patch --type=merge singleinstancedatabases.database.oracle.com sidb-sample -p '{"spec": {"enableTCPS": true}}' +``` + +When TCPS connections are enabled, a Kubernetes event is published notifying the same. This event can be seen by any one of the following commands: +```bash +kubectl describe singleinstancedatabases.database.oracle.com sidb-sample + +kubectl get events +``` + +Once TCPS connections are enabled, the database connect string will change accordingly. The TCPS connections status can also be queried by the following command: +```bash +kubectl get singleinstancedatabase sidb-sample -o "jsonpath={.status.isTcpsEnabled}" +true +``` + +The following steps are required to connect the Database using TCPS: +- You need to download the wallet from the Persistent Volume(PV) attached with the database pod. You can use the following command to get the list of pod: + ```bash + kubectl get po + NAME READY STATUS RESTARTS AGE + sidb-sample-gaqoe 1/1 Running 0 3d14h + ``` +- The location of the wallet inside the pod is as `/opt/oracle/oradata/clientWallet/$ORACLE_SID`. **Let us assume the `ORACLE_SID` is `ORCL1` for the upcoming example commands**. The sample command to download the wallet is as follows: + ```bash + kubectl cp sidb-sample-gaqoe:/opt/oracle/oradata/clientWallet/ORCL1 + ``` +- This wallet includes the sample `tnsnames.ora` and `sqlnet.ora` files. All the TNS entries for the database (corresponding to the CDB and PDB) resides in `tnsnames.ora` file. You need to go inside the downloaded wallet directory and set the `TNS_ADMIN` environment variable to point to the current directory as follows: + ```bash + # After going inside the downloaded wallet directory + export TNS_ADMIN=$(pwd) + ``` + After this, you can connect using SQL\*Plus using the following sample commands: + ```bash + sqlplus sys@ORCL1 as sysdba + + sqlplus system@ORCL1 + ``` +**NOTE:** +- Only database server authentication is supported (no mTLS). +- When TCPS is enabled, a self-signed certificate is generated and stored inside the wallets. For users' convenience, a client-side wallet is generated and stored at `/opt/oracle/oradata/clientWallet/$ORACLE_SID` location. +- The self-signed certificate used with TCPS has validity for 3 years. After the certificate is expired, it will be renewed by the `OraOperator` automatically. You need to download the wallet again after the auto-renewal. + +### Specifying Custom Ports +As mentioned in the section [Setup Database with LoadBalancer](#setup-database-with-loadbalancer), there are two kubernetes services possible for the database: NodePort and LoadBalancer. You can specify which port to use with these services by editing the `servicePort` field of the [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file. + +If the `LoadBalancer` is enabled, the `servicePort` will be the opened load balancer port for database connections. + +In case of `NodePort` service, the `servicePort` will be the opened port on the Kubernetes nodes for database connections. In this case, the allowed range for the `servicePort` is 30000-32767. + + ## OracleRestDataService Resource The Oracle Database Operator creates the `OracleRestDataService` as a custom resource. We will refer `OracleRestDataService` as ORDS from now onwards. Creating ORDS as a custom resource enables the RESTful API access to the Oracle Database in K8s and enables it to be managed as a native Kubernetes object. From 8e59edbd482b8ea529e3653b1d3d18ae2e032c90 Mon Sep 17 00:00:00 2001 From: abhisbyk Date: Mon, 29 Aug 2022 13:01:26 +0530 Subject: [PATCH 17/37] Configurable renewCertDuration and Nodeport svc abrupt deletion fix Signed-off-by: abhisbyk --- .../v1alpha1/singleinstancedatabase_types.go | 4 +- .../singleinstancedatabase_webhook.go | 21 ++++++++ ...se.oracle.com_singleinstancedatabases.yaml | 8 +-- .../samples/sidb/singleinstancedatabase.yaml | 4 ++ .../singleinstancedatabase_controller.go | 50 +++++++++++-------- 5 files changed, 60 insertions(+), 27 deletions(-) diff --git a/apis/database/v1alpha1/singleinstancedatabase_types.go b/apis/database/v1alpha1/singleinstancedatabase_types.go index d5b26e78..a65ed9dd 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_types.go +++ b/apis/database/v1alpha1/singleinstancedatabase_types.go @@ -67,6 +67,7 @@ type SingleInstanceDatabaseSpec struct { ArchiveLog bool `json:"archiveLog,omitempty"` ForceLogging bool `json:"forceLog,omitempty"` EnableTCPS bool `json:"enableTCPS,omitempty"` + CertRenewDuration string `json:"certRenewDuration,omitempty"` CloneFrom string `json:"cloneFrom,omitempty"` ReadinessCheckPeriod int `json:"readinessCheckPeriod,omitempty"` @@ -151,8 +152,7 @@ type SingleInstanceDatabaseStatus struct { // +kubebuilder:default:=false IsTcpsEnabled bool `json:"isTcpsEnabled"` CertCreationTimestamp string `json:"certCreationTimestamp,omitempty"` - DbHostname string `json:"dbHostname,omitempty"` - DbPort int `json:"dbPort,omitempty"` + CertRenewDuration string `json:"certRenewDuration,omitempty"` // +patchMergeKey=type // +patchStrategy=merge diff --git a/apis/database/v1alpha1/singleinstancedatabase_webhook.go b/apis/database/v1alpha1/singleinstancedatabase_webhook.go index 0d450871..0d76ee74 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_webhook.go +++ b/apis/database/v1alpha1/singleinstancedatabase_webhook.go @@ -40,6 +40,7 @@ package v1alpha1 import ( "strings" + "time" dbcommons "github.com/oracle/oracle-database-operator/commons/database" @@ -244,6 +245,26 @@ func (r *SingleInstanceDatabase) ValidateCreate() error { } } + // Certificate Renew Duration Validation + if r.Spec.CertRenewDuration != "" { + duration, err := time.ParseDuration(r.Spec.CertRenewDuration) + if err != nil { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec").Child("certRenewDuration"), r.Spec.CertRenewDuration, + "Please provide valid string to parse the certRenewDuration.")) + } + maxLimit, _ := time.ParseDuration("26000h") + minLimit, _ := time.ParseDuration("1m") + if duration > maxLimit || duration < minLimit { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec").Child("certRenewDuration"), r.Spec.CertRenewDuration, + "Please specify certRenewDuration in the range: 1m to 26000h")) + } + } else { + // Setting the default value + r.Spec.CertRenewDuration = "26000h" + } + if len(allErrs) == 0 { return nil } diff --git a/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml b/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml index b8383ef8..fbbe19ca 100644 --- a/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml +++ b/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml @@ -77,6 +77,8 @@ spec: type: object archiveLog: type: boolean + certRenewDuration: + type: string charset: type: string cloneFrom: @@ -177,6 +179,8 @@ spec: type: string certCreationTimestamp: type: string + certRenewDuration: + type: string charset: type: string cloneFrom: @@ -263,10 +267,6 @@ spec: datafilesPatched: default: "false" type: string - dbHostname: - type: string - dbPort: - type: integer edition: type: string flashBack: diff --git a/config/samples/sidb/singleinstancedatabase.yaml b/config/samples/sidb/singleinstancedatabase.yaml index 9adc9601..f861b31c 100644 --- a/config/samples/sidb/singleinstancedatabase.yaml +++ b/config/samples/sidb/singleinstancedatabase.yaml @@ -47,6 +47,10 @@ spec: ## Enable TCPS enableTCPS: false + ## Certificate Renewal Duration: The time after which certificates will be renewed if TCPS connections are enabled; can be in hours(h), minutes(m) and seconds(s) + ## Maximum value is 26000h + #certRenewDuration: 26000h + ## NA if cloning from a SourceDB (cloneFrom is set) ## Specify both sgaSize and pgaSize (in MB) or dont specify both ## Specify Non-Zero value to use diff --git a/controllers/database/singleinstancedatabase_controller.go b/controllers/database/singleinstancedatabase_controller.go index 09b54fdb..67402b76 100644 --- a/controllers/database/singleinstancedatabase_controller.go +++ b/controllers/database/singleinstancedatabase_controller.go @@ -197,22 +197,22 @@ func (r *SingleInstanceDatabaseReconciler) Reconcile(ctx context.Context, req ct } } - // Configure TCPS - result, err = r.configTcps(singleInstanceDatabase, readyPod, ctx, req) + // Update DB config + result, err = r.updateDBConfig(singleInstanceDatabase, readyPod, ctx, req) if result.Requeue { r.Log.Info("Reconcile queued") return result, nil } - // Update DB config - result, err = r.updateDBConfig(singleInstanceDatabase, readyPod, ctx, req) + // Update Init Parameters + result, err = r.updateInitParameters(singleInstanceDatabase, readyPod, ctx, req) if result.Requeue { r.Log.Info("Reconcile queued") return result, nil } - // Update Init Parameters - result, err = r.updateInitParameters(singleInstanceDatabase, readyPod, ctx, req) + // Configure TCPS + result, err = r.configTcps(singleInstanceDatabase, readyPod, ctx, req) if result.Requeue { r.Log.Info("Reconcile queued") return result, nil @@ -1114,7 +1114,7 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex extSvcPort = extSvc.Spec.Ports[1].NodePort } - if extSvc.Spec.Type != extSvcType || extSvcPort != svcPort || extSvc.Spec.Ports[1].TargetPort.IntVal != extSvcTargetPort { + if extSvc.Spec.Type != extSvcType || (m.Spec.ServicePort != 0 && extSvcPort != svcPort) || extSvc.Spec.Ports[1].TargetPort.IntVal != extSvcTargetPort { // Deleting th service log.Info("Deleting service", "name", extSvcName) err := r.Delete(ctx, extSvc) @@ -1794,16 +1794,15 @@ func (r *SingleInstanceDatabaseReconciler) updateClientWallet(m *dbapi.SingleIns port = extSvc.Spec.Ports[1].NodePort } } - if host != "" && host != m.Status.DbHostname && port != int32(m.Status.DbPort) { - _, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", - ctx, req, false, "bash", "-c", fmt.Sprintf(dbcommons.ClientWalletUpdate, host, port)) - if err != nil { - r.Log.Error(err, err.Error()) - return err - } - m.Status.DbHostname = host - m.Status.DbPort = int(port) + + r.Log.Info("Updating the client wallet...") + _, err := dbcommons.ExecCommand(r, r.Config, readyPod.Name, readyPod.Namespace, "", + ctx, req, false, "bash", "-c", fmt.Sprintf(dbcommons.ClientWalletUpdate, host, port)) + if err != nil { + r.Log.Error(err, err.Error()) + return err } + } else { r.Log.Info("Unable to get the service while updating the clientWallet", "Service.Namespace", extSvc.Namespace, "Service.Name", extSvcName) return getExtSvcErr @@ -1842,8 +1841,9 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat eventMsg = "TCPS Enabled." r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) - // 26040h = 1085 days - futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: func() time.Duration { requeueDuration, _ := time.ParseDuration("26040h"); return requeueDuration }()} + requeueDuration, _ := time.ParseDuration(m.Spec.CertRenewDuration) + requeueDuration += func() time.Duration { requeueDuration, _ := time.ParseDuration("1s"); return requeueDuration }() + futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: requeueDuration} // update clientWallet err = r.updateClientWallet(m, readyPod, ctx, req) @@ -1879,7 +1879,7 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat // Certificates are renewed when 10 days remain for certs expiry certCreationTimestamp, _ := time.Parse(time.RFC3339, m.Status.CertCreationTimestamp) duration := time.Since(certCreationTimestamp) - allowdDuration, _ := time.ParseDuration("26000h") + allowdDuration, _ := time.ParseDuration(m.Spec.CertRenewDuration) if duration > allowdDuration { m.Status.Status = dbcommons.StatusUpdating r.Status().Update(ctx, m) @@ -1898,8 +1898,16 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat eventMsg := "TCPS Certificates Renewed at time %s," r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg, time.Now().Format(time.RFC3339)) - // 26040h = 1085 days - futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: func() time.Duration { requeueDuration, _ := time.ParseDuration("26040h"); return requeueDuration }()} + requeueDuration, _ := time.ParseDuration(m.Spec.CertRenewDuration) + requeueDuration += func() time.Duration { requeueDuration, _ := time.ParseDuration("1s"); return requeueDuration }() + futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: requeueDuration} + } + if m.Status.CertRenewDuration != m.Spec.CertRenewDuration { + requeueDuration, _ := time.ParseDuration(m.Spec.CertRenewDuration) + requeueDuration += func() time.Duration { requeueDuration, _ := time.ParseDuration("1s"); return requeueDuration }() + futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: requeueDuration} + + m.Status.CertRenewDuration = m.Spec.CertRenewDuration } // update clientWallet err := r.updateClientWallet(m, readyPod, ctx, req) From 04765b2133cd52f4e0d189e3112c1dac6e9e2220 Mon Sep 17 00:00:00 2001 From: abhisbyk Date: Mon, 29 Aug 2022 13:33:07 +0530 Subject: [PATCH 18/37] Cert renewal duration is changed to 26280h (3 yrs) Signed-off-by: abhisbyk --- apis/database/v1alpha1/singleinstancedatabase_webhook.go | 6 +++--- config/samples/sidb/singleinstancedatabase.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/apis/database/v1alpha1/singleinstancedatabase_webhook.go b/apis/database/v1alpha1/singleinstancedatabase_webhook.go index 0d76ee74..8dd0b250 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_webhook.go +++ b/apis/database/v1alpha1/singleinstancedatabase_webhook.go @@ -253,16 +253,16 @@ func (r *SingleInstanceDatabase) ValidateCreate() error { field.Invalid(field.NewPath("spec").Child("certRenewDuration"), r.Spec.CertRenewDuration, "Please provide valid string to parse the certRenewDuration.")) } - maxLimit, _ := time.ParseDuration("26000h") + maxLimit, _ := time.ParseDuration("26280h") minLimit, _ := time.ParseDuration("1m") if duration > maxLimit || duration < minLimit { allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("certRenewDuration"), r.Spec.CertRenewDuration, - "Please specify certRenewDuration in the range: 1m to 26000h")) + "Please specify certRenewDuration in the range: 1m to 26280h")) } } else { // Setting the default value - r.Spec.CertRenewDuration = "26000h" + r.Spec.CertRenewDuration = "26280h" } if len(allErrs) == 0 { diff --git a/config/samples/sidb/singleinstancedatabase.yaml b/config/samples/sidb/singleinstancedatabase.yaml index f861b31c..739779d8 100644 --- a/config/samples/sidb/singleinstancedatabase.yaml +++ b/config/samples/sidb/singleinstancedatabase.yaml @@ -48,8 +48,8 @@ spec: enableTCPS: false ## Certificate Renewal Duration: The time after which certificates will be renewed if TCPS connections are enabled; can be in hours(h), minutes(m) and seconds(s) - ## Maximum value is 26000h - #certRenewDuration: 26000h + ## Maximum value is 26280h (3 years), Minimum value is 1m + #certRenewDuration: 26280h ## NA if cloning from a SourceDB (cloneFrom is set) ## Specify both sgaSize and pgaSize (in MB) or dont specify both From d45dc52b4a006a70623c79e48fce9729a1d54de1 Mon Sep 17 00:00:00 2001 From: abhisbyk Date: Mon, 29 Aug 2022 14:11:39 +0530 Subject: [PATCH 19/37] Disabling the cert renewal logic if tcpsCertRenewInterval is not given in the yaml file, default 2 years Signed-off-by: abhisbyk --- .../v1alpha1/singleinstancedatabase_types.go | 22 +++--- .../singleinstancedatabase_webhook.go | 16 ++--- .../samples/sidb/singleinstancedatabase.yaml | 6 +- .../sidb/singleinstancedatabase_tcps.yaml | 68 +++++++++++++++++++ .../singleinstancedatabase_controller.go | 15 ++-- docs/sidb/README.md | 4 +- 6 files changed, 97 insertions(+), 34 deletions(-) create mode 100644 config/samples/sidb/singleinstancedatabase_tcps.yaml diff --git a/apis/database/v1alpha1/singleinstancedatabase_types.go b/apis/database/v1alpha1/singleinstancedatabase_types.go index a65ed9dd..3eb64d07 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_types.go +++ b/apis/database/v1alpha1/singleinstancedatabase_types.go @@ -57,17 +57,17 @@ type SingleInstanceDatabaseSpec struct { // +k8s:openapi-gen=true // +kubebuilder:validation:Pattern=`^[a-zA-Z0-9]+$` // +kubebuilder:validation:MaxLength:=12 - Sid string `json:"sid,omitempty"` - Charset string `json:"charset,omitempty"` - Pdbname string `json:"pdbName,omitempty"` - LoadBalancer bool `json:"loadBalancer,omitempty"` - ServicePort int `json:"servicePort,omitempty"` - ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` - FlashBack bool `json:"flashBack,omitempty"` - ArchiveLog bool `json:"archiveLog,omitempty"` - ForceLogging bool `json:"forceLog,omitempty"` - EnableTCPS bool `json:"enableTCPS,omitempty"` - CertRenewDuration string `json:"certRenewDuration,omitempty"` + Sid string `json:"sid,omitempty"` + Charset string `json:"charset,omitempty"` + Pdbname string `json:"pdbName,omitempty"` + LoadBalancer bool `json:"loadBalancer,omitempty"` + ServicePort int `json:"servicePort,omitempty"` + ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` + FlashBack bool `json:"flashBack,omitempty"` + ArchiveLog bool `json:"archiveLog,omitempty"` + ForceLogging bool `json:"forceLog,omitempty"` + EnableTCPS bool `json:"enableTCPS,omitempty"` + TcpsCertRenewInterval string `json:"tcpsCertRenewInterval,omitempty"` CloneFrom string `json:"cloneFrom,omitempty"` ReadinessCheckPeriod int `json:"readinessCheckPeriod,omitempty"` diff --git a/apis/database/v1alpha1/singleinstancedatabase_webhook.go b/apis/database/v1alpha1/singleinstancedatabase_webhook.go index 8dd0b250..8a269747 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_webhook.go +++ b/apis/database/v1alpha1/singleinstancedatabase_webhook.go @@ -246,25 +246,21 @@ func (r *SingleInstanceDatabase) ValidateCreate() error { } // Certificate Renew Duration Validation - if r.Spec.CertRenewDuration != "" { - duration, err := time.ParseDuration(r.Spec.CertRenewDuration) + if r.Spec.TcpsCertRenewInterval != "" { + duration, err := time.ParseDuration(r.Spec.TcpsCertRenewInterval) if err != nil { allErrs = append(allErrs, - field.Invalid(field.NewPath("spec").Child("certRenewDuration"), r.Spec.CertRenewDuration, - "Please provide valid string to parse the certRenewDuration.")) + field.Invalid(field.NewPath("spec").Child("tcpsCertRenewInterval"), r.Spec.TcpsCertRenewInterval, + "Please provide valid string to parse the tcpsCertRenewInterval.")) } maxLimit, _ := time.ParseDuration("26280h") minLimit, _ := time.ParseDuration("1m") if duration > maxLimit || duration < minLimit { allErrs = append(allErrs, - field.Invalid(field.NewPath("spec").Child("certRenewDuration"), r.Spec.CertRenewDuration, - "Please specify certRenewDuration in the range: 1m to 26280h")) + field.Invalid(field.NewPath("spec").Child("tcpsCertRenewInterval"), r.Spec.TcpsCertRenewInterval, + "Please specify tcpsCertRenewInterval in the range: 1m to 26280h")) } - } else { - // Setting the default value - r.Spec.CertRenewDuration = "26280h" } - if len(allErrs) == 0 { return nil } diff --git a/config/samples/sidb/singleinstancedatabase.yaml b/config/samples/sidb/singleinstancedatabase.yaml index 739779d8..edb56e0a 100644 --- a/config/samples/sidb/singleinstancedatabase.yaml +++ b/config/samples/sidb/singleinstancedatabase.yaml @@ -47,9 +47,9 @@ spec: ## Enable TCPS enableTCPS: false - ## Certificate Renewal Duration: The time after which certificates will be renewed if TCPS connections are enabled; can be in hours(h), minutes(m) and seconds(s) - ## Maximum value is 26280h (3 years), Minimum value is 1m - #certRenewDuration: 26280h + ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled; can be in hours(h), minutes(m) and seconds(s) + ## Maximum value is 26280h (3 years), Minimum value is 1m; Default value is 17520h (2 years) + certRenewDuration: 17520h ## NA if cloning from a SourceDB (cloneFrom is set) ## Specify both sgaSize and pgaSize (in MB) or dont specify both diff --git a/config/samples/sidb/singleinstancedatabase_tcps.yaml b/config/samples/sidb/singleinstancedatabase_tcps.yaml new file mode 100644 index 00000000..67e9a9bf --- /dev/null +++ b/config/samples/sidb/singleinstancedatabase_tcps.yaml @@ -0,0 +1,68 @@ +# +# Copyright (c) 2022, Oracle and/or its affiliates. +# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. +# + +apiVersion: v1 +kind: Secret +metadata: + name: db-admin-secret + namespace: default +type: Opaque +stringData: + # Specify your DB password here + oracle_pwd: + +--- + +apiVersion: database.oracle.com/v1alpha1 +kind: SingleInstanceDatabase +metadata: + # Creates base sidb-sample. Use singleinstancedatabase_clone.yaml for cloning + # and singleinstancedatabase_patch.yaml for patching + name: sidb-sample + namespace: default +spec: + + ## Use only alphanumeric characters for sid + sid: ORCL1 + + ## DB edition. + edition: enterprise + + ## Secret containing SIDB password mapped to secretKey + adminPassword: + secretName: db-admin-secret + + ## DB character set + charset: AL32UTF8 + + ## PDB name + pdbName: orclpdb1 + + ## Enable/Disable ArchiveLog. Should be true to allow DB cloning + archiveLog: true + + ## Enable TCPS + enableTCPS: true + + ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled; can be in hours(h), minutes(m) and seconds(s) + ## Maximum value is 26280h (3 years), Minimum value is 1m; Default value is 17520h (2 years) + certRenewDuration: 17520h + + ## Database image details + image: + pullFrom: container-registry.oracle.com/database/enterprise:latest + pullSecrets: oracle-container-registry-secret + + ## size is the required minimum size of the persistent volume + ## storageClass is specified for automatic volume provisioning + ## accessMode can only accept one of ReadWriteOnce, ReadWriteMany + persistence: + size: 100Gi + ## oci-bv applies to OCI block volumes. Use "standard" storageClass for dynamic provisioning in Minikube. Update as appropriate for other cloud service providers + storageClass: "oci-bv" + accessMode: "ReadWriteOnce" + + ## Count of Database Pods. + replicas: 1 diff --git a/controllers/database/singleinstancedatabase_controller.go b/controllers/database/singleinstancedatabase_controller.go index 67402b76..af89e906 100644 --- a/controllers/database/singleinstancedatabase_controller.go +++ b/controllers/database/singleinstancedatabase_controller.go @@ -1841,7 +1841,7 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat eventMsg = "TCPS Enabled." r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) - requeueDuration, _ := time.ParseDuration(m.Spec.CertRenewDuration) + requeueDuration, _ := time.ParseDuration(m.Spec.TcpsCertRenewInterval) requeueDuration += func() time.Duration { requeueDuration, _ := time.ParseDuration("1s"); return requeueDuration }() futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: requeueDuration} @@ -1874,12 +1874,11 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat eventMsg = "TCPS Disabled." r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) - } else if m.Spec.EnableTCPS && m.Status.IsTcpsEnabled { + } else if m.Spec.EnableTCPS && m.Status.IsTcpsEnabled && m.Spec.TcpsCertRenewInterval != "" { // Cert Renewal Logic - // Certificates are renewed when 10 days remain for certs expiry certCreationTimestamp, _ := time.Parse(time.RFC3339, m.Status.CertCreationTimestamp) duration := time.Since(certCreationTimestamp) - allowdDuration, _ := time.ParseDuration(m.Spec.CertRenewDuration) + allowdDuration, _ := time.ParseDuration(m.Spec.TcpsCertRenewInterval) if duration > allowdDuration { m.Status.Status = dbcommons.StatusUpdating r.Status().Update(ctx, m) @@ -1898,16 +1897,16 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat eventMsg := "TCPS Certificates Renewed at time %s," r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg, time.Now().Format(time.RFC3339)) - requeueDuration, _ := time.ParseDuration(m.Spec.CertRenewDuration) + requeueDuration, _ := time.ParseDuration(m.Spec.TcpsCertRenewInterval) requeueDuration += func() time.Duration { requeueDuration, _ := time.ParseDuration("1s"); return requeueDuration }() futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: requeueDuration} } - if m.Status.CertRenewDuration != m.Spec.CertRenewDuration { - requeueDuration, _ := time.ParseDuration(m.Spec.CertRenewDuration) + if m.Status.CertRenewDuration != m.Spec.TcpsCertRenewInterval { + requeueDuration, _ := time.ParseDuration(m.Spec.TcpsCertRenewInterval) requeueDuration += func() time.Duration { requeueDuration, _ := time.ParseDuration("1s"); return requeueDuration }() futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: requeueDuration} - m.Status.CertRenewDuration = m.Spec.CertRenewDuration + m.Status.CertRenewDuration = m.Spec.TcpsCertRenewInterval } // update clientWallet err := r.updateClientWallet(m, readyPod, ctx, req) diff --git a/docs/sidb/README.md b/docs/sidb/README.md index eb951bf6..f3c5af1d 100644 --- a/docs/sidb/README.md +++ b/docs/sidb/README.md @@ -551,8 +551,8 @@ The following steps are required to connect the Database using TCPS: ``` **NOTE:** - Only database server authentication is supported (no mTLS). -- When TCPS is enabled, a self-signed certificate is generated and stored inside the wallets. For users' convenience, a client-side wallet is generated and stored at `/opt/oracle/oradata/clientWallet/$ORACLE_SID` location. -- The self-signed certificate used with TCPS has validity for 3 years. After the certificate is expired, it will be renewed by the `OraOperator` automatically. You need to download the wallet again after the auto-renewal. +- When TCPS is enabled, a self-signed certificate is generated and stored inside the wallets. For users' convenience, a client-side wallet is generated and stored at `/opt/oracle/oradata/clientWallet/$ORACLE_SID` location in the pod. +- The self-signed certificate used with TCPS has validity for 2 years. After the certificate is expired, it will be renewed by the `OraOperator` automatically. You need to download the wallet again after the auto-renewal. ### Specifying Custom Ports As mentioned in the section [Setup Database with LoadBalancer](#setup-database-with-loadbalancer), there are two kubernetes services possible for the database: NodePort and LoadBalancer. You can specify which port to use with these services by editing the `servicePort` field of the [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file. From 5af166beac850a7a93ea7e1817379b22b24aff64 Mon Sep 17 00:00:00 2001 From: abhisbyk Date: Mon, 29 Aug 2022 15:44:17 +0530 Subject: [PATCH 20/37] Updating attr name in the yaml file Signed-off-by: abhisbyk --- config/samples/sidb/singleinstancedatabase.yaml | 2 +- config/samples/sidb/singleinstancedatabase_tcps.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/samples/sidb/singleinstancedatabase.yaml b/config/samples/sidb/singleinstancedatabase.yaml index edb56e0a..efcbcee1 100644 --- a/config/samples/sidb/singleinstancedatabase.yaml +++ b/config/samples/sidb/singleinstancedatabase.yaml @@ -49,7 +49,7 @@ spec: ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled; can be in hours(h), minutes(m) and seconds(s) ## Maximum value is 26280h (3 years), Minimum value is 1m; Default value is 17520h (2 years) - certRenewDuration: 17520h + tcpsCertRenewInterval: 17520h ## NA if cloning from a SourceDB (cloneFrom is set) ## Specify both sgaSize and pgaSize (in MB) or dont specify both diff --git a/config/samples/sidb/singleinstancedatabase_tcps.yaml b/config/samples/sidb/singleinstancedatabase_tcps.yaml index 67e9a9bf..22d3695a 100644 --- a/config/samples/sidb/singleinstancedatabase_tcps.yaml +++ b/config/samples/sidb/singleinstancedatabase_tcps.yaml @@ -48,7 +48,7 @@ spec: ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled; can be in hours(h), minutes(m) and seconds(s) ## Maximum value is 26280h (3 years), Minimum value is 1m; Default value is 17520h (2 years) - certRenewDuration: 17520h + tcpsCertRenewInterval: 17520h ## Database image details image: From dbf8f5f5bb95dbfcf0868c9830844b0519040972 Mon Sep 17 00:00:00 2001 From: abhisbyk Date: Mon, 29 Aug 2022 16:01:34 +0530 Subject: [PATCH 21/37] Yaml file comment corrections Signed-off-by: abhisbyk --- config/samples/sidb/singleinstancedatabase.yaml | 4 +++- config/samples/sidb/singleinstancedatabase_tcps.yaml | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/config/samples/sidb/singleinstancedatabase.yaml b/config/samples/sidb/singleinstancedatabase.yaml index efcbcee1..2a3c9cae 100644 --- a/config/samples/sidb/singleinstancedatabase.yaml +++ b/config/samples/sidb/singleinstancedatabase.yaml @@ -47,8 +47,10 @@ spec: ## Enable TCPS enableTCPS: false - ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled; can be in hours(h), minutes(m) and seconds(s) + ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled. + ## tcpsCertRenewInterval can be in hours(h), minutes(m) and seconds(s); e.g. 17520h, 8760h etc. ## Maximum value is 26280h (3 years), Minimum value is 1m; Default value is 17520h (2 years) + ## If this field is commented out/removed from the yaml, it will disable the auto-renewal feature for TCPS certificate tcpsCertRenewInterval: 17520h ## NA if cloning from a SourceDB (cloneFrom is set) diff --git a/config/samples/sidb/singleinstancedatabase_tcps.yaml b/config/samples/sidb/singleinstancedatabase_tcps.yaml index 22d3695a..8089c85c 100644 --- a/config/samples/sidb/singleinstancedatabase_tcps.yaml +++ b/config/samples/sidb/singleinstancedatabase_tcps.yaml @@ -46,8 +46,10 @@ spec: ## Enable TCPS enableTCPS: true - ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled; can be in hours(h), minutes(m) and seconds(s) + ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled. + ## tcpsCertRenewInterval can be in hours(h), minutes(m) and seconds(s); e.g. 17520h, 8760h etc. ## Maximum value is 26280h (3 years), Minimum value is 1m; Default value is 17520h (2 years) + ## If this field is commented out/removed from the yaml, it will disable the auto-renewal feature for TCPS certificate tcpsCertRenewInterval: 17520h ## Database image details From 4655cf5cf09cafc833cf500674ee68af5dd3e7d8 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 1 Sep 2022 12:45:49 +0000 Subject: [PATCH 22/37] Additional TCPS features --- .../v1alpha1/singleinstancedatabase_types.go | 42 +++-- .../singleinstancedatabase_webhook.go | 18 +- commons/database/constants.go | 3 + ...se.oracle.com_singleinstancedatabases.yaml | 25 ++- .../samples/sidb/singleinstancedatabase.yaml | 10 +- .../oraclerestdataservice_controller.go | 3 +- .../singleinstancedatabase_controller.go | 162 +++++++++++++----- docs/sidb/README.md | 13 +- 8 files changed, 202 insertions(+), 74 deletions(-) diff --git a/apis/database/v1alpha1/singleinstancedatabase_types.go b/apis/database/v1alpha1/singleinstancedatabase_types.go index 3eb64d07..7edd9af8 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_types.go +++ b/apis/database/v1alpha1/singleinstancedatabase_types.go @@ -61,7 +61,8 @@ type SingleInstanceDatabaseSpec struct { Charset string `json:"charset,omitempty"` Pdbname string `json:"pdbName,omitempty"` LoadBalancer bool `json:"loadBalancer,omitempty"` - ServicePort int `json:"servicePort,omitempty"` + ListenerPort int `json:"listenerPort,omitempty"` + TcpsListenerPort int `json:"tcpsListenerPort,omitempty"` ServiceAnnotations map[string]string `json:"serviceAnnotations,omitempty"` FlashBack bool `json:"flashBack,omitempty"` ArchiveLog bool `json:"archiveLog,omitempty"` @@ -131,28 +132,31 @@ type SingleInstanceDatabaseStatus struct { DatafilesPatched string `json:"datafilesPatched,omitempty"` ConnectString string `json:"connectString,omitempty"` ClusterConnectString string `json:"clusterConnectString,omitempty"` + TcpsConnectString string `json:"tcpsConnectString,omitempty"` StandbyDatabases map[string]string `json:"standbyDatabases,omitempty"` // +kubebuilder:default:="false" - DatafilesCreated string `json:"datafilesCreated,omitempty"` - Sid string `json:"sid,omitempty"` - Edition string `json:"edition,omitempty"` - Charset string `json:"charset,omitempty"` - Pdbname string `json:"pdbName,omitempty"` - InitSgaSize int `json:"initSgaSize,omitempty"` - InitPgaSize int `json:"initPgaSize,omitempty"` - CloneFrom string `json:"cloneFrom,omitempty"` - FlashBack string `json:"flashBack,omitempty"` - ArchiveLog string `json:"archiveLog,omitempty"` - ForceLogging string `json:"forceLog,omitempty"` - OemExpressUrl string `json:"oemExpressUrl,omitempty"` - OrdsReference string `json:"ordsReference,omitempty"` - PdbConnectString string `json:"pdbConnectString,omitempty"` - ApexInstalled bool `json:"apexInstalled,omitempty"` - PrebuiltDB bool `json:"prebuiltDB,omitempty"` + DatafilesCreated string `json:"datafilesCreated,omitempty"` + Sid string `json:"sid,omitempty"` + Edition string `json:"edition,omitempty"` + Charset string `json:"charset,omitempty"` + Pdbname string `json:"pdbName,omitempty"` + InitSgaSize int `json:"initSgaSize,omitempty"` + InitPgaSize int `json:"initPgaSize,omitempty"` + CloneFrom string `json:"cloneFrom,omitempty"` + FlashBack string `json:"flashBack,omitempty"` + ArchiveLog string `json:"archiveLog,omitempty"` + ForceLogging string `json:"forceLog,omitempty"` + OemExpressUrl string `json:"oemExpressUrl,omitempty"` + OrdsReference string `json:"ordsReference,omitempty"` + PdbConnectString string `json:"pdbConnectString,omitempty"` + TcpsPdbConnectString string `json:"tcpsPdbConnectString,omitempty"` + ApexInstalled bool `json:"apexInstalled,omitempty"` + PrebuiltDB bool `json:"prebuiltDB,omitempty"` // +kubebuilder:default:=false IsTcpsEnabled bool `json:"isTcpsEnabled"` CertCreationTimestamp string `json:"certCreationTimestamp,omitempty"` - CertRenewDuration string `json:"certRenewDuration,omitempty"` + CertRenewInterval string `json:"certRenewInterval,omitempty"` + ClientWalletLoc string `json:"clientWalletLoc,omitempty"` // +patchMergeKey=type // +patchStrategy=merge @@ -172,6 +176,8 @@ type SingleInstanceDatabaseStatus struct { // +kubebuilder:printcolumn:JSONPath=".status.role",name="Role",type="string",priority=1 // +kubebuilder:printcolumn:JSONPath=".status.releaseUpdate",name="Version",type="string" // +kubebuilder:printcolumn:JSONPath=".status.connectString",name="Connect Str",type="string" +// +kubebuilder:printcolumn:JSONPath=".status.tcpsConnectString",name="TCPS Connect Str",type="string" +// +kubebuilder:printcolumn:JSONPath=".status.tcpsPdbConnectString",name="TCPS Pdb Connect Str",type="string", priority=1 // +kubebuilder:printcolumn:JSONPath=".status.pdbConnectString",name="Pdb Connect Str",type="string",priority=1 // +kubebuilder:printcolumn:JSONPath=".status.oemExpressUrl",name="Oem Express Url",type="string" diff --git a/apis/database/v1alpha1/singleinstancedatabase_webhook.go b/apis/database/v1alpha1/singleinstancedatabase_webhook.go index 8a269747..af52efc2 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_webhook.go +++ b/apis/database/v1alpha1/singleinstancedatabase_webhook.go @@ -235,14 +235,24 @@ func (r *SingleInstanceDatabase) ValidateCreate() error { } } - // servicePort validation + // servicePort and tcpServicePort validation if !r.Spec.LoadBalancer { // NodePort service is expected. In this case servicePort should be in range 30000-32767 - if r.Spec.ServicePort != 0 && (r.Spec.ServicePort < 30000 || r.Spec.ServicePort > 32767) { + if r.Spec.ListenerPort != 0 && (r.Spec.ListenerPort < 30000 || r.Spec.ListenerPort > 32767) { allErrs = append(allErrs, - field.Invalid(field.NewPath("spec").Child("servicePort"), r.Spec.ServicePort, - "servicePort should be in 30000-32767 range.")) + field.Invalid(field.NewPath("spec").Child("listenerPort"), r.Spec.ListenerPort, + "listenerPort should be in 30000-32767 range.")) } + if r.Spec.TcpsListenerPort != 0 && (r.Spec.TcpsListenerPort < 30000 || r.Spec.TcpsListenerPort > 32767) { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec").Child("tcpsListenerPort"), r.Spec.TcpsListenerPort, + "tcpsListenerPort should be in 30000-32767 range.")) + } + } + if r.Spec.ListenerPort != 0 && r.Spec.TcpsListenerPort != 0 && r.Spec.ListenerPort == r.Spec.TcpsListenerPort { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec").Child("tcpsListenerPort"), r.Spec.TcpsListenerPort, + "listenerPort and tcpsListenerPort can not be equal.")) } // Certificate Renew Duration Validation diff --git a/commons/database/constants.go b/commons/database/constants.go index 3aeb66f7..9ce63500 100644 --- a/commons/database/constants.go +++ b/commons/database/constants.go @@ -495,3 +495,6 @@ const DisableTcpsCMD string = "$ORACLE_BASE/$CONFIG_TCPS_FILE disable" // TCPS clientWallet update command const ClientWalletUpdate string = "sed -i -e 's/HOST.*$/HOST=%s)/g' -e 's/PORT.*$/PORT=%d)/g' ${ORACLE_BASE}/oradata/clientWallet/${ORACLE_SID}/tnsnames.ora" + +// TCPS clientWallet location +const ClientWalletLocation string = "/opt/oracle/oradata/clientWallet/%s" diff --git a/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml b/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml index fbbe19ca..5acf2b04 100644 --- a/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml +++ b/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml @@ -33,6 +33,13 @@ spec: - jsonPath: .status.connectString name: Connect Str type: string + - jsonPath: .status.tcpsConnectString + name: TCPS Connect Str + type: string + - jsonPath: .status.tcpsPdbConnectString + name: TCPS Pdb Connect Str + priority: 1 + type: string - jsonPath: .status.pdbConnectString name: Pdb Connect Str priority: 1 @@ -77,8 +84,6 @@ spec: type: object archiveLog: type: boolean - certRenewDuration: - type: string charset: type: string cloneFrom: @@ -122,6 +127,8 @@ spec: sgaTarget: type: integer type: object + listenerPort: + type: integer loadBalancer: type: boolean nodeSelector: @@ -158,14 +165,16 @@ spec: additionalProperties: type: string type: object - servicePort: - type: integer sid: description: SID must be alphanumeric (no special characters, only a-z, A-Z, 0-9), and no longer than 12 characters. maxLength: 12 pattern: ^[a-zA-Z0-9]+$ type: string + tcpsCertRenewInterval: + type: string + tcpsListenerPort: + type: integer required: - image type: object @@ -179,10 +188,12 @@ spec: type: string certCreationTimestamp: type: string - certRenewDuration: + certRenewInterval: type: string charset: type: string + clientWalletLoc: + type: string cloneFrom: type: string clusterConnectString: @@ -338,6 +349,10 @@ spec: type: object status: type: string + tcpsConnectString: + type: string + tcpsPdbConnectString: + type: string required: - isTcpsEnabled - persistence diff --git a/config/samples/sidb/singleinstancedatabase.yaml b/config/samples/sidb/singleinstancedatabase.yaml index 2a3c9cae..dbfbe8e9 100644 --- a/config/samples/sidb/singleinstancedatabase.yaml +++ b/config/samples/sidb/singleinstancedatabase.yaml @@ -89,9 +89,13 @@ spec: ## if loadBalService : false, service type = "NodePort" else "LoadBalancer" loadBalancer: false - ## If loadBalancer is enabled, the servicePort is the load balancer port number - ## If loadBalancer is disabled, the servicePort is the NodePort(should be in range 30000-32767) - #servicePort: 30001 + ## 'listenerPort' and 'tcpsListenerPort' fields customizes port cofigurations for normal and tcps database listeners + ## 'tcpsListenerPort' will come in effect only when 'enableTCPS' field is set + ## If loadBalancer is enabled, the listenerPort, tcpsListenerPort will be the load balancer ports + ## If loadBalancer is disabled, the listenerPort, tcpsListenerPort will be the node ports(should be in range 30000-32767) + ## If enableTCPS is set, and listenerPort is commented/not mentioned in the YAML file, only TCPS endpoint will be exposed + #listenerPort: 30001 + #tcpsListenerPort: 30002 ## Service Annotations (Cloud provider specific), for configuring the service (e.g. private LoadBalancer service) #serviceAnnotations: diff --git a/controllers/database/oraclerestdataservice_controller.go b/controllers/database/oraclerestdataservice_controller.go index 319b0d1e..267759aa 100644 --- a/controllers/database/oraclerestdataservice_controller.go +++ b/controllers/database/oraclerestdataservice_controller.go @@ -582,7 +582,7 @@ func (r *OracleRestDataServiceReconciler) instantiatePodSpec(m *dbapi.OracleRest { Name: "init-permissions", Image: m.Spec.Image.PullFrom, - Command: []string{"/bin/sh", "-c", fmt.Sprintf("chown %d:%d /opt/oracle/ords/config/ords", int(dbcommons.ORACLE_UID), int(dbcommons.DBA_GUID))}, + Command: []string{"/bin/sh", "-c", fmt.Sprintf("chown %d:%d /opt/oracle/ords/config/ords || true", int(dbcommons.ORACLE_UID), int(dbcommons.DBA_GUID))}, SecurityContext: &corev1.SecurityContext{ // User ID 0 means, root user RunAsUser: func() *int64 { i := int64(0); return &i }(), @@ -728,6 +728,7 @@ func (r *OracleRestDataServiceReconciler) instantiatePodSpec(m *dbapi.OracleRest SecurityContext: &corev1.PodSecurityContext{ RunAsUser: func() *int64 { i := int64(dbcommons.ORACLE_UID); return &i }(), RunAsGroup: func() *int64 { i := int64(dbcommons.DBA_GUID); return &i }(), + FSGroup: func() *int64 { i := int64(dbcommons.DBA_GUID); return &i }(), }, ImagePullSecrets: []corev1.LocalObjectReference{ diff --git a/controllers/database/singleinstancedatabase_controller.go b/controllers/database/singleinstancedatabase_controller.go index af89e906..46210899 100644 --- a/controllers/database/singleinstancedatabase_controller.go +++ b/controllers/database/singleinstancedatabase_controller.go @@ -567,7 +567,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns initContainers = append(initContainers, corev1.Container{ Name: "init-permissions", Image: m.Spec.Image.PullFrom, - Command: []string{"/bin/sh", "-c", fmt.Sprintf("chown %d:%d /opt/oracle/oradata", int(dbcommons.ORACLE_UID), int(dbcommons.ORACLE_GUID))}, + Command: []string{"/bin/sh", "-c", fmt.Sprintf("chown %d:%d /opt/oracle/oradata || true", int(dbcommons.ORACLE_UID), int(dbcommons.ORACLE_GUID))}, SecurityContext: &corev1.SecurityContext{ // User ID 0 means, root user RunAsUser: func() *int64 { i := int64(0); return &i }(), @@ -857,6 +857,10 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns i := int64(dbcommons.ORACLE_GUID) return &i }(), + FSGroup: func() *int64 { + i := int64(dbcommons.ORACLE_GUID) + return &i + }(), }, ImagePullSecrets: []corev1.LocalObjectReference{ { @@ -1040,7 +1044,7 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex log := r.Log.WithValues("createOrReplaceSVC", req.NamespacedName) /** Two k8s services gets created: - 1. One service is ClusterIP service for cluster only communications on the listener port, + 1. One service is ClusterIP service for cluster only communications on the listener port 1521, 2. One service is NodePort/LoadBalancer (according to the YAML specs) for users to connect **/ @@ -1051,25 +1055,25 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex clusterSvcName := m.Name extSvcName := m.Name + "-ext" - // svcPort is the intended port for extSvc taken from singleinstancedatabase YAML file + // svcPort is the intended port for extSvc taken from singleinstancedatabase YAML file for normal database connection // If loadBalancer is true, it would be the listener port otherwise it would be node port svcPort := func() int32 { - if m.Spec.ServicePort != 0 { - return int32(m.Spec.ServicePort) + if m.Spec.ListenerPort != 0 { + return int32(m.Spec.ListenerPort) } else { - if m.Spec.EnableTCPS { - return dbcommons.CONTAINER_TCPS_PORT - } else { - return dbcommons.CONTAINER_LISTENER_PORT - } + return dbcommons.CONTAINER_LISTENER_PORT } }() - // extSvcTargetPort is used to check the target port of the extSvc when TCPS is enabled/disabled - extSvcTargetPort := dbcommons.CONTAINER_LISTENER_PORT - if m.Spec.EnableTCPS { - extSvcTargetPort = dbcommons.CONTAINER_TCPS_PORT - } + // tcpsSvcPort is the intended port for extSvc taken from singleinstancedatabase YAML file for TCPS connection + // If loadBalancer is true, it would be the listener port otherwise it would be node port + tcpsSvcPort := func() int32 { + if m.Spec.TcpsListenerPort != 0 { + return int32(m.Spec.TcpsListenerPort) + } else { + return dbcommons.CONTAINER_TCPS_PORT + } + }() // Querying for the K8s service resources getClusterSvcErr := r.Get(ctx, types.NamespacedName{Name: clusterSvcName, Namespace: m.Namespace}, clusterSvc) @@ -1106,15 +1110,39 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex log.Error(getExtSvcErr, "Error encountered in obtaining the service", extSvcName) return requeueY, getExtSvcErr } else { - // extSvc service found - var extSvcPort int32 - if extSvc.Spec.Type == corev1.ServiceType("LoadBalancer") { - extSvcPort = extSvc.Spec.Ports[1].Port - } else if extSvc.Spec.Type == corev1.ServiceType("NodePort") { - extSvcPort = extSvc.Spec.Ports[1].NodePort + // Counting required number of ports in extSvc + requiredPorts := 2 + if m.Spec.EnableTCPS && m.Spec.ListenerPort != 0 { + requiredPorts = 3 + } + + // Obtaining all ports of the extSvc k8s service + var targetPorts []int32 + for _, port := range extSvc.Spec.Ports { + if extSvc.Spec.Type == corev1.ServiceType("LoadBalancer") { + targetPorts = append(targetPorts, port.Port) + } else if extSvc.Spec.Type == corev1.ServiceType("NodePort") { + targetPorts = append(targetPorts, port.NodePort) + } + } + + deleteSvc := false + if extSvc.Spec.Type != extSvcType || len(extSvc.Spec.Ports) != requiredPorts { + deleteSvc = true + } else { + // Match for the ports + if (m.Spec.ListenerPort != 0 && svcPort != targetPorts[1]) || (m.Spec.TcpsListenerPort != 0 && tcpsSvcPort != targetPorts[len(targetPorts)-1]) { + deleteSvc = true + } + if m.Spec.ListenerPort == 0 && m.Spec.TcpsListenerPort == 0 { + if (m.Spec.EnableTCPS && extSvc.Spec.Ports[1].TargetPort.IntVal != dbcommons.CONTAINER_TCPS_PORT) || + (!m.Spec.EnableTCPS && extSvc.Spec.Ports[1].TargetPort.IntVal != dbcommons.CONTAINER_LISTENER_PORT) { + deleteSvc = true + } + } } - if extSvc.Spec.Type != extSvcType || (m.Spec.ServicePort != 0 && extSvcPort != svcPort) || extSvc.Spec.Ports[1].TargetPort.IntVal != extSvcTargetPort { + if deleteSvc { // Deleting th service log.Info("Deleting service", "name", extSvcName) err := r.Delete(ctx, extSvc) @@ -1132,6 +1160,8 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex m.Status.ConnectString = dbcommons.ValueUnavailable m.Status.PdbConnectString = dbcommons.ValueUnavailable m.Status.OemExpressUrl = dbcommons.ValueUnavailable + m.Status.TcpsConnectString = dbcommons.ValueUnavailable + m.Status.TcpsPdbConnectString = dbcommons.ValueUnavailable // New service has to be created ports := []corev1.ServicePort{ @@ -1143,21 +1173,56 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex } if m.Spec.LoadBalancer { - ports = append(ports, corev1.ServicePort{ - Name: "listener", - Protocol: corev1.ProtocolTCP, - Port: svcPort, - TargetPort: intstr.FromInt(int(extSvcTargetPort)), - }) + if m.Spec.EnableTCPS { + if m.Spec.ListenerPort != 0 { + ports = append(ports, corev1.ServicePort{ + Name: "listener", + Protocol: corev1.ProtocolTCP, + Port: svcPort, + TargetPort: intstr.FromInt(int(dbcommons.CONTAINER_LISTENER_PORT)), + }) + } + ports = append(ports, corev1.ServicePort{ + Name: "listener-tcps", + Protocol: corev1.ProtocolTCP, + Port: tcpsSvcPort, + TargetPort: intstr.FromInt(int(dbcommons.CONTAINER_TCPS_PORT)), + }) + } else { + ports = append(ports, corev1.ServicePort{ + Name: "listener", + Protocol: corev1.ProtocolTCP, + Port: svcPort, + TargetPort: intstr.FromInt(int(dbcommons.CONTAINER_LISTENER_PORT)), + }) + } } else { - ports = append(ports, corev1.ServicePort{ - Name: "listener", - Protocol: corev1.ProtocolTCP, - Port: extSvcTargetPort, - TargetPort: intstr.FromInt(int(extSvcTargetPort)), - }) - if m.Spec.ServicePort != 0 { - ports[1].NodePort = svcPort + if m.Spec.EnableTCPS { + if m.Spec.ListenerPort != 0 { + ports = append(ports, corev1.ServicePort{ + Name: "listener", + Protocol: corev1.ProtocolTCP, + Port: dbcommons.CONTAINER_LISTENER_PORT, + NodePort: svcPort, + }) + } + ports = append(ports, corev1.ServicePort{ + Name: "listener-tcps", + Protocol: corev1.ProtocolTCP, + Port: dbcommons.CONTAINER_TCPS_PORT, + }) + if m.Spec.TcpsListenerPort != 0 { + ports[len(ports)-1].NodePort = tcpsSvcPort + } + } else { + ports = append(ports, corev1.ServicePort{ + Name: "listener", + Protocol: corev1.ProtocolTCP, + Port: dbcommons.CONTAINER_LISTENER_PORT, + }) + if m.Spec.ListenerPort != 0 { + ports[len(ports)-1].NodePort = svcPort + } } } @@ -1194,6 +1259,10 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex m.Status.ConnectString = lbAddress + ":" + fmt.Sprint(extSvc.Spec.Ports[1].Port) + "/" + strings.ToUpper(sid) m.Status.PdbConnectString = lbAddress + ":" + fmt.Sprint(extSvc.Spec.Ports[1].Port) + "/" + strings.ToUpper(pdbName) m.Status.OemExpressUrl = "https://" + lbAddress + ":" + fmt.Sprint(extSvc.Spec.Ports[0].Port) + "/em" + if m.Spec.EnableTCPS { + m.Status.TcpsConnectString = lbAddress + ":" + fmt.Sprint(extSvc.Spec.Ports[len(extSvc.Spec.Ports)-1].Port) + "/" + strings.ToUpper(sid) + m.Status.TcpsPdbConnectString = lbAddress + ":" + fmt.Sprint(extSvc.Spec.Ports[len(extSvc.Spec.Ports)-1].Port) + "/" + strings.ToUpper(pdbName) + } } } else { m.Status.ClusterConnectString = extSvc.Name + "." + extSvc.Namespace + ":" + fmt.Sprint(extSvc.Spec.Ports[1].Port) + "/" + strings.ToUpper(sid) @@ -1202,6 +1271,10 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex m.Status.ConnectString = nodeip + ":" + fmt.Sprint(extSvc.Spec.Ports[1].NodePort) + "/" + strings.ToUpper(sid) m.Status.PdbConnectString = nodeip + ":" + fmt.Sprint(extSvc.Spec.Ports[1].NodePort) + "/" + strings.ToUpper(pdbName) m.Status.OemExpressUrl = "https://" + nodeip + ":" + fmt.Sprint(extSvc.Spec.Ports[0].NodePort) + "/em" + if m.Spec.EnableTCPS { + m.Status.TcpsConnectString = nodeip + ":" + fmt.Sprint(extSvc.Spec.Ports[len(extSvc.Spec.Ports)-1].NodePort) + "/" + strings.ToUpper(sid) + m.Status.TcpsPdbConnectString = nodeip + ":" + fmt.Sprint(extSvc.Spec.Ports[len(extSvc.Spec.Ports)-1].NodePort) + "/" + strings.ToUpper(pdbName) + } } } @@ -1786,12 +1859,12 @@ func (r *SingleInstanceDatabaseReconciler) updateClientWallet(m *dbapi.SingleIns if host == "" { host = extSvc.Status.LoadBalancer.Ingress[0].IP } - port = extSvc.Spec.Ports[1].Port + port = extSvc.Spec.Ports[len(extSvc.Spec.Ports)-1].Port } } else { host = dbcommons.GetNodeIp(r, ctx, req) if host != "" { - port = extSvc.Spec.Ports[1].NodePort + port = extSvc.Spec.Ports[len(extSvc.Spec.Ports)-1].NodePort } } @@ -1836,6 +1909,7 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat // Updating the Status and publishing the event m.Status.CertCreationTimestamp = time.Now().Format(time.RFC3339) m.Status.IsTcpsEnabled = true + m.Status.ClientWalletLoc = fmt.Sprintf(dbcommons.ClientWalletLocation, m.Spec.Sid) r.Status().Update(ctx, m) eventMsg = "TCPS Enabled." @@ -1869,6 +1943,7 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat // Updating the Status and publishing the event m.Status.CertCreationTimestamp = "" m.Status.IsTcpsEnabled = false + m.Status.ClientWalletLoc = "" r.Status().Update(ctx, m) eventMsg = "TCPS Disabled." @@ -1901,13 +1976,20 @@ func (r *SingleInstanceDatabaseReconciler) configTcps(m *dbapi.SingleInstanceDat requeueDuration += func() time.Duration { requeueDuration, _ := time.ParseDuration("1s"); return requeueDuration }() futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: requeueDuration} } - if m.Status.CertRenewDuration != m.Spec.TcpsCertRenewInterval { + if m.Status.CertRenewInterval != m.Spec.TcpsCertRenewInterval { requeueDuration, _ := time.ParseDuration(m.Spec.TcpsCertRenewInterval) requeueDuration += func() time.Duration { requeueDuration, _ := time.ParseDuration("1s"); return requeueDuration }() futureRequeue = ctrl.Result{Requeue: true, RequeueAfter: requeueDuration} - m.Status.CertRenewDuration = m.Spec.TcpsCertRenewInterval + m.Status.CertRenewInterval = m.Spec.TcpsCertRenewInterval + } + // update clientWallet + err := r.updateClientWallet(m, readyPod, ctx, req) + if err != nil { + r.Log.Error(err, "Error in updating tnsnames.ora clientWallet...") + return requeueY, nil } + } else if m.Spec.EnableTCPS && m.Status.IsTcpsEnabled && m.Spec.TcpsCertRenewInterval == "" { // update clientWallet err := r.updateClientWallet(m, readyPod, ctx, req) if err != nil { diff --git a/docs/sidb/README.md b/docs/sidb/README.md index f3c5af1d..8ad98e4f 100644 --- a/docs/sidb/README.md +++ b/docs/sidb/README.md @@ -555,11 +555,18 @@ The following steps are required to connect the Database using TCPS: - The self-signed certificate used with TCPS has validity for 2 years. After the certificate is expired, it will be renewed by the `OraOperator` automatically. You need to download the wallet again after the auto-renewal. ### Specifying Custom Ports -As mentioned in the section [Setup Database with LoadBalancer](#setup-database-with-loadbalancer), there are two kubernetes services possible for the database: NodePort and LoadBalancer. You can specify which port to use with these services by editing the `servicePort` field of the [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file. +As mentioned in the section [Setup Database with LoadBalancer](#setup-database-with-loadbalancer), there are two kubernetes services possible for the database: NodePort and LoadBalancer. You can specify which port to use with these services by editing the `listenerPort` and `tcpsListenerPort` fields of the [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file. -If the `LoadBalancer` is enabled, the `servicePort` will be the opened load balancer port for database connections. +`listenerPort` is intended for normal database connections. Similarly, `tcpsListenerPort` is intended for TCPS database connections. -In case of `NodePort` service, the `servicePort` will be the opened port on the Kubernetes nodes for database connections. In this case, the allowed range for the `servicePort` is 30000-32767. +If the `LoadBalancer` is enabled, the `listenerPort`, and `tcpsListenerPort` will be the opened ports on the Load Balancer for normal and TCPS database connections respectively. + +In case of `NodePort` service, `listenerPort`, and `tcpsListenerPort` will be the opened ports on the Kubernetes nodes for for normal and TCPS database connections respectively. In this case, the allowed range for the `listenerPort`, and `tcpsListenerPort` is 30000-32767. + +**NOTE:** +- `listenerPort` and `tcpsListenerPort` can not have same values. +- `tcpsListenerPort` will come into effect only when TCPS connections are enabled (i.e. `enableTCPS` field is set in [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file). +- If TCPS connections are enabled, and `listenerPort` is commented/removed in the [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file, only TCPS endpoint will be exposed. ## OracleRestDataService Resource From aa37794a964ddd190d06b0630f687d27320ba49b Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 8 Sep 2022 06:01:25 +0000 Subject: [PATCH 23/37] Bugfixes related to TCPS changes --- apis/database/v1alpha1/pdb_webhook.go | 5 +- .../v1alpha1/singleinstancedatabase_types.go | 2 +- .../singleinstancedatabase_webhook.go | 5 + commons/database/constants.go | 19 ++++ commons/database/utils.go | 2 +- ...se.oracle.com_singleinstancedatabases.yaml | 8 +- .../singleinstancedatabase_controller.go | 95 +++++++++++++++++-- docs/sidb/README.md | 4 +- 8 files changed, 120 insertions(+), 20 deletions(-) diff --git a/apis/database/v1alpha1/pdb_webhook.go b/apis/database/v1alpha1/pdb_webhook.go index 573a8729..1fb2f4cf 100644 --- a/apis/database/v1alpha1/pdb_webhook.go +++ b/apis/database/v1alpha1/pdb_webhook.go @@ -38,9 +38,8 @@ /* MODIFIED (MM/DD/YY) ** rcitton 07/14/22 - 33822886 -*/ - - + */ + package v1alpha1 import ( diff --git a/apis/database/v1alpha1/singleinstancedatabase_types.go b/apis/database/v1alpha1/singleinstancedatabase_types.go index 7edd9af8..cfd22425 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_types.go +++ b/apis/database/v1alpha1/singleinstancedatabase_types.go @@ -177,8 +177,8 @@ type SingleInstanceDatabaseStatus struct { // +kubebuilder:printcolumn:JSONPath=".status.releaseUpdate",name="Version",type="string" // +kubebuilder:printcolumn:JSONPath=".status.connectString",name="Connect Str",type="string" // +kubebuilder:printcolumn:JSONPath=".status.tcpsConnectString",name="TCPS Connect Str",type="string" -// +kubebuilder:printcolumn:JSONPath=".status.tcpsPdbConnectString",name="TCPS Pdb Connect Str",type="string", priority=1 // +kubebuilder:printcolumn:JSONPath=".status.pdbConnectString",name="Pdb Connect Str",type="string",priority=1 +// +kubebuilder:printcolumn:JSONPath=".status.tcpsPdbConnectString",name="TCPS Pdb Connect Str",type="string", priority=1 // +kubebuilder:printcolumn:JSONPath=".status.oemExpressUrl",name="Oem Express Url",type="string" // SingleInstanceDatabase is the Schema for the singleinstancedatabases API diff --git a/apis/database/v1alpha1/singleinstancedatabase_webhook.go b/apis/database/v1alpha1/singleinstancedatabase_webhook.go index af52efc2..f2127e1e 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_webhook.go +++ b/apis/database/v1alpha1/singleinstancedatabase_webhook.go @@ -254,6 +254,11 @@ func (r *SingleInstanceDatabase) ValidateCreate() error { field.Invalid(field.NewPath("spec").Child("tcpsListenerPort"), r.Spec.TcpsListenerPort, "listenerPort and tcpsListenerPort can not be equal.")) } + if r.Spec.EnableTCPS && r.Spec.TcpsListenerPort == 0 && r.Spec.ListenerPort == 1522 { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec").Child("listenerPort"), r.Spec.ListenerPort, + "listenerPort can not be 1522 as the default port for tcpsListenerPort is 1522.")) + } // Certificate Renew Duration Validation if r.Spec.TcpsCertRenewInterval != "" { diff --git a/commons/database/constants.go b/commons/database/constants.go index 9ce63500..712f9819 100644 --- a/commons/database/constants.go +++ b/commons/database/constants.go @@ -498,3 +498,22 @@ const ClientWalletUpdate string = "sed -i -e 's/HOST.*$/HOST=%s)/g' -e 's/PORT.* // TCPS clientWallet location const ClientWalletLocation string = "/opt/oracle/oradata/clientWallet/%s" + +// Service Patch Payloads +// Three port payload: one OEM express, one TCP and one TCPS port +const ThreePortPayload string = "{\"spec\": { \"ports\": [{\"name\": \"xmldb\", \"port\": 5500, \"protocol\": \"TCP\"},{%s},{%s}]}}" + +// Two port payload: one OEM express, one TCP/TCPS port +const TwoPortPayload string = "{\"spec\": { \"ports\": [{\"name\": \"xmldb\", \"port\": 5500, \"protocol\": \"TCP\"},{%s}]}}" + +// Payload section for listener port +const LsnrPort string = "\"name\": \"listener\", \"protocol\": \"TCP\", \"port\": %d" + +// Payload section for listener node port +const LsnrNodePort string = "\"name\": \"listener\", \"protocol\": \"TCP\", \"port\": 1521, \"nodePort\": %d" + +// Payload section for TCPS port +const TcpsPort string = "\"name\": \"listener-tcps\", \"protocol\": \"TCP\", \"port\": %d" + +// Payload section for TCPS node port +const TcpsNodePort string = "\"name\": \"listener-tcps\", \"protocol\": \"TCP\", \"port\": 1522, \"nodePort\": %d" diff --git a/commons/database/utils.go b/commons/database/utils.go index 100b532c..91fdfb09 100644 --- a/commons/database/utils.go +++ b/commons/database/utils.go @@ -686,6 +686,6 @@ func PatchService(config *rest.Config, namespace string, ctx context.Context, re // Trying to patch the service resource using Strategic Merge strategy log.Info("Patching the service", "Service", svcName) - _, err = client.CoreV1().Services(namespace).Patch(ctx, svcName, types.StrategicMergePatchType, []byte(payload), metav1.PatchOptions{}) + _, err = client.CoreV1().Services(namespace).Patch(ctx, svcName, types.MergePatchType, []byte(payload), metav1.PatchOptions{}) return err } diff --git a/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml b/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml index 5acf2b04..493800d6 100644 --- a/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml +++ b/config/crd/bases/database.oracle.com_singleinstancedatabases.yaml @@ -36,14 +36,14 @@ spec: - jsonPath: .status.tcpsConnectString name: TCPS Connect Str type: string - - jsonPath: .status.tcpsPdbConnectString - name: TCPS Pdb Connect Str - priority: 1 - type: string - jsonPath: .status.pdbConnectString name: Pdb Connect Str priority: 1 type: string + - jsonPath: .status.tcpsPdbConnectString + name: TCPS Pdb Connect Str + priority: 1 + type: string - jsonPath: .status.oemExpressUrl name: Oem Express Url type: string diff --git a/controllers/database/singleinstancedatabase_controller.go b/controllers/database/singleinstancedatabase_controller.go index 46210899..7e432188 100644 --- a/controllers/database/singleinstancedatabase_controller.go +++ b/controllers/database/singleinstancedatabase_controller.go @@ -1127,17 +1127,38 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex } deleteSvc := false - if extSvc.Spec.Type != extSvcType || len(extSvc.Spec.Ports) != requiredPorts { + patchSvc := false + if extSvc.Spec.Type != extSvcType { deleteSvc = true } else { - // Match for the ports - if (m.Spec.ListenerPort != 0 && svcPort != targetPorts[1]) || (m.Spec.TcpsListenerPort != 0 && tcpsSvcPort != targetPorts[len(targetPorts)-1]) { - deleteSvc = true + // Conditions to determine whether to patch or not + if len(extSvc.Spec.Ports) != requiredPorts { + patchSvc = true } - if m.Spec.ListenerPort == 0 && m.Spec.TcpsListenerPort == 0 { - if (m.Spec.EnableTCPS && extSvc.Spec.Ports[1].TargetPort.IntVal != dbcommons.CONTAINER_TCPS_PORT) || - (!m.Spec.EnableTCPS && extSvc.Spec.Ports[1].TargetPort.IntVal != dbcommons.CONTAINER_LISTENER_PORT) { - deleteSvc = true + + if (m.Spec.ListenerPort != 0 && svcPort != targetPorts[1]) || (m.Spec.EnableTCPS && m.Spec.TcpsListenerPort != 0 && tcpsSvcPort != targetPorts[len(targetPorts)-1]) { + patchSvc = true + } + + if m.Spec.LoadBalancer { + if m.Spec.EnableTCPS { + if m.Spec.TcpsListenerPort == 0 && tcpsSvcPort != targetPorts[len(targetPorts)-1] { + patchSvc = true + } + } else { + if m.Spec.ListenerPort == 0 && svcPort != targetPorts[1] { + patchSvc = true + } + } + } else { + if m.Spec.EnableTCPS { + if m.Spec.TcpsListenerPort == 0 && tcpsSvcPort != extSvc.Spec.Ports[len(targetPorts)-1].TargetPort.IntVal { + patchSvc = true + } + } else { + if m.Spec.ListenerPort == 0 && svcPort != extSvc.Spec.Ports[1].TargetPort.IntVal { + patchSvc = true + } } } } @@ -1145,13 +1166,69 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex if deleteSvc { // Deleting th service log.Info("Deleting service", "name", extSvcName) - err := r.Delete(ctx, extSvc) + // Setting GracePeriodSeconds to 0 for instant deletion + delOpts := &client.DeleteOptions{} + var gracePeriod client.GracePeriodSeconds = 0 + gracePeriod.ApplyToDelete(delOpts) + + err := r.Delete(ctx, extSvc, delOpts) if err != nil { r.Log.Error(err, "Failed to delete service", "name", extSvcName) return requeueN, err } isExtSvcFound = false } + + if patchSvc { + // Reset connect strings whenever patching happens + m.Status.Status = dbcommons.StatusUpdating + m.Status.ConnectString = dbcommons.ValueUnavailable + m.Status.PdbConnectString = dbcommons.ValueUnavailable + m.Status.OemExpressUrl = dbcommons.ValueUnavailable + m.Status.TcpsConnectString = dbcommons.ValueUnavailable + m.Status.TcpsPdbConnectString = dbcommons.ValueUnavailable + + // Payload formation for patching the service + var payload string + if m.Spec.LoadBalancer { + if m.Spec.EnableTCPS { + if m.Spec.ListenerPort != 0 { + payload = fmt.Sprintf(dbcommons.ThreePortPayload, fmt.Sprintf(dbcommons.LsnrPort, svcPort), fmt.Sprintf(dbcommons.TcpsPort, tcpsSvcPort)) + } else { + payload = fmt.Sprintf(dbcommons.TwoPortPayload, fmt.Sprintf(dbcommons.TcpsPort, tcpsSvcPort)) + } + } else { + payload = fmt.Sprintf(dbcommons.TwoPortPayload, fmt.Sprintf(dbcommons.LsnrPort, svcPort)) + } + } else { + if m.Spec.EnableTCPS { + if m.Spec.ListenerPort != 0 && m.Spec.TcpsListenerPort != 0 { + payload = fmt.Sprintf(dbcommons.ThreePortPayload, fmt.Sprintf(dbcommons.LsnrNodePort, svcPort), fmt.Sprintf(dbcommons.TcpsNodePort, tcpsSvcPort)) + } else if m.Spec.ListenerPort != 0 { + payload = fmt.Sprintf(dbcommons.ThreePortPayload, fmt.Sprintf(dbcommons.LsnrNodePort, svcPort), fmt.Sprintf(dbcommons.TcpsPort, tcpsSvcPort)) + } else if m.Spec.TcpsListenerPort != 0 { + payload = fmt.Sprintf(dbcommons.TwoPortPayload, fmt.Sprintf(dbcommons.TcpsNodePort, tcpsSvcPort)) + } else { + payload = fmt.Sprintf(dbcommons.TwoPortPayload, fmt.Sprintf(dbcommons.TcpsPort, tcpsSvcPort)) + } + } else { + if m.Spec.ListenerPort != 0 { + payload = fmt.Sprintf(dbcommons.TwoPortPayload, fmt.Sprintf(dbcommons.LsnrNodePort, svcPort)) + } else { + payload = fmt.Sprintf(dbcommons.TwoPortPayload, fmt.Sprintf(dbcommons.LsnrPort, svcPort)) + } + } + } + + //Attemp Service Pathcing + log.Info("Patching the service", "Service.Name", extSvc.Name, "payload", payload) + err := dbcommons.PatchService(r.Config, m.Namespace, ctx, req, extSvcName, payload) + if err != nil { + log.Error(err, "Failed to patch Service") + } + //Requeue once after patching + return requeueY, err + } } if !isExtSvcFound { diff --git a/docs/sidb/README.md b/docs/sidb/README.md index 8ad98e4f..29a1fe6b 100644 --- a/docs/sidb/README.md +++ b/docs/sidb/README.md @@ -63,8 +63,8 @@ To obtain a quick database status, use the following command as an example: ```sh $ kubectl get singleinstancedatabase sidb-sample -NAME EDITION STATUS VERSION CONNECT STR OEM EXPRESS URL -sidb-sample Enterprise Healthy 19.3.0.0.0 10.0.25.54:1521/ORCLCDB https://10.0.25.54:5500/em +NAME EDITION STATUS VERSION CONNECT STR TCPS CONNECT STR OEM EXPRESS URL +sidb-sample Enterprise Healthy 19.3.0.0.0 10.0.25.54:1521/ORCL1 Unavailable https://10.0.25.54:5500/em ``` #### Detailed Status From ea6244597e2af11d3e12e0a6183dbe662d080d66 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 9 Sep 2022 11:00:24 +0000 Subject: [PATCH 24/37] Bugfixes for 34578825 and 34582087 --- .../v1alpha1/singleinstancedatabase_webhook.go | 18 ++++++++++-------- commons/database/constants.go | 4 ++-- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/apis/database/v1alpha1/singleinstancedatabase_webhook.go b/apis/database/v1alpha1/singleinstancedatabase_webhook.go index f2127e1e..e2671082 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_webhook.go +++ b/apis/database/v1alpha1/singleinstancedatabase_webhook.go @@ -243,25 +243,27 @@ func (r *SingleInstanceDatabase) ValidateCreate() error { field.Invalid(field.NewPath("spec").Child("listenerPort"), r.Spec.ListenerPort, "listenerPort should be in 30000-32767 range.")) } - if r.Spec.TcpsListenerPort != 0 && (r.Spec.TcpsListenerPort < 30000 || r.Spec.TcpsListenerPort > 32767) { + if r.Spec.EnableTCPS && r.Spec.TcpsListenerPort != 0 && (r.Spec.TcpsListenerPort < 30000 || r.Spec.TcpsListenerPort > 32767) { allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("tcpsListenerPort"), r.Spec.TcpsListenerPort, "tcpsListenerPort should be in 30000-32767 range.")) } + } else { + // LoadBalancer Service is expected. + if r.Spec.EnableTCPS && r.Spec.TcpsListenerPort == 0 && r.Spec.ListenerPort == 1522 { + allErrs = append(allErrs, + field.Invalid(field.NewPath("spec").Child("listenerPort"), r.Spec.ListenerPort, + "listenerPort can not be 1522 as the default port for tcpsListenerPort is 1522.")) + } } - if r.Spec.ListenerPort != 0 && r.Spec.TcpsListenerPort != 0 && r.Spec.ListenerPort == r.Spec.TcpsListenerPort { + if r.Spec.EnableTCPS && r.Spec.ListenerPort != 0 && r.Spec.TcpsListenerPort != 0 && r.Spec.ListenerPort == r.Spec.TcpsListenerPort { allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("tcpsListenerPort"), r.Spec.TcpsListenerPort, "listenerPort and tcpsListenerPort can not be equal.")) } - if r.Spec.EnableTCPS && r.Spec.TcpsListenerPort == 0 && r.Spec.ListenerPort == 1522 { - allErrs = append(allErrs, - field.Invalid(field.NewPath("spec").Child("listenerPort"), r.Spec.ListenerPort, - "listenerPort can not be 1522 as the default port for tcpsListenerPort is 1522.")) - } // Certificate Renew Duration Validation - if r.Spec.TcpsCertRenewInterval != "" { + if r.Spec.EnableTCPS && r.Spec.TcpsCertRenewInterval != "" { duration, err := time.ParseDuration(r.Spec.TcpsCertRenewInterval) if err != nil { allErrs = append(allErrs, diff --git a/commons/database/constants.go b/commons/database/constants.go index 712f9819..9df367b0 100644 --- a/commons/database/constants.go +++ b/commons/database/constants.go @@ -507,13 +507,13 @@ const ThreePortPayload string = "{\"spec\": { \"ports\": [{\"name\": \"xmldb\", const TwoPortPayload string = "{\"spec\": { \"ports\": [{\"name\": \"xmldb\", \"port\": 5500, \"protocol\": \"TCP\"},{%s}]}}" // Payload section for listener port -const LsnrPort string = "\"name\": \"listener\", \"protocol\": \"TCP\", \"port\": %d" +const LsnrPort string = "\"name\": \"listener\", \"protocol\": \"TCP\", \"port\": %d, \"targetPort\": 1521" // Payload section for listener node port const LsnrNodePort string = "\"name\": \"listener\", \"protocol\": \"TCP\", \"port\": 1521, \"nodePort\": %d" // Payload section for TCPS port -const TcpsPort string = "\"name\": \"listener-tcps\", \"protocol\": \"TCP\", \"port\": %d" +const TcpsPort string = "\"name\": \"listener-tcps\", \"protocol\": \"TCP\", \"port\": %d, \"targetPort\": 1522" // Payload section for TCPS node port const TcpsNodePort string = "\"name\": \"listener-tcps\", \"protocol\": \"TCP\", \"port\": 1522, \"nodePort\": %d" From f47cdc1027023c964220a9aed621af9d38bdb06e Mon Sep 17 00:00:00 2001 From: matteo malvezzi Date: Thu, 15 Sep 2022 15:30:30 +0200 Subject: [PATCH 25/37] ords 22.2 https implementation --- README.md | 6 +- apis/database/v1alpha1/cdb_types.go | 15 + apis/database/v1alpha1/cdb_webhook.go | 6 + apis/database/v1alpha1/pdb_types.go | 18 +- .../v1alpha1/zz_generated.deepcopy.go | 85 +++ .../{onpremdb => multitenant}/cdb.yaml | 0 .../{onpremdb => multitenant}/cdb_secret.yaml | 0 .../{onpremdb => multitenant}/pdb_clone.yaml | 0 .../{onpremdb => multitenant}/pdb_create.yaml | 0 .../{onpremdb => multitenant}/pdb_delete.yaml | 0 .../{onpremdb => multitenant}/pdb_modify.yaml | 0 .../{onpremdb => multitenant}/pdb_plug.yaml | 0 .../{onpremdb => multitenant}/pdb_secret.yaml | 0 .../{onpremdb => multitenant}/pdb_unplug.yaml | 0 controllers/database/cdb_controller.go | 45 +- controllers/database/pdb_controller.go | 89 +++- docs/{onpremdb => multitenant}/README.md | 100 ++-- .../images/K8S_SECURE1.png | Bin .../images/K8S_SECURE2.png | Bin .../images/K8S_SECURE3.png | Bin .../images/K8S_SECURE4.png | Bin docs/multitenant/openssl_schema.jpg | Bin 0 -> 54221 bytes .../provisioning/add_replica.log | 0 .../provisioning/add_replica.md | 0 .../provisioning/add_replica.yaml | 0 .../provisioning/cdb.log | 0 .../provisioning/cdb.yaml | 0 .../provisioning/cdb_crd_resource.md | 0 .../provisioning/cdb_secret.log | 0 .../provisioning/cdb_secret.yaml | 0 .../provisioning/clone_pdb.log | 0 .../provisioning/clone_pdb.md | 0 .../provisioning/clone_pdb.yaml | 0 .../provisioning/create_pdb.log | 0 .../provisioning/create_pdb.md | 0 .../provisioning/create_pdb.yaml | 0 .../provisioning/delete_pdb.log | 0 .../provisioning/delete_pdb.md | 0 .../provisioning/delete_pdb.yaml | 0 .../example_setup_using_oci_oke_cluster.md | 0 .../provisioning/known_issues.md | 0 .../provisioning/modify_pdb.log | 0 .../provisioning/modify_pdb.md | 0 .../provisioning/modify_pdb_close.yaml | 0 .../provisioning/modify_pdb_open.yaml | 0 docs/multitenant/provisioning/ords_image.log | 503 ++++++++++++++++++ docs/multitenant/provisioning/ords_image.md | 64 +++ .../provisioning/pdb.log | 0 .../provisioning/pdb.yaml | 0 .../provisioning/pdb_crd_resource.md | 0 .../provisioning/pdb_secret.log | 0 .../provisioning/pdb_secret.yaml | 0 .../provisioning/plug_pdb.log | 0 .../provisioning/plug_pdb.md | 0 .../provisioning/plug_pdb.yaml | 0 .../provisioning/unplug_pdb.log | 0 .../provisioning/unplug_pdb.md | 0 .../provisioning/unplug_pdb.yaml | 0 .../provisioning/validation_error.md | 0 docs/onpremdb/provisioning/ords_image.log | 263 --------- docs/onpremdb/provisioning/ords_image.md | 68 --- oracle-database-operator.yaml | 84 ++- ords/Dockerfile | 63 +-- ords/cdbadmin.properties.tmpl | 3 - ords/ords_params.properties.tmpl | 16 - ords/runOrds.sh | 75 --- ords/runOrdsSSL.sh | 193 +++++++ ords/setupwebuser.sh | 20 - ords/standalone.properties.tmpl | 8 - 69 files changed, 1170 insertions(+), 554 deletions(-) rename config/samples/{onpremdb => multitenant}/cdb.yaml (100%) rename config/samples/{onpremdb => multitenant}/cdb_secret.yaml (100%) rename config/samples/{onpremdb => multitenant}/pdb_clone.yaml (100%) rename config/samples/{onpremdb => multitenant}/pdb_create.yaml (100%) rename config/samples/{onpremdb => multitenant}/pdb_delete.yaml (100%) rename config/samples/{onpremdb => multitenant}/pdb_modify.yaml (100%) rename config/samples/{onpremdb => multitenant}/pdb_plug.yaml (100%) rename config/samples/{onpremdb => multitenant}/pdb_secret.yaml (100%) rename config/samples/{onpremdb => multitenant}/pdb_unplug.yaml (100%) rename docs/{onpremdb => multitenant}/README.md (59%) rename docs/{onpremdb => multitenant}/images/K8S_SECURE1.png (100%) rename docs/{onpremdb => multitenant}/images/K8S_SECURE2.png (100%) rename docs/{onpremdb => multitenant}/images/K8S_SECURE3.png (100%) rename docs/{onpremdb => multitenant}/images/K8S_SECURE4.png (100%) create mode 100644 docs/multitenant/openssl_schema.jpg rename docs/{onpremdb => multitenant}/provisioning/add_replica.log (100%) rename docs/{onpremdb => multitenant}/provisioning/add_replica.md (100%) rename docs/{onpremdb => multitenant}/provisioning/add_replica.yaml (100%) rename docs/{onpremdb => multitenant}/provisioning/cdb.log (100%) rename docs/{onpremdb => multitenant}/provisioning/cdb.yaml (100%) rename docs/{onpremdb => multitenant}/provisioning/cdb_crd_resource.md (100%) rename docs/{onpremdb => multitenant}/provisioning/cdb_secret.log (100%) rename docs/{onpremdb => multitenant}/provisioning/cdb_secret.yaml (100%) rename docs/{onpremdb => multitenant}/provisioning/clone_pdb.log (100%) rename docs/{onpremdb => multitenant}/provisioning/clone_pdb.md (100%) rename docs/{onpremdb => multitenant}/provisioning/clone_pdb.yaml (100%) rename docs/{onpremdb => multitenant}/provisioning/create_pdb.log (100%) rename docs/{onpremdb => multitenant}/provisioning/create_pdb.md (100%) rename docs/{onpremdb => multitenant}/provisioning/create_pdb.yaml (100%) rename docs/{onpremdb => multitenant}/provisioning/delete_pdb.log (100%) rename docs/{onpremdb => multitenant}/provisioning/delete_pdb.md (100%) rename docs/{onpremdb => multitenant}/provisioning/delete_pdb.yaml (100%) rename docs/{onpremdb => multitenant}/provisioning/example_setup_using_oci_oke_cluster.md (100%) rename docs/{onpremdb => multitenant}/provisioning/known_issues.md (100%) rename docs/{onpremdb => multitenant}/provisioning/modify_pdb.log (100%) rename docs/{onpremdb => multitenant}/provisioning/modify_pdb.md (100%) rename docs/{onpremdb => multitenant}/provisioning/modify_pdb_close.yaml (100%) rename docs/{onpremdb => multitenant}/provisioning/modify_pdb_open.yaml (100%) create mode 100644 docs/multitenant/provisioning/ords_image.log create mode 100644 docs/multitenant/provisioning/ords_image.md rename docs/{onpremdb => multitenant}/provisioning/pdb.log (100%) rename docs/{onpremdb => multitenant}/provisioning/pdb.yaml (100%) rename docs/{onpremdb => multitenant}/provisioning/pdb_crd_resource.md (100%) rename docs/{onpremdb => multitenant}/provisioning/pdb_secret.log (100%) rename docs/{onpremdb => multitenant}/provisioning/pdb_secret.yaml (100%) rename docs/{onpremdb => multitenant}/provisioning/plug_pdb.log (100%) rename docs/{onpremdb => multitenant}/provisioning/plug_pdb.md (100%) rename docs/{onpremdb => multitenant}/provisioning/plug_pdb.yaml (100%) rename docs/{onpremdb => multitenant}/provisioning/unplug_pdb.log (100%) rename docs/{onpremdb => multitenant}/provisioning/unplug_pdb.md (100%) rename docs/{onpremdb => multitenant}/provisioning/unplug_pdb.yaml (100%) rename docs/{onpremdb => multitenant}/provisioning/validation_error.md (100%) delete mode 100644 docs/onpremdb/provisioning/ords_image.log delete mode 100644 docs/onpremdb/provisioning/ords_image.md delete mode 100644 ords/cdbadmin.properties.tmpl delete mode 100644 ords/ords_params.properties.tmpl delete mode 100644 ords/runOrds.sh create mode 100644 ords/runOrdsSSL.sh delete mode 100644 ords/setupwebuser.sh delete mode 100644 ords/standalone.properties.tmpl diff --git a/README.md b/README.md index 7ab5a833..1c091927 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ In this v0.2.0 release, `OraOperator` supports the following database configurat * Oracle Autonomous Database on dedicated Cloud infrastructure (ADB-D) * Containerized Single Instance databases (SIDB) deployed in the Oracle Kubernetes Engine (OKE) and any k8s where OraOperator is deployed * Containerized Sharded databases (SHARDED) deployed in OKE and any k8s where OraOperator is deployed -* Oracle On-Premises Databases (CDB/PDBs, Exadata) +* Oracle Multitenant Databases (CDB/PDBs) * Oracle Database Cloud Service (DBCS) (VMDB) * Oracle Autonomous Container Database (ACD) (infrastructure) the infrastructure for provisionning Autonomous Databases. @@ -25,7 +25,7 @@ This release of Oracle Database Operator for Kubernetes (the operator) supports * ACD: provision, bind, restart, terminate (soft/hard) * SIDB: Provision, clone, patch (in-place/out-of-place), update database initialization parameters, update database configuration (Flashback, archiving), Oracle Enterprise Manager (EM) Express (a basic observability console), Oracle REST Data Service (ORDS) to support REST based SQL, PDB management, SQL Developer Web, and Application Express (Apex) * SHARDED: Provision/deploy sharded databases and the shard topology, Add a new shard, Delete an existing shard -* On-Premises Database: Bind to a CDB, Create a  PDB, Plug a  PDB, Unplug a PDB, Delete a PDB, Clone a PDB, Open/Close a PDB +* Oracle Multitenant Database: Bind to a CDB, Create a  PDB, Plug a  PDB, Unplug a PDB, Delete a PDB, Clone a PDB, Open/Close a PDB * Database Cloud Service: Provision, Bind, Scale Up/Down, Liveness Probe, Manual Backup The upcoming releases will support new configurations, operations and capabilities. @@ -99,7 +99,7 @@ The quickstarts are designed for specific database configurations: * [Oracle Autonomous Container Database](./docs/acd/README.md) * [Containerized Oracle Single Instance Database](./docs/sidb/README.md) * [Containerized Oracle Sharded Database](./docs/sharding/README.md) -* [Oracle On-Premises Database](./docs/onpremdb/README.md) +* [Oracle Multitenant Database](./docs/multitenant/README.md) * [Oracle Database Cloud Service](./docs/dbcs/README.md) YAML file templates are available under [`/config/samples`](./config/samples/). You can copy and edit these template files to configure them for your use cases. diff --git a/apis/database/v1alpha1/cdb_types.go b/apis/database/v1alpha1/cdb_types.go index ccf5b0f5..0687cb7f 100644 --- a/apis/database/v1alpha1/cdb_types.go +++ b/apis/database/v1alpha1/cdb_types.go @@ -51,12 +51,19 @@ type CDBSpec struct { CDBName string `json:"cdbName,omitempty"` // Name of the CDB Service ServiceName string `json:"serviceName,omitempty"` + + TestVariable string `json:"TestVariable,omitempty"` + // Password for the CDB System Administrator SysAdminPwd CDBSysAdminPassword `json:"sysAdminPwd,omitempty"` // User in the root container with sysdba priviledges to manage PDB lifecycle CDBAdminUser CDBAdminUser `json:"cdbAdminUser,omitempty"` // Password for the CDB Administrator to manage PDB lifecycle CDBAdminPwd CDBAdminPassword `json:"cdbAdminPwd,omitempty"` + + CDBTlsKey CDBTLSKEY `json:"cdbTlsKey,omitempty"` + CDBTlsCrt CDBTLSCRT `json:"cdbTlsCrt,omitempty"` + // Password for user ORDS_PUBLIC_USER ORDSPwd ORDSPassword `json:"ordsPwd,omitempty"` // ORDS server port. For now, keep it as 8888. TO BE USED IN FUTURE RELEASE. @@ -120,6 +127,14 @@ type WebServerPassword struct { Secret CDBSecret `json:"secret"` } +type CDBTLSKEY struct { + Secret CDBSecret `json:"secret"` +} + +type CDBTLSCRT struct { + Secret CDBSecret `json:"secret"` +} + // CDBStatus defines the observed state of CDB type CDBStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster diff --git a/apis/database/v1alpha1/cdb_webhook.go b/apis/database/v1alpha1/cdb_webhook.go index 2d591223..5ca7f9b8 100644 --- a/apis/database/v1alpha1/cdb_webhook.go +++ b/apis/database/v1alpha1/cdb_webhook.go @@ -92,6 +92,12 @@ func (r *CDB) ValidateCreate() error { allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("serviceName"), "Please specify CDB Service name")) } + + if r.Spec.TestVariable == "" { + allErrs = append(allErrs, + field.Required(field.NewPath("spec").Child("TestVariable"), "Please specify CDB testvarable")) + } + if r.Spec.SCANName == "" { allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("scanName"), "Please specify SCAN Name for CDB")) diff --git a/apis/database/v1alpha1/pdb_types.go b/apis/database/v1alpha1/pdb_types.go index c5b95c81..a73d3dd8 100644 --- a/apis/database/v1alpha1/pdb_types.go +++ b/apis/database/v1alpha1/pdb_types.go @@ -47,6 +47,10 @@ type PDBSpec struct { // INSERT ADDITIONAL SPEC FIELDS - desired state of cluster // Important: Run "make" to regenerate code after modifying this file + PDBTlsKey PDBTLSKEY `json:"pdbTlsKey,omitempty"` + PDBTlsCrt PDBTLSCRT `json:"pdbTlsCrt,omitempty"` + PDBTlsCat PDBTLSCAT `json:"pdbTlsCat,omitempty"` + // Name of the CDB Custom Resource that runs the ORDS container CDBResName string `json:"cdbResName,omitempty"` // Name of the CDB @@ -132,6 +136,18 @@ type PDBSecret struct { Key string `json:"key"` } +type PDBTLSKEY struct { + Secret PDBSecret `json:"secret"` +} + +type PDBTLSCRT struct { + Secret PDBSecret `json:"secret"` +} + +type PDBTLSCAT struct { + Secret PDBSecret `json:"secret"` +} + // PDBStatus defines the observed state of PDB type PDBStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster @@ -157,7 +173,7 @@ type PDBStatus struct { // +kubebuilder:object:root=true // +kubebuilder:subresource:status -// +kubebuilder:printcolumn:JSONPath=".status.connString",name="Connect String",type="string",description="The connect string to be used" +// +kubebuilder:printcolumn:JSONPath=".status.connString",name="Connect_String",type="string",description="The connect string to be used" // +kubebuilder:printcolumn:JSONPath=".spec.cdbName",name="CDB Name",type="string",description="Name of the CDB" // +kubebuilder:printcolumn:JSONPath=".spec.pdbName",name="PDB Name",type="string",description="Name of the PDB" // +kubebuilder:printcolumn:JSONPath=".status.openMode",name="PDB State",type="string",description="PDB Open Mode" diff --git a/apis/database/v1alpha1/zz_generated.deepcopy.go b/apis/database/v1alpha1/zz_generated.deepcopy.go index 17eb7052..72d93dc2 100644 --- a/apis/database/v1alpha1/zz_generated.deepcopy.go +++ b/apis/database/v1alpha1/zz_generated.deepcopy.go @@ -702,6 +702,8 @@ func (in *CDBSpec) DeepCopyInto(out *CDBSpec) { out.SysAdminPwd = in.SysAdminPwd out.CDBAdminUser = in.CDBAdminUser out.CDBAdminPwd = in.CDBAdminPwd + out.CDBTlsKey = in.CDBTlsKey + out.CDBTlsCrt = in.CDBTlsCrt out.ORDSPwd = in.ORDSPwd out.WebServerUser = in.WebServerUser out.WebServerPwd = in.WebServerPwd @@ -755,6 +757,38 @@ func (in *CDBSysAdminPassword) DeepCopy() *CDBSysAdminPassword { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CDBTLSCRT) DeepCopyInto(out *CDBTLSCRT) { + *out = *in + out.Secret = in.Secret +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CDBTLSCRT. +func (in *CDBTLSCRT) DeepCopy() *CDBTLSCRT { + if in == nil { + return nil + } + out := new(CDBTLSCRT) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CDBTLSKEY) DeepCopyInto(out *CDBTLSKEY) { + *out = *in + out.Secret = in.Secret +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CDBTLSKEY. +func (in *CDBTLSKEY) DeepCopy() *CDBTLSKEY { + if in == nil { + return nil + } + out := new(CDBTLSKEY) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *CatalogSpec) DeepCopyInto(out *CatalogSpec) { *out = *in @@ -1662,6 +1696,9 @@ func (in *PDBSecret) DeepCopy() *PDBSecret { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PDBSpec) DeepCopyInto(out *PDBSpec) { *out = *in + out.PDBTlsKey = in.PDBTlsKey + out.PDBTlsCrt = in.PDBTlsCrt + out.PDBTlsCat = in.PDBTlsCat out.AdminName = in.AdminName out.AdminPwd = in.AdminPwd if in.ReuseTempFile != nil { @@ -1723,6 +1760,54 @@ func (in *PDBStatus) DeepCopy() *PDBStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PDBTLSCAT) DeepCopyInto(out *PDBTLSCAT) { + *out = *in + out.Secret = in.Secret +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PDBTLSCAT. +func (in *PDBTLSCAT) DeepCopy() *PDBTLSCAT { + if in == nil { + return nil + } + out := new(PDBTLSCAT) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PDBTLSCRT) DeepCopyInto(out *PDBTLSCRT) { + *out = *in + out.Secret = in.Secret +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PDBTLSCRT. +func (in *PDBTLSCRT) DeepCopy() *PDBTLSCRT { + if in == nil { + return nil + } + out := new(PDBTLSCRT) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PDBTLSKEY) DeepCopyInto(out *PDBTLSKEY) { + *out = *in + out.Secret = in.Secret +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PDBTLSKEY. +func (in *PDBTLSKEY) DeepCopy() *PDBTLSKEY { + if in == nil { + return nil + } + out := new(PDBTLSKEY) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PITSpec) DeepCopyInto(out *PITSpec) { *out = *in diff --git a/config/samples/onpremdb/cdb.yaml b/config/samples/multitenant/cdb.yaml similarity index 100% rename from config/samples/onpremdb/cdb.yaml rename to config/samples/multitenant/cdb.yaml diff --git a/config/samples/onpremdb/cdb_secret.yaml b/config/samples/multitenant/cdb_secret.yaml similarity index 100% rename from config/samples/onpremdb/cdb_secret.yaml rename to config/samples/multitenant/cdb_secret.yaml diff --git a/config/samples/onpremdb/pdb_clone.yaml b/config/samples/multitenant/pdb_clone.yaml similarity index 100% rename from config/samples/onpremdb/pdb_clone.yaml rename to config/samples/multitenant/pdb_clone.yaml diff --git a/config/samples/onpremdb/pdb_create.yaml b/config/samples/multitenant/pdb_create.yaml similarity index 100% rename from config/samples/onpremdb/pdb_create.yaml rename to config/samples/multitenant/pdb_create.yaml diff --git a/config/samples/onpremdb/pdb_delete.yaml b/config/samples/multitenant/pdb_delete.yaml similarity index 100% rename from config/samples/onpremdb/pdb_delete.yaml rename to config/samples/multitenant/pdb_delete.yaml diff --git a/config/samples/onpremdb/pdb_modify.yaml b/config/samples/multitenant/pdb_modify.yaml similarity index 100% rename from config/samples/onpremdb/pdb_modify.yaml rename to config/samples/multitenant/pdb_modify.yaml diff --git a/config/samples/onpremdb/pdb_plug.yaml b/config/samples/multitenant/pdb_plug.yaml similarity index 100% rename from config/samples/onpremdb/pdb_plug.yaml rename to config/samples/multitenant/pdb_plug.yaml diff --git a/config/samples/onpremdb/pdb_secret.yaml b/config/samples/multitenant/pdb_secret.yaml similarity index 100% rename from config/samples/onpremdb/pdb_secret.yaml rename to config/samples/multitenant/pdb_secret.yaml diff --git a/config/samples/onpremdb/pdb_unplug.yaml b/config/samples/multitenant/pdb_unplug.yaml similarity index 100% rename from config/samples/onpremdb/pdb_unplug.yaml rename to config/samples/multitenant/pdb_unplug.yaml diff --git a/controllers/database/cdb_controller.go b/controllers/database/cdb_controller.go index 9be4477b..5c9bc3ee 100644 --- a/controllers/database/cdb_controller.go +++ b/controllers/database/cdb_controller.go @@ -283,7 +283,8 @@ func (r *CDBReconciler) validateORDSPods(ctx context.Context, req ctrl.Request, if pod.Status.Phase == corev1.PodRunning { // Get ORDS Status out, err := dbcommons.ExecCommand(r, r.Config, pod.Name, pod.Namespace, "", ctx, req, false, "bash", "-c", getORDSStatus) - if strings.Contains(out, "HTTP/1.1 200 OK") || strings.Contains(strings.ToUpper(err.Error()), "HTTP/1.1 200 OK") { + if strings.Contains(out, "HTTP/1.1 200 OK") || strings.Contains(strings.ToUpper(err.Error()), "HTTP/1.1 200 OK") || + strings.Contains(out, "HTTP/2") || strings.Contains(strings.ToUpper(err.Error()), " HTTP/2") { readyPods++ } else if strings.Contains(out, "HTTP/1.1 404 Not Found") || strings.Contains(strings.ToUpper(err.Error()), "HTTP/1.1 404 NOT FOUND") { // Check if DB connection parameters are correct @@ -348,6 +349,36 @@ func (r *CDBReconciler) createPodSpec(cdb *dbapi.CDB) corev1.PodSpec { }, }, }, + /***/ + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cdb.Spec.CDBTlsKey.Secret.SecretName, + }, + Items: []corev1.KeyToPath{ + { + Key: cdb.Spec.CDBTlsKey.Secret.Key, + Path: cdb.Spec.CDBTlsKey.Secret.Key, + }, + }, + }, + }, + + { + Secret: &corev1.SecretProjection{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cdb.Spec.CDBTlsCrt.Secret.SecretName, + }, + Items: []corev1.KeyToPath{ + { + Key: cdb.Spec.CDBTlsCrt.Secret.Key, + Path: cdb.Spec.CDBTlsCrt.Secret.Key, + }, + }, + }, + }, + + /***/ { Secret: &corev1.SecretProjection{ LocalObjectReference: corev1.LocalObjectReference{ @@ -418,6 +449,18 @@ func (r *CDBReconciler) createPodSpec(cdb *dbapi.CDB) corev1.PodSpec { Name: "ORACLE_HOST", Value: cdb.Spec.DBServer, }, + { + Name: "TESTVAR", + Value: cdb.Spec.TestVariable, + }, + { + Name: "TLSCRT", + Value: cdb.Spec.CDBTlsCrt.Secret.Key, + }, + { + Name: "TLSKEY", + Value: cdb.Spec.CDBTlsKey.Secret.Key, + }, { Name: "ORACLE_PORT", Value: strconv.Itoa(cdb.Spec.DBPort), diff --git a/controllers/database/pdb_controller.go b/controllers/database/pdb_controller.go index 42a21b51..099cb921 100644 --- a/controllers/database/pdb_controller.go +++ b/controllers/database/pdb_controller.go @@ -41,7 +41,10 @@ package controllers import ( "bytes" "context" + "crypto/tls" + "crypto/x509" "encoding/json" + //"encoding/pem" "errors" "fmt" "io/ioutil" @@ -401,7 +404,65 @@ func (r *PDBReconciler) callAPI(ctx context.Context, req ctrl.Request, pdb *dbap log := r.Log.WithValues("callAPI", req.NamespacedName) var err error - httpclient := &http.Client{} + + secret := &corev1.Secret{} + + err = r.Get(ctx, types.NamespacedName{Name: pdb.Spec.PDBTlsKey.Secret.SecretName, Namespace: pdb.Namespace}, secret) + + if err != nil { + if apierrors.IsNotFound(err) { + log.Info("Secret not found:" + pdb.Spec.PDBTlsKey.Secret.SecretName) + return "", err + } + log.Error(err, "Unable to get the secret.") + return "", err + } + rsaKeyPEM := secret.Data[pdb.Spec.PDBTlsKey.Secret.Key] + + err = r.Get(ctx, types.NamespacedName{Name: pdb.Spec.PDBTlsCrt.Secret.SecretName, Namespace: pdb.Namespace}, secret) + + if err != nil { + if apierrors.IsNotFound(err) { + log.Info("Secret not found:" + pdb.Spec.PDBTlsCrt.Secret.SecretName) + return "", err + } + log.Error(err, "Unable to get the secret.") + return "", err + } + + rsaCertPEM := secret.Data[pdb.Spec.PDBTlsCrt.Secret.Key] + + err = r.Get(ctx, types.NamespacedName{Name: pdb.Spec.PDBTlsCat.Secret.SecretName, Namespace: pdb.Namespace}, secret) + + if err != nil { + if apierrors.IsNotFound(err) { + log.Info("Secret not found:" + pdb.Spec.PDBTlsCat.Secret.SecretName) + return "", err + } + log.Error(err, "Unable to get the secret.") + return "", err + } + + caCert := secret.Data[pdb.Spec.PDBTlsCat.Secret.Key] + + r.Recorder.Eventf(pdb, corev1.EventTypeWarning, "ORDSINFO", string(rsaKeyPEM)) + r.Recorder.Eventf(pdb, corev1.EventTypeWarning, "ORDSINFO", string(rsaCertPEM)) + r.Recorder.Eventf(pdb, corev1.EventTypeWarning, "ORDSINFO", string(caCert)) + + certificate, err := tls.X509KeyPair([]byte(rsaCertPEM), []byte(rsaKeyPEM)) + if err != nil { + pdb.Status.Msg = "Error tls.X509KeyPair" + return "", err + } + + caCertPool := x509.NewCertPool() + caCertPool.AppendCertsFromPEM(caCert) + + tlsConf := &tls.Config{Certificates: []tls.Certificate{certificate}, RootCAs: caCertPool} + + tr := &http.Transport{TLSClientConfig: tlsConf} + + httpclient := &http.Client{Transport: tr} log.Info("Issuing REST call", "URL", url, "Action", action) @@ -411,7 +472,7 @@ func (r *PDBReconciler) callAPI(ctx context.Context, req ctrl.Request, pdb *dbap } // Get Web Server User - secret := &corev1.Secret{} + //secret := &corev1.Secret{} err = r.Get(ctx, types.NamespacedName{Name: cdb.Spec.WebServerUser.Secret.SecretName, Namespace: cdb.Namespace}, secret) if err != nil { if apierrors.IsNotFound(err) { @@ -440,7 +501,7 @@ func (r *PDBReconciler) callAPI(ctx context.Context, req ctrl.Request, pdb *dbap if action == "GET" { httpreq, err = http.NewRequest(action, url, nil) } else { - //fmt.Println("payload:", payload) + fmt.Println("payload:", payload) jsonValue, _ := json.Marshal(payload) httpreq, err = http.NewRequest(action, url, bytes.NewBuffer(jsonValue)) } @@ -456,12 +517,14 @@ func (r *PDBReconciler) callAPI(ctx context.Context, req ctrl.Request, pdb *dbap resp, err := httpclient.Do(httpreq) if err != nil { + errmsg := err.Error() log.Error(err, "Failed - Could not connect to ORDS Pod", "err", err.Error()) - pdb.Status.Msg = "Could not connect to ORDS Pod" - r.Recorder.Eventf(pdb, corev1.EventTypeWarning, "ORDSError", "Failed: Could not connect to ORDS Pod") + pdb.Status.Msg = "Error: Could not connect to ORDS Pod" + r.Recorder.Eventf(pdb, corev1.EventTypeWarning, "ORDSError", errmsg) return "", err } + r.Recorder.Eventf(pdb, corev1.EventTypeWarning, "Done", pdb.Spec.CDBResName) if resp.StatusCode != http.StatusOK { bb, _ := ioutil.ReadAll(resp.Body) @@ -563,7 +626,7 @@ func (r *PDBReconciler) createPDB(ctx context.Context, req ctrl.Request, pdb *db values["tdeSecret"] = tdeSecret } - url := "http://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + url := "https://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" pdb.Status.TotalSize = pdb.Spec.TotalSize pdb.Status.Phase = pdbPhaseCreate @@ -618,7 +681,7 @@ func (r *PDBReconciler) clonePDB(ctx context.Context, req ctrl.Request, pdb *dba values["tempSize"] = pdb.Spec.TempSize } - url := "http://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + pdb.Spec.SrcPDBName + "/" + url := "https://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + pdb.Spec.SrcPDBName + "/" pdb.Status.Phase = pdbPhaseClone pdb.Status.Msg = "Waiting for PDB to be cloned" @@ -685,7 +748,7 @@ func (r *PDBReconciler) plugPDB(ctx context.Context, req ctrl.Request, pdb *dbap values["asClone"] = strconv.FormatBool(*(pdb.Spec.AsClone)) } - url := "http://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + url := "https://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" pdb.Status.TotalSize = pdb.Spec.TotalSize pdb.Status.Phase = pdbPhasePlug @@ -741,7 +804,7 @@ func (r *PDBReconciler) unplugPDB(ctx context.Context, req ctrl.Request, pdb *db values["tdeExport"] = strconv.FormatBool(*(pdb.Spec.TDEExport)) } - url := "http://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + pdb.Spec.PDBName + "/" + url := "https://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + pdb.Spec.PDBName + "/" pdb.Status.Phase = pdbPhaseUnplug pdb.Status.Msg = "Waiting for PDB to be unplugged" @@ -806,7 +869,7 @@ func (r *PDBReconciler) modifyPDB(ctx context.Context, req ctrl.Request, pdb *db "getScript": strconv.FormatBool(*(pdb.Spec.GetScript))} pdbName := pdb.Spec.PDBName - url := "http://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + pdbName + "/status" + url := "https://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + pdbName + "/status" pdb.Status.Phase = pdbPhaseModify pdb.Status.ModifyOption = pdb.Spec.PDBState + "-" + pdb.Spec.ModifyOption @@ -842,7 +905,7 @@ func (r *PDBReconciler) getPDBState(ctx context.Context, req ctrl.Request, pdb * } pdbName := pdb.Spec.PDBName - url := "http://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + pdbName + "/status" + url := "https://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + pdbName + "/status" pdb.Status.Msg = "Getting PDB state" if err := r.Status().Update(ctx, pdb); err != nil { @@ -882,7 +945,7 @@ func (r *PDBReconciler) mapPDB(ctx context.Context, req ctrl.Request, pdb *dbapi } pdbName := pdb.Spec.PDBName - url := "http://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + pdbName + "/" + url := "https://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + pdbName + "/" pdb.Status.Msg = "Mapping PDB" if err := r.Status().Update(ctx, pdb); err != nil { @@ -1015,7 +1078,7 @@ func (r *PDBReconciler) deletePDBInstance(req ctrl.Request, ctx context.Context, } pdbName := pdb.Spec.PDBName - url := "http://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + pdbName + "/" + url := "https://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + pdbName + "/" pdb.Status.Phase = pdbPhaseDelete pdb.Status.Msg = "Waiting for PDB to be deleted" diff --git a/docs/onpremdb/README.md b/docs/multitenant/README.md similarity index 59% rename from docs/onpremdb/README.md rename to docs/multitenant/README.md index 3c77c1b4..be311094 100644 --- a/docs/onpremdb/README.md +++ b/docs/multitenant/README.md @@ -1,19 +1,23 @@ -# Oracle On-Prem Database Controller + -CDBs and PDBs are the part of the Oracle Database's [Multitenant Architecture](https://docs.oracle.com/en/database/oracle/oracle-database/21/multi/introduction-to-the-multitenant-architecture.html#GUID-AB84D6C9-4BBE-4D36-992F-2BB85739329F). The On-Prem Database Controller is a feature of Oracle DB Operator for Kubernetes (OraOperator) which helps to manage the life cycle of Pluggable Databases (PDBs) in an Oracle Container Database(CDB). -The target CDB (for which the PDB life cycle management is needed) can be running on an on-prem machine and to manage its PDBs, the Oracle DB Operator can run on an on-prem Kubernetes system (For Example: [Oracle Linux Cloud Native Environment or OLCNE](https://docs.oracle.com/en/operating-systems/olcne/)). + +# Oracle Multitenant Database Controller + +CDBs and PDBs are the part of the Oracle Database's [Multitenant Architecture](https://docs.oracle.com/en/database/oracle/oracle-database/21/multi/introduction-to-the-multitenant-architecture.html#GUID-AB84D6C9-4BBE-4D36-992F-2BB85739329F). The Multitenant Database Controller is a feature of Oracle DB Operator for Kubernetes (OraOperator) which helps to manage the life cycle of Pluggable Databases (PDBs) in an Oracle Container Database(CDB). + +The target CDB (for which the PDB life cycle management is needed) can be running on an multitenant machine and to manage its PDBs, the Oracle DB Operator can run on an multitenant Kubernetes system (For Example: [Oracle Linux Cloud Native Environment or OLCNE](https://docs.oracle.com/en/operating-systems/olcne/)). NOTE: The target CDB (for which the PDB life cycle management is needed) **can also** run in a Cloud environment as well (For Example: OCI's [Oracle Base Database Service](https://docs.oracle.com/en-us/iaas/dbcs/doc/bare-metal-and-virtual-machine-db-systems.html)) and to manage its PDBs, the Oracle DB Operator can run on a Kubernetes Cluster running in cloud (For Example: OCI's [Container Engine for Kubernetes or OKE](https://docs.oracle.com/en-us/iaas/Content/ContEng/Concepts/contengoverview.htm#Overview_of_Container_Engine_for_Kubernetes)) -# Oracle DB Operator On-Prem Database Controller Deployment +# Oracle DB Operator Multitenant Database Controller Deployment To deploy OraOperator, use this [Oracle Database Operator for Kubernetes](https://github.com/oracle/oracle-database-operator/blob/main/README.md) step-by-step procedure. -After the Oracle Database Operator is deployed, you can see the DB Operator Pods running in the Kubernetes Cluster. As part of the OraOperator deployment, the On-Prem Database Controller is deployed and we can see the CRDs (Custom Resource Definition) for CDB and PDB in the list of CRDs. The following output is an example of such a deployment: -``` +After the Oracle Database Operator is deployed, you can see the DB Operator Pods running in the Kubernetes Cluster. As part of the OraOperator deployment, the multitenant Database Controller is deployed and we can see the CRDs (Custom Resource Definition) for CDB and PDB in the list of CRDs. The following output is an example of such a deployment: +```bash [root@test-server oracle-database-operator]# kubectl get ns NAME STATUS AGE cert-manager Active 32h @@ -65,9 +69,9 @@ singleinstancedatabases.database.oracle.com 2022-06-22T01:21:40Z The following sections explain the setup and functionality of this controller. -# Prerequsites to manage PDB Life Cycle using Oracle DB Operator On-Prem Database Controller +# Prerequsites to manage PDB Life Cycle using Oracle DB Operator Multitenant Database Controller -Before you want to manage the life cycle of a PDB in a CDB using the Oracle DB Operator On-Prem Database Controller, complete the following steps. +Before you want to manage the life cycle of a PDB in a CDB using the Oracle DB Operator Multitenant Database Controller, complete the following steps. **CAUTION :** You must make the changes specified in this section before you proceed to the next section. @@ -86,90 +90,98 @@ You cannot have an ORDS enabled schema in the container database. To perform the Create the CDB administrator user and grant the required privileges. In this example, the user is `C##DBAPI_CDB_ADMIN`. However, any suitable common user name can be used. -Also, create a user named SQL_ADMIN. You can use the below set of SQL commands for this purpose: - -``` +```SQL SQL> conn /as sysdba -- Create the below users at the database level: -DROP USER SQL_ADMIN CASCADE; -CREATE USER SQL_ADMIN IDENTIFIED BY welcome1; ALTER SESSION SET "_oracle_script"=true; DROP USER C##DBAPI_CDB_ADMIN cascade; CREATE USER C##DBAPI_CDB_ADMIN IDENTIFIED BY CONTAINER=ALL ACCOUNT UNLOCK; GRANT SYSOPER TO C##DBAPI_CDB_ADMIN CONTAINER = ALL; GRANT SYSDBA TO C##DBAPI_CDB_ADMIN CONTAINER = ALL; GRANT CREATE SESSION TO C##DBAPI_CDB_ADMIN CONTAINER = ALL; -GRANT CREATE SESSION TO SQL_ADMIN; -GRANT SYSDBA TO SQL_ADMIN; - + -- Verify the account status of the below usernames. They should not be in locked status: col username for a30 col account_status for a30 -select username, account_status from dba_users where username in ('ORDS_PUBLIC_USER','SQL_ADMIN','C##DBAPI_CDB_ADMIN','APEX_PUBLIC_USER','APEX_REST_PUBLIC_USER'); +select username, account_status from dba_users where username in ('ORDS_PUBLIC_USER','C##DBAPI_CDB_ADMIN','APEX_PUBLIC_USER','APEX_REST_PUBLIC_USER'); ``` - - ### Reference Setup: Example of a setup using OCI OKE(Kubernetes Cluster) and a CDB in Cloud (OCI Exadata Database Cluster) Please refer [here](./provisioning/example_setup_using_oci_oke_cluster.md) for steps to configure a Kubernetes Cluster and a CDB. This example uses an OCI OKE Cluster as the Kubernetes Cluster and a CDB in OCI Exadata Database service. - + ## Oracle REST Data Service (ORDS) Image - Oracle DB Operator On-Prem Database controller requires the Oracle REST Data Services (ORDS) image for PDB Lifecycle Management in the target CDB. + Oracle DB Operator Multitenant Database controller requires the Oracle REST Data Services (ORDS) image for PDB Lifecycle Management in the target CDB. You can build this image by using the ORDS [Dockerfile](../../../ords/Dockerfile) - - > **_NOTE:_** Download the required binaries to build this image i.e. Oracle Rest Data Services 'ords-< version >.zip', from http://www.oracle.com/technetwork/developer-tools/rest-data-services/downloads/index.html - - > **_NOTE:_** The current version of Oracle DB Operator On-Prem Controller has been tested with `ORDS 21.4.3` version. + + > **_NOTE:_** The current version of Oracle DB Operator Multitenant Controller has been tested with **ords 22.2.1.202.1302** version. -Please refer [here](./provisioning/ords_image.md) for the steps to build ORDS Docker Image with `ORDS 21.4.3` version. +Please refer [here](./provisioning/ords_image.md) for the steps to build ORDS Docker Image with **ords 22.2.1.202.1302** version. + ## Kubernetes Secrets - Oracle DB Operator On-Prem Database Controller uses Kubernetes Secrets to store usernames and passwords to manage the life cycle operations of a PDB in the target CDB. + Oracle DB Operator Multitenant Database Controller uses Kubernetes Secrets to store usernames and passwords to manage the life cycle operations of a PDB in the target CDB. In addition to that ,in order to use https protocol, all certificates need to be stored using Kubernests Secret. ### Secrets for CDB CRD - Create a secret file as shown here: [config/samples/onpremdb/cdb_secret.yaml](../../config/samples/onpremdb/cdb_secret.yaml). Modify this file with the `base64` encoded values of the required passwords for CDB and use it to create the required secrets. + Create a secret file as shown here: [config/samples/multitenant/cdb_secret.yaml](../../config/samples/multitenant/cdb_secret.yaml). Modify this file with the `base64` encoded values of the required passwords for CDB and use it to create the required secrets. - ```sh - $ kubectl apply -f cdb_secret.yaml + ```bash + kubectl apply -f cdb_secret.yaml ``` **Note:** In order to get the base64 encoded value for a password, please use the below command like below at the command prompt. The value you get is the base64 encoded value for that password string. - ```sh + ```bash echo -n "" | base64 ``` - **Note:** On successful creation of the CDB Resource, the CDB secrets would be deleted from the Kubernetes system. + **Note:** On successful creation of the CDB Resource, the CDB secrets would be deleted from the Kubernetes . ### Secrets for PDB CRD - Create a secret file as shown here: [config/samples/onpremdb/pdb_secret.yaml](../../config/samples/onpremdb/pdb_secret.yaml). Modify this file with the `base64` encoded values of the required passwords for PDB and use it to create the required secrets. + Create a secret file as shown here: [config/samples/multitenant/pdb_secret.yaml](../../config/samples/multitenant/pdb_secret.yaml). Modify this file with the `base64` encoded values of the required passwords for PDB and use it to create the required secrets. - ```sh - $ kubectl apply -f pdb_secret.yaml + ```bash + kubectl apply -f pdb_secret.yaml ``` **NOTE:** Refer to command provided above to encode the password using base64. - **NOTE:** Don't leave plaintext files containing sensitive data on disk. After loading the Secret, remove the plaintext file or move it to secure storage. + **NOTE:** Don't leave plaintext files containing sensitive data on disk. After loading the Secret, remove the plaintext file or move it to secure storage. +### Secrets for CERTIFICATES + +Create certificates and key on your local host and use them to create Kubernet secret + +```bash +genrsa -out ca.key 2048 +openssl req -new -x509 -days 365 -key ca.key -subj "/C=CN/ST=GD/L=SZ/O=oracle, Inc./CN=oracle Root CA" -out ca.crt +openssl req -newkey rsa:2048 -nodes -keyout tls.key -subj "/C=CN/ST=GD/L=SZ/O=oracle, Inc./CN=cdb-dev-ords" -out server.csr +/usr/bin/echo "subjectAltName=DNS:cdb-dev-ords,DNS:www.example.com" > extfile.txt +openssl x509 -req -extfile extfile.txt -days 365 -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out tls.crt +``` + +```bash +kubectl create secret tls db-tls --key="tls.key" --cert="tls.crt" -n oracle-database-operator-system +kubectl create secret generic db-ca --from-file=ca.crt -n oracle-database-operator-system +``` + +image_not_found +**Note:** On successful creation of the certificates secret creation remove files or move to secure storage . + ## Kubernetes CRD for CDB -The Oracle Database Operator On-Prem Controller creates the CDB kind as a custom resource that models a target CDB as a native Kubernetes object. This is used only to create Pods to connect to the target CDB to perform PDB-LM operations. These CDB resources can be scaled up and down based on the expected load using replicas. Each CDB resource follows the CDB CRD as defined here: [config/crd/bases/database.oracle.com_cdbs.yaml](../../config/crd/bases/database.oracle.com_cdbs.yaml) +The Oracle Database Operator Multitenant Controller creates the CDB kind as a custom resource that models a target CDB as a native Kubernetes object. This is used only to create Pods to connect to the target CDB to perform PDB-LM operations. These CDB resources can be scaled up and down based on the expected load using replicas. Each CDB resource follows the CDB CRD as defined here: [config/crd/bases/database.oracle.com_cdbs.yaml](../../config/crd/bases/database.oracle.com_cdbs.yaml) -To create a CDB CRD, a sample .yaml file is available here: [config/samples/onpremdb/cdb.yaml](../../config/samples/onpremdb/cdb.yaml) +To create a CDB CRD, a sample .yaml file is available here: [config/samples/multitenant/cdb.yaml](../../config/samples/multitenant/cdb.yaml) -**Note:** The password and username fields in this `cdb.yaml` yaml are Kubernetes Secrets created earlier. Please see the section [Kubernetes Secrets](ORACLE_ONPREMDB_CONTROLLER_README.md#kubernetes-secrets) for more information. Please see [Kubernetes Private Registry Documenation]( https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) for creating secrets for pulling images from docker private registry. +**Note:** The password and username fields in this *cdb.yaml* yaml are Kubernetes Secrets created earlier. Please see the section [Kubernetes Secrets](https://kubernetes.io/docs/concepts/configuration/secret/) for more information. Please see [Kubernetes Private Registry Documenation]( https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/) for creating secrets for pulling images from docker private registry. 1. [Use Case: Create a CDB CRD Resource](./provisioning/cdb_crd_resource.md) 2. [Use Case: Add another replica to an existing CDB CRD Resource](./provisioning/add_replica.md) @@ -177,13 +189,13 @@ To create a CDB CRD, a sample .yaml file is available here: [config/samples/onpr + ## Kubernetes CRD for PDB -The Oracle Database Operator On-Prem Controller creates the PDB kind as a custom resource that models a PDB as a native Kubernetes object. There is a one-to-one mapping between the actual PDB and the Kubernetes PDB Custom Resource. You cannot have more than one Kubernetes resource for a target PDB. This PDB resource can be used to perform PDB-LM operations by specifying the action attribute in the PDB Specs. Each PDB resource follows the PDB CRD as defined here: [config/crd/bases/database.oracle.com_pdbs.yaml](../../../config/crd/bases/database.oracle.com_pdbs.yaml) +The Oracle Database Operator Multitenant Controller creates the PDB kind as a custom resource that models a PDB as a native Kubernetes object. There is a one-to-one mapping between the actual PDB and the Kubernetes PDB Custom Resource. You cannot have more than one Kubernetes resource for a target PDB. This PDB resource can be used to perform PDB-LM operations by specifying the action attribute in the PDB Specs. Each PDB resource follows the PDB CRD as defined here: [config/crd/bases/database.oracle.com_pdbs.yaml](../../../config/crd/bases/database.oracle.com_pdbs.yaml) -To create a PDB CRD Resource, a sample .yaml file is available here: [config/samples/onpremdb/pdb_create.yaml](../../config/samples/onpremdb/pdb_create.yaml) +To create a PDB CRD Resource, a sample .yaml file is available here: [config/samples/multitenant/pdb_create.yaml](../../config/samples/multitenant/pdb_create.yaml) -# Use Cases for PDB Lifecycle Management Operations using Oracle DB Operator On-Prem Controller +# Use Cases for PDB Lifecycle Management Operations using Oracle DB Operator Multitenant Controller -Using Oracle DB Operator On-Prem Controller, you can perform the following PDB-LM operations: CREATE, CLONE, MODIFY, DELETE, UNPLUG, PLUG. +Using Oracle DB Operator Multitenant Controller, you can perform the following PDB-LM operations: CREATE, CLONE, MODIFY, DELETE, UNPLUG, PLUG. 1. [Create PDB](./provisioning/create_pdb.md) 2. [Clone PDB](./provisioning/clone_pdb.md) @@ -200,4 +212,4 @@ Please check [here](./provisioning/validation_error.md) for the details to look ## Known issues -Please refer [here](./provisioning/known_issues.md) for the known issues related to Oracle DB Operator On-Prem Controller. +Please refer [here](./provisioning/known_issues.md) for the known issues related to Oracle DB Operator Multitenant Controller. diff --git a/docs/onpremdb/images/K8S_SECURE1.png b/docs/multitenant/images/K8S_SECURE1.png similarity index 100% rename from docs/onpremdb/images/K8S_SECURE1.png rename to docs/multitenant/images/K8S_SECURE1.png diff --git a/docs/onpremdb/images/K8S_SECURE2.png b/docs/multitenant/images/K8S_SECURE2.png similarity index 100% rename from docs/onpremdb/images/K8S_SECURE2.png rename to docs/multitenant/images/K8S_SECURE2.png diff --git a/docs/onpremdb/images/K8S_SECURE3.png b/docs/multitenant/images/K8S_SECURE3.png similarity index 100% rename from docs/onpremdb/images/K8S_SECURE3.png rename to docs/multitenant/images/K8S_SECURE3.png diff --git a/docs/onpremdb/images/K8S_SECURE4.png b/docs/multitenant/images/K8S_SECURE4.png similarity index 100% rename from docs/onpremdb/images/K8S_SECURE4.png rename to docs/multitenant/images/K8S_SECURE4.png diff --git a/docs/multitenant/openssl_schema.jpg b/docs/multitenant/openssl_schema.jpg new file mode 100644 index 0000000000000000000000000000000000000000..4453d52f27eaa83c6891a9ba6c8bef418b89ec8e GIT binary patch literal 54221 zcmeFYXH-*L`!9+O6#=D7bwdc!0@Ax~DS?D0p$VZ#3j_#AZ-Q=>-U9?e-IN4El@LM| zlpd;t(2Gh_ssaK6f`|Q{_uTW|`{9mrzTETS{+~6*T5GH|pZQyJ&fgq!K4tx$`u&OS z78DEt(_OegM|a`;p!+>br$u+^;y?1w`O>*uroa4;T%)JILVx`l1H<)e*RL}$-ehFB z!F=QTbtV=j=9{-}vD{){WMyN$#ddyw>z_(4{G)m4^0jltTQ{!XIDh)TDZk&*v0S@& z^U{M$7li08vRt^ta^ZIu9sfU{dg;PHLicaEdWHVlQar7L3?;Q8yKluLxosRAb{iVxSFJ8EI z{#Kjiyb~{7xq8Ttzu1HvQm+=W17u;f??*pte&-Z6 zrS5QW$(uR)zC;%hJ31GC&(JYmI>%tS#6tIg?$q>u;{2bwmvH{pbefZBZ0>h(xBaf((-Y+Ap8iFrmU3CnmkDXsd@$M? zs#`XpCj+2dN+B$8S|jO5#Vi-1g@W1{hStZkEuSu39INO zb~D6tCI;=&_3KbNYQZVSc6{D6@L|7Qk+XK)eYgmPR{Z?p?%m(w+h1+AfTe+O0s;B~&x@{@Ym%})Zs}OGmMP9Ll#Z~hZtldxQ28|)#v-Tn@49iMQbwia4ZtoU|(b0L3|4;FQujmY( zK4rbW9e{QcE$QQtb;K{XC&OO-3bq3;lE<1wl#fvK}&nZ^>J` zC!t+%ak;K9kSIQ5;X?I5T_H$n{;K<~94O|=;*urFnvjF^3ddT)o5~c*ZJ*Khw5s}m zRVI&)jeWs$2C-st0HKGRv#E@j{DW9r2S6!)JxUCevl(9Cg7**Js#=ks-ZZR9MW@xW zJY%H5y;lzA-SMZP$WlnYBlajbH^>pdz3M0)SH~Y>N+=&vjj6Cq5-B@sN;s_3P>#(; zhOw7j4U~>6+ppm>7Ot$DELC|%;>g%{na5YQD;O1=g(d2I_?j+ zVQuSf3La&R=(H+}6?yI;(DVEtF~?jDwmhwn215SAk`z*7PWR-~W5plThj~GSPFTUu zk=S3?NKyMEkMn&%3VE|24_PQIgC#rR?6OV+HWt)2vF|l#*uR#qYL3m?&pPuH-3pk` zUYU;fH=nFh?mE`~O(z<)ztviNXH1gqUUFi2uW08k^6|2$agS+rW2v{L@x+#hStDLGv#t0h_3P{MPaewQK) zwP)Q0jrh%O&Y2%lio!C$a&l%ThNiJUu!9`~#BP(1vBN@Q$-#Izf!f=#?#HF&!K-Me zof_}OY2(oT_i=H}{f(y~WYc`PwO$8$B&3oeVUnGmjuVBY7YWG;NZz%o&23IPSY2qI zWc1@q@%bj`yj9OMv$O`J!jpjAZ}7AkYLO4K7(F=RVA$N^iY|1A!JW->!QL=GB?*;Utot&OD)dW^b&AbKze=Rgl<-L>46l2p!a4qa}%ynbRynDGdTYsrJHbILv7bpS_u zInL?^-f#y9$(pWNk-^|1P@+AX^sstlWj46X3K(x)7~8_j{F|E1xM7#607P?sX zl{=eLTYqYh>T+;gmQ}N8E3^ z2OYqA(8P6p#hg#>!rQ8OkIgkWFWtS-xmNTrZXCApxyR)?>N-T;P4oG;p9Np!F4U|= zyBYG2Z}N6ANF;xzth{Xi_s z;V|??$o)HqawdxI4C{5kWCy#aPABs#YuALy@4wIMdk!AE?3nxlznX4&5Un-6ce}9f zY2_}y_hMJMMD^l8rzdrdf{^9*VM#II!EchszwqUP5|YCcg2N7qkXlFC z>EXv|VPcKRg!~`ohQwz8<&s^ksI%3X9D9C@W^$=@ZNhpuS#N?r)zNaCJ-OE*o`Ekl zUO~0Kfzs7B20Z14ScwUzStmN%y#4rMJnY{1^ygzyGSOO1d`)Ioo#?ovafOQZR0)5w z2h~>(jR%rQAU+s^R3(yk#Ca@*`C>4?kzXm^!bMZ{cV(j0Yq_|*u=_K?OSi_F-+CdH zg{ihXLfebLm+WY?>U?oepAhPUcJ}-JE9j9-i~f4G?O8QP#m&LkuAdd<{p=pzPP_ND z?HsJmO9iD5y(iRP;w?a1pp#JQa0~vt+vqUSIR|kimRc;!E(F#1^5CgsTPZ))P`2@d zO_Yf3UuNPQZwF^|g}Mls7{nlfao8F{c)PU#4+>~>zia5R)Ya9_P)$RRkX*(4WYibg zXdzXdSP(FRaZcPI#I)!!@l^uZAsuVjxKSlW5er|{*e=GulHXzeyIW1)Ef852lx?X) z2wrN^;ZGgsN6vPsu-O|p-~Sefl;TZFbH)`8p~2DIAtH946A4?;v8URzjZ6s7{C;T#y<<%~a96vbKdz6JlS_#>x zeDU18B8GM-LNO~6V#5$-vu^SP(R&*ehK6II>Ud73fHLLr2#7O8*=)Be%iTX;ml*0P z=Dc_)tI=3;IOSQ4-ghLPrI``2$03O;WW{OzA@z7h#J8&lWTg~BR1ql_Wu=&;usUm4 z(h&0FyYB;8=R<(K_5XK^@9V(y(N@)2{^s%E$FH%|I5Ug9SP{wI96v;Q!Trm`hu?L9 zUdWI0w?vQYN}I7Ts=~tMgur7Nc7*S3a3=fiwfN)^w|2hYbWpvKvzRTL-*ms}ZtWVM zn#ld8^Q}E9JGFg=1cgTsE)#d@zSGk(d| zD%tAG!_ONWO2s|n&QT~zWaiD;qL9tM@$?;Pj4#7Edv_KXmQ@v=0mC&EdwTfvFBRLe*0mqmknta7Oiq$8|qjIcu0U zBtR5OsRIom_lytJqYbb3x_YlW-TqCt$^V<~W25L&j#s95Q$-VpF(fOwMEhrms>rNY zwmfDvYlB5@ANzuyP~bZHChLPxHz6Y^Jm`-_)3Gjj+*&fLp*{>#EKC}31Hh9if8vC- zZ7}#*yEh5EHohFWb2;#c)!1e3lLss%Q7-I5_w~Lhg7lPrVt>AFE+pYjO{$7x5R( z|AUJ+|ASHtW!EkLgNy3_lfCc6{|4Iq2N(Z?y>#U-GX8^$ul|F*7t8+(>fh7<|BoGr zsW9Bflw^nTZ*h4bHe9|bWjJM^r~MD*_O=;2te+*sQ|uAH>Fy*2U8Zqo9+v&!J zs~Lw`hQ7!==|Y-d5b63+zvxe7)U&9ez)GKmot*<+ck+PhEN%YnNYHyc&21YCbC4KoRCaNFBVf$6s7z6r zv^?|amIwfi4A$M(sr7WJMGdOfO>+r%P&Q75&EnUcw;M$B?Zs--cFvOOe$&~Y_of*R z&n|9O(4+w-jko`waFJc~a2uTYGNt2!z!%nB$11svz#IPsBa0`%5yxC0h_b%O;dn=XS zq-$fV4;s?^@H;$iK~KNCN}HK*1`KhCZ{;Gd0-5FCQz8u5D{1*g?1&V*LxTa^&ixTM z5u)ljH}{3VBJc}W*A`QYZtC?A#@Oo0(lis(E0Y2yKw-3&w9E$uoR_a=yn7E*j41+} z-()^tPGM3g`{1m&5>Qfp_~El-x;8koSa~*b8OM?;;RKM?=#EN4m-66hdPSco&=bog z9u@Yuk+m2VUuwBQBAju;I%k*IB(eeSQRUt7D5tyk(?mDL0)z}KEbW;HKF zI3ZAZQXMd@{3*s+w>BhobDFGG_Shn2sfm5d3E9G%bB$Vwm{zm0NGC2Uah5%`4HR0E zt0O>D7to98=~Z>X=_LfE?-wXmmTq&^0=lmZ$_MzXt5a_qH9z*N}K#xk_-_}mBa@IqoJuPxI=rB@rJbhGSxj=h^M%4_C*}2-x#{b(j;a$GRUo3f;w+5z+RqvWwNiX(f&~gc zT&BfaLqYc`QnEI9BL#Eja?>#{p(?S6OJYcH!>j_(W#~#%>AYpD&hqmM{a#-{71t}9 zs%ZNfuVz=iRrM^;GAaEr3jqp5hGHP4yzv-@+Qk`kD8UV{+%|wq+MZyKJbL(bz!fIlEQl}CB zCh`8|2@@gtvKruoJyNIHBC=#i9Flr2o6b)({`|KgbZNle74FrsV*VJMJ>dP8!%H_r z3MehRm>p0jk4*(J3%bD~#9(H+wW%rl|5Dj)aJAk_`!VzwZ2KC)lm1J)H7x{hYZ{B{ z21GzKX#*u0+@6S;zJ|rHt7ZRExQa2)^W?twdFV}|fKGgWGE=D?lnY2BH?T)hU7Kl! zGdYVczfi+LmQ9uHm` z^l3b~&sM5g87b7?@5Cacjr8tIIe9&cQbfzaCW@!fXdyYd2SP#u?w&8B0S?oT_vgcl zB~?9Dm&HWGK~3G_pNb!1SsMv=YmYXI9*|x*(}j3QZg^EJ&6++27tq6zq3T&9z=`~a znB@|Xh;z|{H+=^Jf74x9sjaIjjsI9x3D8WWq4G(Y=oytp5_u*>w66h1OD|~?!Le{* zQ*J)`SHo{gdZP3J3U(L4Wdl@ZOmYCwi^%S(eDm=;Kd0jM>e>oM)UD(Bq(`n)kxzm+E z|K~r|(b0uCSxpRO$TF&38==(rNDjJ`qH(>}DVeO9th&vNA4sAx2XfGuGx|UO&6)dh zBHGn4r;)Gmj@@xH`Qe8cJrte_C%Bs&0fn^@hzMzW>XX1KYFA-!N&(7)$&uU zCPHcyfUKNWj2txHS9bS7>}|#u|H3~%bxpZU;S7uA1DiAqB*ZkN5>(jWsKd|-#o5J~ zbf~35Do%|nJEpHemix*7Q2BpcZ`n;@^=C{%_*8l$R6+U;!!<6>Sb*bZOWq5)pVW)6-D z6kD&L-JmaDgK)L%yhtp4zr(!@9in8 z=GG5c8y#p?U*xs_Jd;ZeDNN6CM|nJ&`|+DjB-x?HYDXwqJYn!BhonxX+HGPi%S!h* z?Hv)VL16LJ9JmKMZPQ>Z#3}fqaM))MeZ0qAo_cSywwK4NJ62&A@N|85jld8uh}IvSKM3hfF$VT>%ffgq)&O?PNRvVc z%pzSB-!xBOsxuLn9p_mmRoU~Y>u;|_6Ax6?sGuI^TZBQAXjas9FxC1lUw)>+tfff8 z9MGHk}V`dwxSNg)kGwsht(E^qfJ`+TU}UoGuhGE7EA$`Fuk0e(NId)15PQ=G|qFx~-@;cGD~5d{uqUhqM$UbCj>Dhov3V>8v$zh7ft21;mVaCjOG|CU+pyHIFBUk#Fz)xz`hvY2wd7r9ul2O=-U@l zRzHx*+8>{J*3=wek5vQWJ^iOq)$V)*DAZs^&!o6Xkcpzit$EUHkZz!igjwG;CjEJp zt-UlfyK74=8Ni(BfNnC6DFWub7E6Zou&`3R^fg5UalOzC4jW-MG&hXG`y9H88K}Rb zMD2`=oApxHlS45}e#nsU7E)m0PW3+642y8O^XCOlT2tsNk@joP6-pW^YSbuKQgz-- zZdUS(gr(=NBYwDTLKPFwl4bRs(UupjvQ=(JJK?I7CmxiT0I043(4N{~EZ8?QmG$A$ zq$Y^hRKAy5Y|Lz;?y`~Ks=faqgp6rK_d2X_2i|>cE`BIaS$C6wkw^qgnswZJkbPX5 zPd97OgEtL(0!YNm@AY$)+t?x4QQfRzO zNSU(tyoEs*NTaGDkztsv0xRyB1MW8ih#YO7d;nD7F>@LdUh*}uIIZB#t-FFj9OU>E z%Xx))5RC>2OtSV?$=~isT+OauaJsfG*5yBE`WmV^bVI%@ra@WOe~$(GK%g=;i8`>R zkz)L^V=Cc`VRwOrUsb8oA`ac8UNlA1ZQ9mooUn>V1VjczFV|bvuP$bb1*Gw&kI0|N zhL90?1HN+&yK|QTB!LG4A7Qn~4CO1Nxj2qIwy#96hPb^2AW*ozo_VNVEUx$=TO8Vp zod3N)mT382FTZQdGut-5#M1fpgrSs|iGKE<{Vu)TGy0_*N9{jV?ume|iLLtzy`8Yw z>zWo66-w^KR}a&%3=hfUl1-iW6_iiLe^R9={Z_0yh}tzjxEWY=Sc9o-C*Hr+NHLSW z(;WtfcosoG{y8ckX_n))Yh_3mSj)hSDG=TwVCKF3!wD}J^Za>3+7Rj(?B*nsi_Vnk z^^hI(^$sgm zH=3Eo&n$J|S6!~9+kF*84U|i@v2Fwv|GIOtQ*1EZgI{y3N;Q>GD{E5+^oka7TQaq> zw4zma$d?UiDaUwGO?UHz()YXtjU^(Ay#$G$iarveUk<;F+xjXig?wThSPe0J*<_Nk zUeqU?4Vg&PMns~`ik9<|UAVnhFTMZk!_3lcThNvL<(wS7d)e;c3DIVYE0jg>MoZaX zUwWMhI^sG&7}KS#c|{;Fz&6>{?=7|*!@XgSy&IA|>?<~TD%pRbV--C4DC?8wA*(_* zq$9W)pE}CuoKEz$ntH_+0-t;s?;`TR&}A$U*gNa;ETORg*!_$QkdxxYvhV~^UcCJ3 z1<70t&JPYLk)-ZwC~A*V14OsfXysoDHKz@fR3=Lr(dZ_Ga|4FNYV4~C5tSBr4lnZw z(^p8fG@q}y9$3K@AG{B{W00AvJ)0Ewh&&K<8~J6a+7u~kBEQJ~c4l2LoRLn&m7!aq zRJ3o8VfIvK0J-JT+vkLa=2O*gSMx0%I7cDWKqg%8OUNZc0MX@{T!ZAWzV){SHM{DE z>>1@vJP56mHz~0J`$iNc>+09%=G1j>CwXpF@_g#hgM`e{Caf^|<4S#!%OlUPca`8i zj7OH!9sW5flWj9*cmdBtX~j86HoG0tN??}zX8A&C@24JzUWbWJO>87LeTgj93t4qgowvO*h{rbx4Mp?V^ zz1ujSIi7D_y=UGvy@bxw`GX4^Z}G2EVUL{-Vql-Ay94=NccuKgzvI>tZWR zWhrF)+vncuUdJcKX6EX9W?%RZV-IG~3w_n{SYiX`QX+7o5fF5pe)_q}LAHDUj?I>T z@LZJhNa-7Qo<;YnmE2f*;>Qx2n4uDPCZDb_4)U}aYT~2OI#3)XCqfZ>x*0ek*|&NK zEe}o_&H>;XR1Fli#*o(e)@eh!eFt0IMSZ&oPEeWFSGSu=@m)S&Rj$Nyrr5+a zY?);U@tF&DfX45{k2Jp8)56_46LviDaN)@(y0msd;I|(8>Jt(837a%5NC&PBlVY<=)0M29nNAHQxvJrP6{shkNYLmf9K6gA*=NI&EhHX6j<7Z<@ zY2QfrA!uOmIoTRlYL>cPYQGwwaBF96Xkyxch|cmU?uQ#TV-mjvlW{x(o(IrmCyU-y z@SiFT8l80vxd*~0OiBks8;naj39Y{*Jo9425T>W67!<^y<@zpcIs{h21DT+J2vTT6 zQ6#@_eeXm9NZxPF13DNCN;oOUDE}3&PHlLVIS5ecTnG-{LS3 zv*ug%8nYe80KkI1;;qk}MhN^E~`{W|HGPj>RY?3`$)eSr5}L35IUP z1`Ctm_e))HU7CSVh>zDyi2EmspoA$%INg0|(3P~rUS3LF3kGL8D*CI?|2xF!(i=Rx(w@ca zK8{ik&(q0dE;Zgt>Quk7{QH zY4>Qfptze-Bs6O7yd6mTEc#)?U~hGi9b$q;gTWAuzTu*aPQ0 zi!s;|+5oS_4P$bK`coiQyPjip;?heNkEF>|)Zc`K~;Ry<}&`STCQVqq3&Yj)^AHo8V%-;=7kU#u_B=*7aaKTOQe>}+!o zH?at_HmH1Q2{m2cFMuW02e1eDgVpvPsTSAX*f5km&LzFbKbr}$QM+!2e{XyiJ7RB*q*&Lq+Sc&(K4E$#DaDZ^Ye}VAgkW;#mPQOe+;>ceg0tNMeK3OP47(T!jWbPu9b(Jv|_EQe2U4Y`_!+#zE>=z zAzJlM?+qzaJh!%pv5$M}g$Fb3=W`XOzA82V0ojyHOas+MrEHM;4f|b=H_#x?(x=5@ zSG65Ut7cm(xX~yT6q1B0Fsa^rU1hv$w6=z0Q25q`MugpL1-k>)tuE6SHg8T1{ZT5L zPB88Tz-FZiLkO3>KoHQuANcnUXarSYxAKni zAg)+QhGUR4J(Cc0G00RVX!rPh%^7GtW9KQY%1OO%aFAua`agx{22cskJ-hay!rag%5bW^Wsh%w!YpK1LG! z;JHo$shpJl_&Y9rv4q1QR`y*qA}R4%4;x2=LBnTaLQ1u8UZ4N%3Ag=A|$yM3#zVfOhm02xF zv1;m}Ew+f_)fnB(0QU)64@!rGJ0oGLVF6c-UR=AMW&N1?aSpyZ1e;k^lS;YxXk{DsMN0gnE`Me&F2FdA2l?#G-cXz>5NFI0RxGKAbL=%mhK$#E zfp8*wF3kxID=Tfi4KLG7e@p@sZRPlA>WB%A)?SH?`A8K14}=j(H zHZKo;btGl@Fj}N&b+}YM6oaCdFH?kxsCe$iR;Vyo$L{Cwb2)$@Cwv4@VHRn!CU?F71XB1~$QH)lZRNpsAwiwL>6pWUnof9E z^JZu6oSf&v{J1n8|6w_GZ|jf#eG94372@{{i@pdpOwyb2(vT>UNZ8YaDHIe^IS*eG zF+&vqWKrr~cES$N{h#~VP5-9rSeQ7t6STekn{ME3!4`9-l=}1K_SY-zcQ(I`%=6x3 zVzXT*hP~w-EWbyC`mWllM-?luETl&_Kz5P1Z5ux85TN)R85VOW9jmo#1Fn(l=B4H_ zf7oC!a20Aan^6OvPiwI%KuPtucD6L7 z$f^>EX_4+)<#7VoR4l|MrZJNP;!KtU+>g)>6PUio>Urm?>!UImn-kNoz^w=hdFLOh}Q&o2uoSaVEpq2 zr`f&R$8^_va1a`IK(NQrTM*9NYOYT*Tk*Q9%adcLLlu4+7O#55>O_t)gdv{Ov<$VnZKvkdQprN>eEVTWg)bfLb6;mL-f{hN zLXR)hrkB5RK!);myL=H~TBx^H)mFhQoF3_8#hTV^knF<39bR(LwAkN7X5M`K+L#a` z(b~$s#krHr4A(IG#Rbq>#PbJ_-@vL+E@MW@)Bhf=73Af#w>hn71CF0moM6;kBfhq+ zocyMH{`FAlr2W^cV|)Ep|0e%uho1+&KkFIQD!;90oMzCuCY2$ya5eNB%P?RIt=2+u z_!jOae20rY{fPTp+e%YNV%d3w>man2>s}{*X(vD|%WTt;p49)!N#B;w--6SGeiJni&oZ~!-(U44eDfOp;S#eU64 z%V}am-LMfScn>(!5Q>BdA$2k+K0N&bA8vEB_Q*|cI4!}%3L%lI>z@Al27OUf1R)KNeK-KtPL`EwObbCQ(8&-WO(&kdV-tkP!av z|K>RNtvNRNO(&>f_?wRK{Fx@d+5d}Q{OQq=MTxt5hQpmeQUFi&U`0YuaOp?51siNk zf34q#%)i#UTZKey%Ht8%J}Kx=kw`gE8jfamMOP&$qyw{mh!|^W|@+*+gON6jU5&9Fe!K!D~xbG zit(0jj!v+qb~fgyH0uJj|B{*FtRZzZBD33AU?l?uqI; z@3?Bo*pF%9%A~V+$rs~)4dWe|-VdkXIyKGu*<`wLZCF-ZV|uSVfhjwsYQS?)UiSSx z-*xM7jg%F(X_9`wbw8%uHUS@(8na#1Cub2eO!U3JZ760~L#XS!uMJfr25=c?DlXP@ zPpA{J_9W80`KoR!Nnjs(H3;9uY~p}BumhnJ-TkYN?&(A0J&r4f(cvhDk6u(8$ku%t z2QlBPa77SoNVvWSGp|j3rgvFnz#!ntuL48aNMx65jkvqaPYc~sdL0c{&4=Yy_q^3j zi&|$mKjV7XcWtLjR5NhAiF;S<+(M_b=iqA*S|)~m%w%;ni0=kx|2SJKYr==Y1_2>^ zbNa;DpWXwpM~qdREuR#p8Tp`8$DNi6)$D;_^TQh5SRtq@@A4QE*^38r97b&VJLc!! zLoWnE2OCM4GFv-R2k-KbYNOH2iw-l|rZVi#b=G#m%^pu?MaBmzF#{*~s*E>J0ve2x zVa1=vrKw6t_$9GK2e-~@K~S!IZ%Xuaq)Da;!#XSH#{&5(h$tJx_J9m8(a zmG@5OJEDLOc?^W2p2BV4d7mE@tt5h^qMPEg!sk^GvC`{e!dWd3NAATNLpKIFOKGdiv9S#7J8@Hxm_!6J&3Ej(l1;=nKVaRK1c#3(vA@F`q#cB%Fk zeS6x5RPGXomCIkt`xC_P*(u%!!m%MK2f;PCcE4(u zGoWMJdv{2u@7S!&Ns%yDcX3$3Mf-xqZ|z?%+@^;n^77>6-RU6)hxO>1lC%H+`8WIexk3 zQ^SvX8TVRp`6n~|V*ST0ZBHjzwI76xsxvbSdP;2;8ntTlCpvkv0roL)9a2FSdMuR3;+mh-+U*@0^sli1iS zpZD!hk@KW=_vz~}9_O^xeC?ML#>|5102(7t^1lu%W3$iQ+4MG$siXpZTzuegBDhRSjrzv{}2}JkFC5uMkxHREgr9upCcpQ0RA2l;HX2I8;jY2$!lbj)wO<0}!z5z4f$-2GcV;<{u=8{rSZ>|h zt{H~`@l{@V<&68Ijs2_*!SXl|@Icxbn<`4Z=#etHs=S@KIDYJ1Xrxo`*uC7qf#pDd z?BRJ@!dCZK1;&e<#6+{dM%(OuyE~YkPN+Ao@|RXkb}-j@z1R@YxIEKpbOV_oRqtvx zZkA?MJT_U(BeS6axlLh1K z)dA1OdqsiVf~|-zpI>!__IHUnF*B>gDka$7ES4G?+BpWAUGOm zKCPL_CPa9$D?ROAjLY$*N}{af`n%2AR$Ok)jki_s@(SNeL}5xm^*0VLzHuxokLIID z;I)T9B0!!1t~LH3X)oKGhJCzhB9*egzu`BP`fMw-%0=*=P_F_-d}%v&YUF4Ou2zH@P1s=MqS z_f04rnyo`GO)?u@yN)mRT9UH!he5!$GRH@ruH=DQM@UP;*ht~WeW)fo5f&Sk$tHB) z9A*7iMvI;wMJrkU(-(e+Hq%WC7H&k<1SlUs=}OB9lp3T6(5;N zbrv^gmWY`oH4}cU#;LsFgW++L%jD$}oc*i&j|PW!fr_gX;WB^Q{6=J+wQ@XkR zL9yG`Y~bq0fT>8C%L69a)(cXb&TEA0Fjzq>rbm&Xb(Xa$URSDJ@Fi)vHpy0#b4_t` zBKGoP`mD#a`(h+Z8F^mqr0Y-N8QW5m>A^MnK0cnVOk}g z+brWZ-Q3zozOQWqVgdsPnWnA#!9SYqU#;w{pR@)ZzHpyj$Xo~E+uqT@72M4|4wg$M zo8m{Le#w$rDcZQ;<%S?=VYy8R1U2-beKR``MW6RT;KSX&AAhm1t`XYC%m1|Ek=zZD zBV<(tsuSM2QLh^+V&maVzYyGbO%y~suL z63}=G%SZv)LIN3x-5!#mI`V8)6cM0x#?KewC`Fu zSWozC#mp;%esgLZ#PE3apH@n1p1>XrPDWK#@0EDxD9xS-(kW**q zvhH^ez52L(Jn^qzFrB$r(o7hu*}>)#@1xwZrckpk)zUou>ZcA}PpxmY6_y+>J{L$X zrt#gl>&4r+;Qi1u{wlw_(BIRtM|yAwP!%)8a3Ap+%M;TepPHKADQ1bX47pGNg^QAJ z6@0{rNKgWBn~1bU681dJ%4PXE>pj*y;jxK(;g4v@(vr~)u9UeJ6t7sq;N=EIwMv~8 zesHwfsC0EErUgrfhEG-FUkxb<8f{vBDu%JV-21PZtbR);dCSgOstTiB`uklIcl#)dDzxSUd@qt2;~T9O+c>_4@e7hxZm zW;9^F)qbIewbx*^WXMnq31maU6ak-bM>jb$k*kJr8(xrzT(Lg?iG}e9!}>UnNzmF& zL)65hu9)F~g(SE2vK-7P=%qf2+$N0VJ&>!+4oHJ727dJ4-#1Eqe0|Swez~?VhL?YA0X@FO?+i4zWztSa$X`m zk_P%=EY_!Rhl^tW-h0j6aoh=I{;n(&jHwNBp!iS>K$AH>KLD{L_!=#8 z+R?xhDp#@;zTVxlV(fNs2r=P0<>oV>T=VG}aREXk?wjyRWIkF_wjDP#FRT)YRm||3 z(uD>c#!P!@G04gOyz_|sbMrY}m}IvTB$ENGZ$^tdEb75fSIHT@p-^xr<=ZV*eW*BQ_8^`MU8mVEeffPSX2-Y?_59{~6t-cjkmP1@M# z42lW-mGo9fp$ossZK^Q*Cf~S0(79TYK-{yoa7IeYO}Q*nl9|Mq>5nKswVB)0fZ4G)LJRR@RXeZ%T#;hDDhS$1o1LlDz9%ecN z5wB$tpRs$gz{Vne4;BV52~Z^&28SB7%cL@b6#xEY#qMS*^K0Sduts{OU`HT-nskx= z7P*vj%w019Ln233QsG#j2m;QMb)-bF_T5|$?Wy~P@{{Fc6E9LlJy!CxGU3Mct*x~5 z$pWVJ0D!4?AzMCu%9Rqib=NwIMkrw=y?}esw>t1`i_Q3%Tr2b=BUOlk=*Ym*@#WQz z2C_Ed^x1GKRc{F4$T?Oe?F0@~WA#KB#R6IH)-Bke@=JqL>Ytj%2$MHnryar~O&r$` z(uT}PTH}n6{XZSnWGNMm{mPp}h?0$;%Sr4*5Is(y3k?gJ^}YQo)B5R*Co`hNX(W6a zX3;*HeW0YsSG}zrri>Hw7QEW&zY8_E z5*oywY&YQ2*R&s6E#L8^Y%uoO#8US*+TZ~6sM8O(&N97ra7n{xSV!Wfu~3zmT?(%9J4WBL%N#XL=r_6c4e&^Kk~;ftmZY$y>NFewIqii z2-PTCmB|%P^STQ`w@h3ZbfKy#pEcO?G%+ttQhCfuybo@VddX`^cjWYJ?|6D}VOA7c zDaM*(zYv@c1c|BEzqAcpny7hkKQI<++`JXgP>-jLHC{K$9kV-=SB|Qpm@ek8s`oFP z(jxRhz{o=@v7|3ofSo4W^{J))m^Hvhr(Ir!1n!hbnJOIrP#k_Iol(X41 zJq2!7j7uBke3~7c6V`Vek?x^CI>OmDw$gaIWh6lTHXBNa@&7h4k|ww!hT)848zPgN zT?AA}UY^<|r2!n$vSSG&Qe%m3wHby35<_KGKF83dl!?vv?($JjYH2HDfRJ9>lwt)g zwsbv{Ge&YlqK|c2savMcg^+D!tn!|CsuyMEjgh5%L66Pr-TCy)8IP$L`q=03b{mS) z3>*_#qfXz<^+3h$0t%Oev~XiWO1HJu1V933V4Sqyo|IZhqMGoS!P(2R%g?xsv&6)c zA*|Z#Q58;0tkba|ucw(-pzgjn1Gc0poR(?4f*8HWAq40!Q17SzT{OvPuAkeirn zw7X)2x;!{Z%^-FBI_=K!(9e-T<=l5AUZW|rj*%j;pRp0rFZnAyJaTKR_pirTj1N7H z^KiKq4ow3V!7O}P#`K@fJ&i@Y7EP&oW!b%1?=V;(G2*Lnl`?YIS8^(N-FbQ}?7k#t z!nlAq=iEy3xwnt0aW85#tY_w?|C?8LJi=utZ(^)HeDojQRmzN=e*D<3CvFoVJY|^2 zscD}-3h^O_?0>ZSZW_BdHXw}M5f;y}EHhI&XaY$oZ&9AZg2^UM{s**2Mb^peG%b->+q-hCi7Anu>iXme`K6al?0RwJwRHJnouJN};5^ zBuu!UFw1C7YPuW;6o>Y(2Ed?lmrq85UMnN9+ggj1PF#IkkCQt zRZ*mcs-a1js&pyR;d%M3bKZ5Iv+nV%yY4;j{o~C)$x7DDWWKX!&z_mRKf7>U+r}sk z1(ez}DTBxWftqSU{k}lhQv2-}vq2d8eWAxu^%}+k7K$QET6QZ4k%3HTJRkbWxR4!I zO1ZY_xg4{ILdZ2epRe%_tbX#kWQ6?9@^q8>G)3?x%c8RC`C*G2T&~VHhxz&N z7OgO}RpycOZ4MC~1M3XeGhe?dk=A=rfRNs5_r#@p=X&KEL{L@<)`H-~_T~@xH2!bp z!tXwc8$_3czM3@CNtna-zZ@(?ju&Y!&u_9MLFBA9ecrt;_=K(7=SrC2!Ad_!^ddnh zOZPL=M6m-x&g-S)Kyv(M;9WefiB4miN3_Rh^|m>}FuXM+4HW%L#Y{wsJmyg*QY!6O zH>(IpJ48Q7(WU*XnQ1VxL* zsN<&~;ZGJg{jl^Mc z+Ug5*8iEP8l7tC}dd_e~Yh}`#+x>-nOP+o`m|~FbI?`@9jyfGskDRx7cyXqwOn*~H zgx<5!>)QxGxW|uI(n9^P0 zHTVoOB{?Oqv0wKMsE}@2ZZNR0xcRx5Uqsop{$x*AqB(p&f+J(R&5RFWc44O zr+RTdczC0&!uqAt{zM>JcACD?l3hzXyKs@ap62UfV8z(W#{PK6(?Je;=%G9eMi^YWIl!d@(N959`amt#N*nM>j^9G{Heo9aMp|8 zEuSXl=|Y5`U?)ZTDD%Caou4rCM*}3(qWFEO(!5cM^58W{WKvbi*=fB#%i+s8)GVR{ z!Hkbfq+^xrd-QQ8RYxo4VcwM&`Xy1k;y)hTBOsuf{)AO>y1b2g6k1@9i1ONBC#Mh( z+l!YLVV)&l`#j?u4B(B|M$hEk+imJi6i;C{_SfqPY&{+T%xBN>erU{e%E*)rQM>k< zZsGc~KV+U%R5Kq1xj#1_>Q%M8(%dLYVjH=nq@Kdq{Go;vSCINm(|Z`=6rC_y$S_eW zUa9lueui|rhwQ_vB?f`fR5VqZ-ut5lPah!_6hYENjP#3ppY9G+}to85XE99dZ9GDjWGyDt3Z8;I`Ol_p-7fuav@x#6xv14vigP-u1THSb$8k$bA{SwkFFfY^D zSLtyD-q2p_(r{i+n2Wv1Czhnz&rvt(CKk!Ew4usXKbF&8%Akla-ib9PF*&JsUW3p*7rkJykk*gSHypKr|Y98sB_pht83(3!D@dxzF zo+o-ovp1KsyPB7Dj$i(gKc$>(UcF-e@3(DC!Q@u)?WU1On=c=7G$?XXtaty2xRv9r z&t7gVq+D^8;GE+(2#TQB)^snVZOh&4K|dw9jA~p)?NtdY@a;{w? zUYII9ZM+%}s5#pJz;RVaYN*@QVxrbvgGV)A#Dv;|RD5f5Zw24B7`)As5RN07Y~xC= z%Clz-!b9{UKS5R423f27a@FxLc>H`a$Mr`)qsKJcr!pRIDnzLWuDV~4F} z|E_uw#aBOxN?&K;X|?aBOcMrY65+Y)ghh<}Z>K>8CgfIj>Nm#v}-8wuEg(336L1qp23)fhfe-1Vse zI-zsXg9StMvS5ZxpEzl#Ae2LT>Q?C2!|_CJ3y|wI0gaZjdH-TVrKFRlh!KvaYcPrPrWJ?YLeB{%y$G@oatDyQ#nw2k`!}!b2Yf{eDI6BT>VUctEDCh^wi@ zj)g7jr;fabb7bMUs%^J8=taY?&!K$QIY%zR(WR)b#4J; z@?0UJ8Hg2s8juYh)qe9v{)c8|ekB@Luir(*+~Wq0==-ZlNb{3qVxQ%SOBvMO}?P_ffd$gR#8G^+1KIPiWk+iX0xm4YrvkO-WGxy09?UhpX zr(X@a33;8JdsaBNpEeR8Is`qk*^QB7hq;U}a?qD)_dF+^v5BYR6 ze`;GsN+rBi_EE!lUL{~&TslxJKU&aXQs36BRpWUM18er1SSUy za=-0@PA=;Idh+3iI74e*z|(D-f-TdQ{g>|2`r3~Cjg5{Tj6$BNM!FN5B2nToX=fyX z^!80slE}p~D3Tt=IT@a~O?u~86YR(YYkvg?U<~f0F*M3=1DUvLy-(a%TgIqS1-90& zStiPf-SNO3iY7|7yfw|x5Ysbpo2#*?#3R*TBUrJ(e5iaPMN%)nf&)P>C*g8n1?$ud|6FiP8bz6slim;@f=Dtx{IVo{=9k4+ zU_)cmhD1Vd-u{l5CgTiz2sOc7_N zAY!8VVqFJ}PKiTzathl&CkFPVe7oLDlUZhgT(o~F>#Zh6AH6b8@}MB^goq1%iPo2M z!-)9928R5VOiE+Bx@^1NI<+k59x4&3mx^oaML-hi93le16x9Dt~8% z?(oDQ4kbh8+}?Z+HI{mz%+|9#Py#xWF~(VX)2%V!)m!|eF7+nhLC@2z%}TB&)p0cX zq4{9p)#XC&L2byyhd;jiyAGv2AZk_PsxRivm-{h(zuS~KqZAcjRC2~2yG1SMiZSY? z$bMnwOzt&Mc-)T&S%25n4HhzoJk4n%m41l9e;p8n1fW(Gn>um7NLB&uXZ2bM6J~E| zLg#Je0{qNz0|uR*E^-VFHOhnqs_*U@{!Wlm@;99tGY8H>vU;f`vhdZfqoK8osKEw{ zF}Ayj$`Yc!gybyq?G>l9KNq zcsp&HZcDY?BievYN(d` zf#bQ}7fDNx+`b5XTnBWnp$E9B6>5sz@s}>RRsBSkJI9t?4V=T;GdbS1%-!06VZJw8 z9MN2VR3?E86A=M&cp$Z+YKQAfeiS!!#5pDJ_O!!S@Qh1auh)1;OrRO^RJcK6tLrwZ z-AJ|Aph|^$_dz$whHMfC)&G{)WumrY86HjT<#%sqsU8qUuUTI&#c} zfcq@p@guXk`2lu@NMLmY|r{9Kli$nm0Lod@zKH2TvrZPS7&AhA}>%f_kyx#g0Jl;3Q487IBD=w{(@mRr{)s6zv^G~!t_ae(QQPYlZD$5={NU!Dol-*8shISQpZN1lcH2+ImxCmPXnDU|QUy$*Q zJ$y+}yFJ6modA9P=l5Wq3;pKL`!gbvC1naCJ2~GKkpzHMO$eqW(DT{p6loQgTUd)j zdJ&9>1W8I&u-r0tfY3JM==1Z6QCmE$tgEY!q z@5jH4Us(?Jc<6QERl4gn54fwi0Q!`7HTIX9Jfsk-xVp@Rgr!UvOFoP(Oub_&`A-GN zuJ4khR`?3EG>Ego1Kb_0jvo>*{I+WnNZ#L?hhIyj=Ji;Vdz>1K8{dsV$)aQ#*=@AYx0Mv+MOo9=Q3%eXCn>)I&l8a%Pjd zOA3dClYjCxewS0+EkniS&)`he3cLe*$sM;J zd>OrdXr^nSmvaVgQk~UO_!|U9jB{ls_}?j6`o)eV%@tsWLtzgcl26*)sm2T46$) zN#GN^S=NwWkoT9eKv83P6yJ;3r7{iCSQr-=-;=Z?OS=3;pr|>(XI{eEsgNx-KTb+0 zzsw(C*5xe<@t50%d{Ptg19q}F%hfE)uDSv7CbijT0lYLw6>f?SV zLV!E9S}BDmiz}v7G>9-t;=55Z3i#4P*U(K3&Rs@(?D_q+vJN07Ne&4y9@;12k z5xzg-@NYU-;zXd=`e{gg@bjI;%2^|zOPpPo(%3uI@QHCN6Mh(`eN^^AJ`Z>z?m1jj zU-YvUZ*>PtNk%avWiF zsxNMw*(f~S(QJn>d`RrZh*7KZl#-s$KXuva+0l69-94t)rW~WQ>p78Lw!pF#qF|g^ zTrOhrO4I+<3}3BoWGZ&cxca9aqP}D$iSy&1&fNuCu=F_tHIX05lGFkcTk)lukDbPE z=X!xmHSUD;Gem?sEiCm`ndtZ@F1wvPmkKskVBn%BM-)FGe0iZrQBGn(hRyQ<%y*9R zodtz5D7-k6Qr~$+$;CEB?sAI0_nrWMPM3N){83?qUBvuDkrC1Px5Uq1WVL4sJ51L% z>JLudtEVKR49aaBf^bal)95|E8+`{+w9Jq5Wfd6W*B!&4A9hAVD$54 zTp6YuLl4BHJKZ-!JbVu-QCWEQpaD|}X{SzyzO)iaXsRGRO(iz%d#~MiS2>$FDr^$(C~%)eJYJ(>wb7Kb z-*dT$H{Ojiirvff6&%#-Gc>@H07VhGP{rp}bIjrzte5IuSabt9wT}w#_}drMg$F!` z;0+nxFIU!^bgE}y``lcqQ@f*hq5Q$Gb=f-9%cZ;0D{hafb~O)!zFLBije$(u`!Nw# zk{TlM5@=0JUcVY#-IAZHwUMU?_nF9S3yg2XE@!g6ng>65FRv6bTGihfqk<|VzAB%; z{~b1b8wg{)xWwBI+g>D#h(S27CpnOqsYzU$ zg=coo?XGK-Xh(z=SMQA0fBWopoO!5eF{9!pnI~YVQ{(|J#)PRt{UqP`V9>^(@w}Hp zsZzg~Lg{w}lqu+PbMP+zrQIro91%5hK^bt1KD z!J=oxJ`QnOT7jn~ab&elplIXX0Q^zgR)^d}o-Uc##B*XvV0wN}7dvCwnIWn-!9J5+u$fS61Jdvi-zV=v zZ{>cC?71?1qfO+ejrE%w9iI=~iUjxp&hHPWywV`<@KSB$v%l${@3WspT?4~+pLO5w z=2}{_aN6A>v}>kX)r`tj5$Ac*`!;D$&v(vgxnT%2B^$2uANF4Ofgn$10MAc>k*~0T28^Ro;e-xIqm7?AGrGXtdmPOrCKLD-w%v;>h`w0#W z5Z+V!xvi?I^J_zI);RCfuPsTN4V?;#7YzSMdUxXPHy(#&|ELSBn`$5BdiGN8HJ+*m zxjWkH+RD{i;Kf?jWza29wA-o`+S4y+?Nl`GY;w!}Z#o@&p=)?0$kN)r!DTYbi*pdI zQnsKB(qi1~Ba?4amg-S9id}3?|UFVIs_Nic{AeUxl z8KA5^AZi-(_;VM8+$tYf=(-3MkWcCU>vrpL_&a#`GVJ=AUc~t<=C`F?2;G{3pwO73 zBE50}=}AhPa67@pKV8O}$GA{1>8C_bx8!2wobkr}o*)ElRgqNU*Xh8L%_;=1xJrvE zw2sm*b(29|s;HL`)H>)$;T&-XkFkB)^G?lwidz}#bsen5^D<)6UkQnfusqk(Txz%e z9Oj-o+M;7huD*%7)o>+kM#+%NU~^nFxL#q()c%<4WuH^ZpI^dD2mufVd@WL7b#|wW>1o6v21HM*U zOW1z!7WpE&vWBQua{une6{S>UBQzjU*CwK0H@?P)5ueL42UHl!G8aCGjCVhkA9)0L=Hs|6|8mOWg1znu4t)=a;{Ipyu1#=$zK z?|c6x=eyRC!Zz{P&IC%K%}rX;GexCMO2Sk%+D`jOBoAp5Wmr~rX7&`s14(*_Ik&vS zM_80z9*qdgJl@Cier*1;-D19~W_NeTuQF)cH3JoMkyQ`yb_E9ImA)#efEl)=AAJ076xBw+7ZHC}68-NHb zl?IPj5p4bj-j}lr;A`yK#Vpp_a+Y@GlgC7%^k{G+*(9({j69Y(3J8|l$ZUK65pphv zSc@tq?o{0`Tje}*Bh!&MXrimGvOlSA}FjFZ-Z_uGO3ozqoE_aaZ+B&>B-;H zSKT;rSS|e>8j3DWBHE%^sP!&91MX=A_Y&$gL9jkujA!TPW}uToIZ;$)OjP_6W4BEf%{CUaT2P$fT^FMJ=W;m3woIIgkhR9ipcoM zPsG;jy>v7@Z_p&^eahAFm6xvS_TXNq0EB(s`GWU3N_C0#la-Vx`8nKbUg!9+kSvQLCr$g-O->t}QGvE;l+$&sSK}l%NvebCiaWoEM0Qw%t|Ybr z6^!gpyrIw?1>x9~;fI{K{d&mJ+*Hwur7|@?yA)+m&F(AW^(qNFA!rLGEA(GU)96y? zhO5|Ju3Cqan8T+on9?SOk=fsVdN(p4(G=Yvn}C^0@dfZeECd^d^&NK~P6Wqx_R_1w ze*=G@W8JIXfI3O8fj}Pd(QwZv^q7hKDhPy*}Aio%RkR!a4WX@r4RCYw>d}9KpVqkvmuO=i3g82 zoT(F<<{*CGP-R`qZ<3lbN^eYJ2J{$-i91YKwAPmSPR1yP&%doyPe5Z(K)`-MWcL|O z3?v{`b%lI666ZA_M{S5rTY9*E9&k{}t5pB8W%PL&E0c2jh=FlMGZy7wAR@Y_U%PeZ z$m4TEfMrxmZ2SA5w-oCl{ErI6r~EM{=LNHt!9VS5`I85`f(85{cNzhC*#{m-#gJ|}`dovr=( z2dz6b!ytqr--2j;<@LKVOvC65>nOsf*Kewx zDtlOe{llz|Z8#Fg&#yd=@=FwSt^_htrizq2+u~lxB_8YT7q5;aexzo!=QtPO!Ybot zer-AHK98VSzq51zn-*ep19`p<=5sDmz*VVsqwfV1K|e@X{{HA4jxVW+oO)=)qMh0q z^G~aSK+ZY?1(G=K;_*?bvMfBH{iXH%?{zksug8(^E83#Ga)shr!v!8m1M_ZXt=5d0 zyj<0U@JsxO>h;$xh%rwy{_wVKRyx_zP7S9LtCcjaqb#4a+^0zkNGDghncRFXSgXTg zRSE$z=HCAt0YWTD)?&4n=IgJ+*KutOd6joYOOedL2LRBl&E#`GDexQ`lEo8SD|7T4 z1Pc}`iN-;^#$}}1w=D0<;sbBU|Asg%%@&$jvbkT)x*6Ogb+=Pga~W>@Zh`_GdaTv{L7&pfY7PQZC@$=E}MUICVMwG`mQKneS6xw9qwT)#-(0Wgp%ysm_6%75Q3O?P$l2*n!cInprrf{>jX0 z?dobEQ%=iZ({to)1bQCbE@3Hi-2>Wdl!YCXJU3iai83bds)`B-j5}>f%iu@D%T^2*mP(JC z(3gNiIYLuft(*Qo#$FA;0IyINBWz63dTUi%=_ctWb)46tZ#ee-PAN>z^m(UO-t1+~ zGJod+GgtsF&wnn3R3kc^8s=gtEJnsWGI7BizY=T>TU8?Cr3_zM&k=kYm6Z3R-gd9E zVmZ9Oi3~rkOa0+Ry@rv@HWWI%D>I78no&5w&IgVBMt}%H3h%oSjzV4dn3k845yos0EbPP^DkE16V>&c43-p& zlzAbJf_Hh|sg)7edF*4fA|yMwL_l0YL=*ePmZO&7N^h8mkkRw`;M+6TVtpUqErxXCx)jz zV{Y93Hh~zPQX>yKYocX1+aHdibGaW>sDzZ*5$%|tcXUW0d+{J60eXhN7d7= z2GT>VC%GZWYCsbBpV<`tF}8mORZHpc^!qkkzGO~RiMJj|GB=-Uk2X-IAI8(*k)9;> z6$j7iS}A_tQ{}*K63Wt!ecykP!eqsb3%p*NSrqZ)N%kggel5eIonLoi5tA@odziMX z&zY`O5U%I7{LR$l@~00IqL24&ac8ianCfqkW93mLQGG`Sj-3`dX1cpW-(UL4d9t3i zSbI}i?_uNNcK0Tv0$A-P7&*cOwL>ggjN`cy#v&U$@}2aksxUymw;bU3cXG2opDGYa zaIVXPi7AC!+&MDuOh%^~;HoO){-(2<&oK)~)pOU~)`*qJhtE>5XJ#&L;9^VS%Z>&I z@FnqePnSu~sJJZs`3AS8Q|Ya_pJ#C<)93dYb$-{iQ{z~Rg6CWMR-AASC(m*VO{YN$t{@0FLFp6TTgFcz9SxXooHd zi>bpX5oD%hvkK$<){L!eH9_!3I1_hj=@B1K1H}87rtP1&to> z*3fJSD@lM_5g<^L8vaS`r;ouQ*D6aMTmxqJIv_1*8#za&G!0a$^q~^X7l=U-=P?;B6DvPRF1T7jJ@2B$J&HtF_Tm6X zy(qsB9Cb1G>$_86tI;-0wX$EhQru)u6jVUv;|B)otEux8Tk2Ci{1~+jN%d_lKtN>O z=C;6{-wC!e&p=4aNdttVFZePI1BSL8g?7c)x9eea;E>pHbYNS)pM(KdjNhkuq^4%3 zh+#9|M{oMEWVM1P>oEHZqn`n0qZ(-z^`Ywm{co2W#5C4?c_jbBpB*fz{!N#9!}GyE zP0xQC+@ewyQR0d`pS%dmj11rjlUtgI@>Md4({KlmPx&gicPKmS=p^+`(YdQJ9U2lf zwC#Om(OCowx`Rz2J!^srO{h`&Afr2}W@F$z-Oq1+XN2Cj3?(@uZhUnPRL!3<>^5tI zik7CqHw9X~>k<4)(=T{-bY5vm8>+Om5~)}T{CZ|i)7q|a5xQA5-%~R1rk})Zmjs!K zD9!n(#H>UX4d>chNs>x{(&XE$dh^6yhz?IY;y1eLa(3bmiOHuV#jsiPDr2cK(49oi z+E!slr_BNyLBpd9Y_>H#vqQhb+vVpbzt7CayCo&ZW#BiYm(J=ctZyZ*zV!cns9^_6<{^9`InM^|Whm zzO2#b$VTd!Bv*8AcmQf2kY$l+zaF1NkT6Rs>#D9;9TwI@zsa9O};^fvOhWYDL^g`8aP?V4{ zAF2Jh06&=6?ZI5o9r=nKkAu3~sHjwKb-3iXManeM$8b*FE7elk%)|$oYr9|7RC3 zT>qQSrKfzt;=BInkT>qd=-w_@2{Iv3LRv^~1?n&w8ak{wK;S^YvMFE#pq@OAOYFq6 zG9=a$U;)dwnvdQat@2ppviy+}lms=qeXmR2=Bc&4ACvUt6LN+Q3>(=wn4Ee|`ZpPU z7$fcuEd2^hRIGk))EInm&e#>8FfTZ26TuniQ3l!RA!H((f1x1Em`!P!h*72Y73}tK zm`}Wg5PuYwmnU7^JkLX*s7394*`|u*FFmEQIn~VV?fRh8RIL%300T z4XeS_h;!8h3GcC<_H$}&!S6yt>TsVh`g_{*DPRjDLNIHj6<4&3b$^fZo5lyy_+@CVW+*2LxC%+@Q-4+c*DpljnFE<=p>k+-y0tFJ7EfXhP;g_D%~s=UlXNx=L3q!sMIo+d%=jQ+`b1( zO=K!Ke$qXPF*q;epcugpH3&ykx@|$ce=(I?@vK{#CYMX!IUKkw_L(D`#-T9zV**<( z$o_rbw>aR2(-uGP-*ms3sitoD$3r2=HF8Iy;`MBhU0lm6R0jmutBcXI$`&l`A-Eai zIhYIK9zha<=hB>RA5d_YU362AbF51p+DPgv{c4$c0u^ZNpq`p)Uk}F)m9g0GELGd6 z>XYB;1_yrE%Qo+%oeZ`fbTBq51l$dcvOAyV&g2*!NDY77yhz62}bN6~fYGjTKEq*hQSNbZo zK|!kld>>g0VtGYe7#1Z$ZkZ75V;iqnY{*Z`$Cv4+l>w`=#n+RA7ps|tD@nvu!_NDh zHU{y0I4h_4?xKv#zR}x+4vTBObp&rETMSe^suDtxBzF_%-$*@p_#3-Hq`LDh`n`-l z&54>GZbXvOanP5YBNGJdTm@QO!5v7n(piMEXllYqS^K0v zJOAG!+y9gIWPgKWqO9kyXYo(hADe3yR|L$L=RPYw4&L*j6Mg-EQuBX9S)grVGZ4ph zVm~+C_=?R~bT#8Z+;w{SZ#v$Si4P3_mZ#;vp_%{9D(EepH>gh9f74m+@2xLhGJ`e> zJ=NosKWy%34!D25=r^_Adm{NBa+aa`{%Xe3-tPMHgOUL5;9LWeZx3png@?z>a?5ri z*ar00`IJcnBM8@S;Fu8C$FlV$(x-|h{2lUT49Kbq@6^yHZpl)e?dfNS=`bI_7;s@f zZ@AKumuhMzYpa=Z-HqpH?G5+fyj9h3v<4m(m@r9N zPQYcjM?p=Z1dgaje|&7;YJUrs?FKh=j^N~z8*Qyy1>iS(+p4NcJDju2ir8^DdaJE@ zTN(RzZ_WRN0{wBV^CONdZ{6Jhy^Ailp7~arSGjkbA^A34`%LpC7CBJ>&#FvOFCs-O z+9Vfsd-953_`^{v&W@sCVj$NqrZN7I zJbrD>LS4r%`=wd13Ph&O&8cI^#bA!(KkWZ-RkWyacw(nPa5}jC47Vo|nTc_^PXcym zdE10C%(ZFHrc18Q;sCgsxpGow+o?lpsq%fC<6J5%-MBOA9J{f=Xh&Feni!CiL3tJ8 zt*C&^Xz*wl8=wm2Kn)WZk1sk^w!d>uefNUb;nh-TACOrj{cpNz!H4#%d)7$PQo>D` ze&CihmwWx0`}qqK3zP~@`RJ$WmsfoOergtvo?2<$mX2IqweFi0VAOhtsG$yVT^#TB zqj2dAK>^^d!RK-;hEc0*f3jdeYs{v_#L0f9f%GB2SXi2sL!h9n_Pvpw=-QieOq}Fj zHkN`l!29C`0$2bAK-EnB-_fzxd7aW68=h&e>%5xmD1L*|eBn3uFc7Kj{E{3wSk1w_ znuO27pE^*j^T&tv=GW{9#@xOKF@6B<*rhz%L zP5m=;mT=DrC+B28>~;Kzd4G!4ka3; zhw$+iM&)uc@XQVa(?`z%TwNfv58{B@B6$W$>-IU_WNM-HdJU91MAv$ahl#=&6A5DT zxfGF?8#T+?i`sC~ZNS-uFW$2*cf&SuV)wzw)3{%E-x*48%<*nsyBzU^=5be|q1-ZQ z^WpSpGaX$i@ITYX=lnOs2?U*(?F=*iMV=35qe`Es0js&4SC9$GP?KZja#+n^-sW=1)ai?pIfqD ztqcegb&|E57j9@P{D`ZP5Zevpv|YyM7X{ap0hdTs=!TmJ{qP^c?)Td_ZP8Bw z0hF#35t(IvHcefY3_TGCSf`(JcN3R97t~)Jhn{!U*)CK`V#5}3NCrbFV=AdcxV`;k3mdUae@ciN8v{V7A!iRjHn*K(kIND zOEBFVcW70@#`lUDGA7qQBlz-9mU4&vrTOLLW~ii1Bf=%R^3fXRj7XqLx__i!3hVOx zd1We2I`t4eT64ghcS?XMYS zUZqeT#-ace5y9ng}8=KwB6U1y<>bGfhVN}amu?1o>i0Rl``>ODF1pPgc~hh%mCYJbvhXm5@K+YMOETkP!6P~=ndDs6Htzuv_BHd6Y1SDzXSLG#7ex`R{c zvtWd4If2@#iR7pPTq_J?g#ucD(o)X6w+!06LGZ`a{9$cW;B+9c{EWR; zIl5PQ>m9JZ{arSBjL+^#2KpPjrIxlA zhiWE#$v-nwD474+HrumjXb*Mkq|s`nDVugZKcG-&YQmWLLb-5~CYoPB>E?erpa1JG zna;8X=xQ7H`2)jbXHp=A>;&d!{5A-a`!`+0mF6{O30azcfMo$*Cvx7SZSb-!c0{bC zMpFcT%`-59B~iBS`57VgjN6;?p$v(0y1k0_jmp)yD^I!9p)J81|(O#|GwNpS&ehgOlVLNzTMy~Ab;PshbZ=;w$ zQ>4!e#BI*%EW(LhQt?Lpv(6d2Tyaggm0IB?1}(qD?o?$r5C;*p1j`2GSUor?w_ub5 z&x{4EW6r>y^~~!shn{Sc$d6wVccTVG-n#$nOaD<@rfp-H51JOzr(V>o=XUOb+lDCWEhzsqOxuFfXc zl5JVE(R@rs%iPpkxtXOf&vkw1RZglPKhhC3cWf*}pDH^zr%xb9ewLe(; z$W%JBTs4uy$IfjuGe*a@yl%+7*X@jEGQ@zssRRCUs+{jzg1es0g)iypfA-w4C)V)j z;qc?V)K9TrmIe8=2sA6Gg2#vYH%Mn6zd;d3vAkuoOwCz*`C4;_XJmu(NZ|WYf7d=F zy4Pv5LL9nA`rM~45L=Vxrwggl1)Vv>meM5trkk7&t>rF;xrs*4GEM+7{O|QlNDq5z z2BC&0^Xl8x1BH{jYX&Ly_Cepl-|wuS_UOYDB3IZndk%J59obH7#=f92b^`;^Pl{wl z-bkyy>d4*c-Y{4#U&@CzJGnJKzZ(>Qh=(beA92|pGeZk2WBkk+W*O?Wvv}&oi?_4S zoQlp8$%A;y55|g+q*Rv@C1-toB+Hw3$`u`X8KNh+veT=fOOr$u*(J+WOu$bqvnw5g z#lLO9obJCWN(X(vv);{{>RY@EZrtbF%lCh@GeEH~ub%$MW^_*UXrNI%2t|AIb!_8X z#8k$p(||$p)Ao^P7tQ~;_!ZNsHkuDqiB;}1#?Ns9H)?Jaxn z3!>N&Zc}YaDOAR|pyyvNKf3?04ZiUI%g6HHHxh}XPL)l5EJd;rJh>)@zD05J6}>_S zowI33ozrtO$ZUr!T2-Tk2Ld#bD(oXP^rbE&ZzVv%IT4~-Xgm>=pLwA z!8EeQJrf=S^6iP;Ux!F)LdK`qnFejMa#z@;*MbDOJjL=+I`mH=8yNOQEb&=>&u5o; z=)<>3$cGA3Nb`_dflk!dEhmMubllp>HD{PSZF?3F=x@5ZfZm$jh?*nIPxV_9dCHX> z>87e|T8gc$bFm#6iaw?0oVXWO)A^}LAJ*@@XPWfkWATf!SMIRgg8PTR zDDLTCYe&5pV-cqNuIfQyA1nEf^d}&9Gv^-!aCQ*UhAtb88H|`7j}4w0s5q*aoKKINyd*5^RojEgi?wVO^va*t_|NYCC@BjXVc+1=tx|4}y!jIBK0X8TJ)kxCy#O4=vIm$E^}% znPAr=CMh1jUX2i6ZMCZp7jE;z z!oaLEGDh2XILkm;Ss4&xLn>3jYSm2DqI)jv2MQuoX}fOwfiA`s8c9s)p9@#>DrOkd z+otN1x33jCJqrv}gT+0COjL!E^I_uaLU;W_cYeXWzF-sU(C93aFkumEgh;dz;bv=1 ze0g?La+^|r1Z}Q_8=6)M9)UT*ztiW&mcEh@xlcVvb$Vb9dw_g5zz`f{lc-x5P>M}j zc-j`R!pmCY>_PA=JONj7CweLkv#;jF(El7CZcbxeXAUzM2r~}0^wID=*%jUn?x7E3 z9tzb1rg=ww@AN?#800Lc=N#2#I2qq34i*Ap7N;_WnMCwyOr}Ax<{BfCRCZP$S5{Fj zJ>ojdwk=U7;f#WeZqjLivw_+1;cfSaluDHMYsHi{6rO|&_pX)ozG*zsogGA*VXtY? zYu{ciV=~EVd&J>f^=4v^VWcQ}z-pZ5)BWR6yemHWs^-qwgHOe7OFm7?*VA%_tRC!| zx$I6`ICY^q$Yi5dNn&S@O`J>>+1M6rI|S^^w3VvL+Sf{tM6|lPCuurN5;|(*=_o+O zWp)eAHM3z>LUIReH>3XH=91~xI0ZL{uj&f9uIYbHplvr(+GF~Gx3myC!|8pEZ*Hwm zkM`kF^3*DV+e+gsa_P<3Pd((Prr1{&Dk@N@Su+bKjC$m(N{`cP#}A@sj9vG7D}t}~ z%s-{L34@L!Yf}#=&t}D>%J3*@@z!waG^1 zg*xFMV>|Taw78Ps2Gl#Ns{WAqfCIS$LX|wp7tcs6^_<-M_0}o&F^72gj9wYsN-rx} zk4arqHIgTiGpAaaB(3W>9ibq(>nbtrhX)az&jDxCx3=DH%B|C%10)c{&8vrBc55#8 z5Z$X0`(DIRMAO@IfYPpcqrtSo)QbS57jOTfEoClz3aIDtKvz?Sv8ikat!sWIEThu( zQQ91R;=O6*Ekb?eJ9=b~Yo^FM68iGp zT8W8kG1=^3xK?5`B_#!TlwxwiCfP0nrq6$-3j*S2{p1Y7S*8{J812siGz_OrEy5BN zUVi3{qeQGqPcl@ejpSEZrccWNy_nifvJPk}ewCS+=JG=_SZ&p9x6C=o+!E95a^;>( zv_WTaTqp&~#IyP~%s^Qlt`jZ6TY>Nlm`pt-{*C6rgE z!`^As=kmlSmQpfRO>4c@QI$Ol*~P9PzflC@$A9iQS@x(KEoT!kacUnu8p>L;MtqYdS9li()O^3kQnVYfnInC&C`UuT zo;}fsaU_=D+z*!5BfVEY{cs8xe6_0UCKV^SDD?)MI)6zis}Iueudf&(RaR}J0aDxS zR0|$eg!lxy)agQiVobqmC$@0s2TzJekZ$^_W^X2=%NHb=et?2^jIs$BaRq~~-J zJc1ER&EoOQCc4Yw#8GV(Tl~w}Xj)fRY}!_`MdmP8 za-)hNo^nr4%4^HeTsPd=#pr2Y8Zoj-s{eRRtq(hO{9PDX{dOj$!pdX z>Xj0m@6RTF2olI#s~ zlJ|?7bhL~6r(FkCan}s0J)BswwK>&$J?{@Q{6BH*{nPXO$;L}qQsk%5)a&fI;yHkK z@O@sa{mfoi@l4@0#DdXnnx}tq7=S5>7{&i#&PuY1X*kt=vC(o{?W8dCjl|g%7rvgI z++7M!tBQ!6j$!mr`+~vArEh{YgRVEQU$yAz-Dmty2YQ{Rs@)LIs}7?AwOs|*mwr|O z33cy*J}N|oAXSz#mO*x32Q_mPT%0drQZ)BD-t==22bosZ$Z%x(4K|S-fuYPDA>o-5 zDH4Z>BQ5W!@%vJiI|&0IOor7elEffc-0baDPs^9kdmx#1ZPsd7Fn4XzGitdwqC6gc z#s3a%OG@1G7+C#Mqd0sdfFx%I5dn(kg|Y|Hqtwra#`}ZN@OOss-^J+%?A9Myj;~;J z6ZP~xFU2QYhbv`^*ap(Ce>V2vj)7@Wg(`m>-<}>Yk^s_%(mmLZ{3smKqwCRC)ewS0?I`rh9ecll3B04xSj0T*)MHobcBW1czeQCeJF9y_lGgzOji_v6Uy*an zD<(Tg$dP=_MLpj~#rj??PO;po?`_pI#N+@Ido@AA!%1;z%7vv$_VruTZSpjFGXqQ= zfjajnLnET)O!8$aT;M%$RWbx^RNYajzM9Y7xd!ss4+mO8PF1shn?Fy$<{z(U5_Z(% zDP8-bVI~Az=RhaHqF;t?S+vQHNOpGd`Pj|O3eL1?_2G5F4rjt=_zLh5a!wZ#hX=-3 zXIs0GSJIOn{oc`q>ZRKny|pTY)LKh--1qE6cS!Aw0#H3LNx6B6LxA@j@Vb6Dv$AGQ zI_+26DN0jeJz=FC$%s9>V+WAE6ulK9SXLcMkzq zZ+m?Evu1bc_mSNT1%HgJ{%K?Vv$Orf_Fs6vHm}l|Hb1+44(P66E8sAaII~=>UcJC_ zq#t-$_wW1L`yR!c!iYgD;y&SZllq6#94;=uI9F+qNT0rV;=)xGI4A|k2$8cApLy4< zj`$t8NYP%_Gs5B3z~hpYFtH=%tJ}YPmoN0q=#y7gJPj#$O=b|pP`VNZI-T3AN*!{M6w)z=pp{yQoZt4=;$)b zw&F6QTBT{Cfb0S6Rk5N|5wEdYMmBqmk{?EW_$D%hcn_>{gw%a_^@p7!GhcJdo_K)e zIY3Huf|Mi`&Sy1IjjT4laoTR|oYtTAQ?7EIYnbV{grikV{tGRpo$mskyf8^GtBza= z{+N0$$8WQZ2ZB6p`h3ij`uvcXX<5E?6C-4FbN?_gj(|7x$!?`TrAF@n&jHnLgy0`$ z1ww|qzOVXN>w7mSqN~XB3Ns+Xv?d(Rs-l_xHHpsPw2vf&I&)?@^M}U9B`c>KSXf0n zm()>SJ4i#J?Vi+J$8eI1AGEPtPkJI0!<*bL+-BI%C4c8EE&VJ13sciC3)L5_jqx|Y zQ-Mbn*)LvHO~7VIf;xy$pXk%8Ggm*ChD&u=&DI3L9u*_Ai^%t=XTu?KBiTG)TU%QM zjA%)k$FRh|+`z*1fkqfdbE!J)o7nu8NLr&w$C#4PHI=GGkpTHs^V8+|<_GTQfMcLg z0Mlv)Z@HBY%XK^9J87_l2v8*Ldeiu_UA^0S;$cF&)x9nH9~8;!EDDu3q}t~0UJ%s0 zwE;aUr4Isq8W*f?$ig(qR_~kxe(9G>^C%$sQ9Ws!ny>|}yknRoD=B$p`j|pwmBMCT zk9GVH0+CFv@osUZ*V~b>r37W#5A*2k$-Fr#*QO`RW|RLMAfomy<}8%$Fm=(d{@t9pmDkK*j=(8AgDrL5(v>o_ zxM43%NNQlTDC_GmUe0A%s^t4H2sEACZkM_)uk~ZMO)M&7%C9g?ltjS1P_U4-tLqx|r1{t=%>+%fDH; zMk93%bN^x?+ap;VwxwC-Ne`@s$3T1nW+cXPkJq$Bg16x<1im;)Sob}UQ;0& zQ>7-JDGr~FiN?CK3G7o_@U9J?nHm#6f|_^qJ`}wE`y&qk{D(-1{{)oD1-(oJPOv`U zBkMUJ)?Qf`QsHvkHs*r6q_ACd?e{p5Sg+F4I5G4dB8s^2iiCEljGQkmOv9cAAGm+} zV>v{jMqiO!hRbs!3tRK7F9;Hrz3(cjIo>L8Q0Zpn)sQ4pIKe8VEUh6ue!rwa^Bc43 zp(iv@F%{K_LW9D|5#uBSE75^x@nZhp>g3J=IQ5=$K)*84ME<8K2|BjQX>$(v!FCQ1 zl-9j8b8Og%z`?nq+`rtL2kP?N&Brxpr+C`At&MIYrvKA2Y|MxMQCZ%o!Xy?k~$@pukZ;sgZaNA@G^wy@^nj?k2gjLj6 z#~iNMmgsX2^E8HDbufCPmQCD2Znl|c4ZG{-XTOwQf2~c4p7a_(?Y>it)SWp;^QcUZ8m5#+gRc1_fG%MByvqa6a ztDba0&aRYebZn`H`eQ8!5}5irHFx1TIi5(vRJpn{%H#Dh)vDnXW!F0kIJ?Qy7CT}X z^!jKws=G4NywFW$6O&>S{`GT1+-gqNqT71b zEu1!7J?;}%{DfR82H|X{bKc%-Im_iL0cTBr151!QH3tlC=uB1W^KPed9cxx}zXx2X zO8k4R{sGs1GQq99^zFnK!ey?`FJu z4k$}iI|uB=A6}=pkzMDVvb1r%u$rQ<)Ncb`0=w}X%_^hp;w;2vcjp{%`Sxcxm=U!! zeakgzxj)Z3Ix#nVS}qz+5&;rmw>G)$5?wRla~5_7LmYANmQ!EatqU!e+gQ_?nkxqW zK#raRa#-oI#&_|2!gz_urtmJ6@zrQ^~~|$(hy1;2{0)1Ef zZakd5wTrNx8sqO4zRI*+`g&x42tlDP^rq^JQ!}0vrr)KQq0ty~+I6aLTNTf51BHe} zBqR*5JV_=cS02|@)_c~+_QR2_CmIUx005e-!Qa~O#fR&c&mO}K&H)~Rh(j)=9wo7> zdhm_i1NEJpEt`iuqI*4fJ)Ck>{Hnp#8u7es{ka15;wQ=Yx)3#ad+nEN*6AL-t|9bI zK%aL;jjVYFdu8zXT4q(XJlK8=zVAwg2x%hWM;{$`;;izM;gp%JeRUft!Eft{%Kmn^ z+Y4{J?pf|MUZz1`Qymgrmn`_Lza0(nU)VqsY62(A{3EJoMy(|nBJ-ZDuUpYO>pTnD zTH;WN3(hp@wBSsn81Ugis=|~enFXIbs!GkMVTeEVGA$jGnJ|of_hCg)!dvH}NOQqP z>DIFA0!Qa|cm<#knTg-{Ka6ZKs zQ!5Z>L+-rvM2s8CB*i$4J!$ccSZWvBcxdoag*u&BO?T<(Ie_1}Y@^$>wGh@}QC!m0 zwc|6N*s;kD*I%6*`vw7tXpUb{{M9QMS1;Xg^=bKa0)MIc@B?&kaZ?j})-$l7YWzLf z{<;|*I3yW~(vg$v;+xlSsk|F4xHS+PL)RN1FzI$OqlcMmx}@F+|HgA1lwE}fx% zhno#27kr$gQimcXm8xET^4Cu9(t+6?m44i^-lDmQlb>=7YY6+S51r{~Hkn-0)z%n? zaiXm9HMij7hav&1p;mX_Iy++NHwHY}5#tt2*CKc68=DVvcgdRi?O}v!V+Qz~XrE!i zPROFwG%J0>O0M#Udf&v>R@YTol^=q3**PN=gFcYy+WvyE=jF4SGX&N*Boso~uw7kE zd`Fo5c=Lm$Q;Cn4?_44z??}aqnbUx=HvE~V(#39KbcY?e1*LK^qq zR?-#1i=FVs%@?>+*DOO4PC7)-0pD}#!46JVUz7)l<_Tp19^~U4ow<*m9FlhH4s# zlxcDEsz#BW9cs%qVE-b?-K2(9{mqFms%|~lqfAA;eD4oUSE`wvIKS^qx~p=BRF4NK zQhzNHZkgGvko35uF%#X7LZi7y&o;_)vNz8Gl+1ROGVj{F99JqqzOYo-#-bc&U8%00 z(igYIwyY`cusHrx$y9W*Ch=LX|02`!c!^0lf9Cz9T>98m9J1QXoi8n>*yY2dk#!n!QWX+dk(&5b~&+Z+JXUZIW-(Sv|+}kr8ag|_W{YQc`k=2j%K`I|6v);S=raQGac&vy>ke{y;>bmlT* z3dCqyO2D}t47jja(e;#@TVUeyIiLXzX@0WIlv!DPs{qdxYZVtwNbh3T=ehQNdG9W& zahRCjA{05=1LXb^_PdxqaxcEr z1y`kZuB@#Ht|JYu(UC(tBG*^MN@w#)R%pB<9ixift9(8qZD*!SACUWEEdI~;h5G#{ z(@t+{fbQWFQt!Oiw;I1BI!~u=&|%X>;T)nx;6#xUo**8br&Stl#Dj4uuBT!O(xZ$n z@WuFTt{SjEt;Fn?ONa@lvICu?>Phz#^{a9Xvh={j4{-e>t&^+zA9*Kn zgKw351n*DA)49+56rn5+KZ>T0<_ldj3F+xn^7`R1?3|SA2kq5jo=780NKBQFPPXe9 zCcD?OIE3$SSOEhv|jEu zshhGk<{qQK>CaQsxW%-NFczrs`Zl&mX^mvaugd78PRusu4`nHiiBx)>u91=9h6<)V6MDRpQM(rh2K!rgGzMlVAyL zRk$39RTixvSG{w$c&Lu-mf@D!&a-z3T`R`gt7#%^T<9&yRfUZ_;?=WQpHp&UydQJz z@_q^~?6_ni0posByS4i>EovS53gz>Pxb8p2%Cr;iA(M5 zVzEbXwVZM;7_GlLY{=akm1J)(reb(1;n^%EajcuFd`$JY2S44hy0KF1Y@~_PF2Wv*Gz(j}wof=|)K#Qk z18T=)j-!;fKnfp&@58ELeKJFGppTKPKLiEYrzg6?&2OFq=qnp+gNnW!JVgurdXzeC zjlOMaw9cCETQ?UiKw_0vx7*Nr1XAQQw|japzkI52cb`zr##$tYBuEtX?@dGoJ2WbY znaM}v5ZuS*9Wc=(=l4^Ta>pm7tq7@~r$3ew+EAxF0_yKql6-=)ak#Yix@ZEj`BP^= z0Ze3g#j;|x_coBmVJ6F#={KYB7wKcl z#v5bxTROIZl-L{uHP%H)GzlG;Xq~x~&85R%^C0d(CBIp~{phexV78{j!#_<)`;{<)Vj{ zHirWWatb{{+eH9?4@Iy4tdc1V*F7i7G;D+5W@)4OCphhxiBA@c#qKtxckQnPQM zR?;jTQUFU_MOxYo95-{H{{B#xW4;Lo`2i63mb)oR}>9zFEA0p3m*uPU^ zVt*1EP{xTzYbwXm7rk{3(It(xSg#@-Wu3ddq<$SI-vl*gMh;EZ(u|pmQhx6c5-_zP zapD^JCO9btk;MozzeBf?SD_|&*iNp61m^+VF$I`h(5P$3d#l?3z~S*<>*D{sFFULG ztEZ0f2cfNbo60H}3DryJC>SS!!gQ5MLFVXGAYR#|Gy0;IYYd9orcB31cLh1zjIYjw z62iW>UqDvv%+oyl+kN}xk5M#Ig3Ir`s-H;Sl}m_9TLbhg=J9un10fwF^BJwVCza1p z&IZ*MHur#Kl%2>>nA5@*NQi?$bB*b!xauhN-I#+sW)`26q>dF3a1g>K+W7O@q$ut7 zx$#YrVYNEhB~eaFsdjBo%qiGy6-z2vG+k5lT8bVpHOUOiJO>yOo;(`#u#c*}_n6DFu#MQ zRIH9fQ&z6EX3ex3yST$Pdpu_ML29EwD#VJx%FBSNEbImfZGg(IN)6Pav$qk;eJ`Er z`bs@YDd%S69P6`4(-fF~?jQ{JOI|H=g}?IWl3b<8SmS1wHEamSt@>`oZD_|apVYf} zM(voCp|`4ZiOis0r19H%I}LhkYWLwq`^L^5HdLsQpzQYeGme;=6=V04 zVg_9`HdU^!2rs)_=v__N1G54N_M?(x0w5zlY1$uEc_Y1tp|l^qWOwvAZWyf)#6&AShnh=Ho`a zf{Ceg#G5$rmwzIFID{fPxV!i0bUYz~vGB`ppY`0esVZrEYhc!D_tl0zubkuKNbo$* zsOVPxpmumx#f)ZNn!n%O8VmpA(n5yW3}t;-g50C<&Hz0MAvSmU?gH15BCcU}k@zWy z_G%!W#dAa)azK^JGf6#Wy~Yz(kl$xpi-9eP#M+7;E<+>N@Fz44bNxxMCC~86qJzks zYTX!5n&Iy?WVG=sf_bo5F`{n5c+Vjpe$GFl?_Cv(j56mJE?*vXr)>|+Bk|Kf%8!nD zBY`}oN{J}7vQ~}ivIu%pxBP;>a-NutMG^Oo(q#~kw{|k#R?z64Tmn}YxolQGImeKT zS5$Vk!+f^Z;TVp1Z-tW}Na0bE5-?bH^VS02-HDw70#x~O0obANWr~V)gQO&*DxNeXtJ$d(l8b4@b zgxWXP=Ur!B{_skz@blhivDlQ|A7?l-H37*c0cF@k7W!q>4{f7x9yU!5|H_MHsg!wT z?j92hBq|U6431o(6}}u5IDe=U*jYxT3u2+k+R7StD^zMos{(E8a1ioEvv2Q*QM!ZD zx=gFpcPBkA!NOL{W~&Q2L$_;VoS!AfCb`{?y`yHMx*W|;P|Grz6lI8iJHWr>iB>(4 zT3z3uhxF?>#aN&j;}SxQe_q#F7udh+qvXtysg)KlB*Hf1YUNVi&tF&|53@j7>#XCO zOS(+`4}k1h&`{mF5gwRRU>2$>SsNEag&b;N^%>h$> zc9|h`Llc|GjzV+BH?^(wd9ZK~$SBPlCfXQ+xs(+&S4wf~_MBf7%S_ZyxHyz`T&lKm zZD)`f9!ZcP_$g!Ht4=4{l%$Ux_u9={-EPL;FcW&1tSz0#Q8Ce(qp69MfG(!U^DIYa zx4!V$juq%_}uZeLQ9MX+0&6N#i&(5N$2dPe$kcY%yf<}rYFc=bZ=C}x}ZJy zL3^msV^=~LR|l^KeXDX2G#)%r$uo_O%z>baQeD3o#T3iiR{GF>lRr+&lc@7UKi6b> zCTLkp{)2Ef63l4W8O)|;tGAJ^l%Xn~&!cvogS~f@BNHgd)i5+!A#c4wXOXDa-@0;t z(oS>kS<-O98h={+1z~~QEQ#y%N&B~I*+jF&hfS$>B@0JhsgE{z#dU}ju0jMne%xxE zsVwc*T3fV>X}1zqT2AD_a_Lr6&DK$HP^9x`JBX!UE_>3ad^X|b+nQzh@`~^yZ*OE{ zj_udExLDo@w@lY}^J1v?ih^a|C3}iFUQOyR7Ask25H4ZrD;5jp4>kA;-RhabQTP>( zVlruHHbx}) zYmu_3@@R8T>R3!)YQfNoP2EbU8i-4k6AkfxbS50pM}5dF%|~+8X_u?JslA+*3*&6( z!8HSGbuu0Uk5u2Snh$ln1PA*qyh^i?MryqW|q`}9=&5vwNs=Z({%c-~L=yNcKy0Sf}bk|As*Av=Oi)4{o z`u3M6?P8m^eIGObNj{?ocDA0UkCb8<$Ph`Kz4sNgPH!RB$X(vHJR@GTqw=g38}kNe z>^v^1g*$$709-1Y8@lBtGTVae+#kO>Tn;rJ@lUQOhy$M)opMI-C7lC&Zquut1EwEG z$?pAR9eo&XVAk?1j66ogBJ1?_ZzWeld<+Q*{_vM#{BM20@cDu2{lbC5Dq;!rG_qCB z++4C?`|#)er&3!x%aXB@Jy~Y9!~I!JKvw|6;^m}I1Kj>t2KM(yrRFA#kXjQ6)OvA$ zrTrz!)&yP4po&6w?@ivtA+G`@BlSIBQs?_rt z=DKm9s9!yF`S1Um!?X~=5pI2EKCe)pMp_O10d4i~FaiH@E#16glWKwYf0p#Bhc(oI zxecw3?w**PB|DBX6@G=43E3yi zyDgFWeu8K_*SlIV^s1S%BAeyTTy0|7(KlQ=S6A_9T|pq-XSpwQ`fWtsTrrkY_1@py zO#hrQ%Hp@gynoI-^cM)w-#OD-V0ahRo$igkx)KXe#dNljO4!@!hBf*Z!WE@FWUsgK zA$v)?ga{f<|N)Y$%vtLOh;YQ$V$8oNy}So&YCK;0Dbx|j?Z;vgp9(fFiGYH#l} zY{%sSle^F%{mp&Gj5^a_4M~7GrZ2Oxb0Ol%^~rPSb<;y!!`+B1XSJ`e9=()e(_%!^;ii+s zBX2VwOPNTY&nqIEW8toqGO1dS6s2Iu$rSXFV5ot3>@k?BN=m{vq4q;KA{)*Dcj0ug zl4ca&Ges&n)E_zPy&xBSVi32`hY0z~5`DDccW?gjGHKzd27>x7VX&*zmab=Wt9L@S zv<|{2-A2_XCWWDLoDHo5=YX`A1L)OkhR=DPsLVM9cXa}el?Tqxuw;+SQ_r=8FsQ-U zIV$3#*?TP6*yfcKbDnEhb~EZ&<#Pt#_@)Lq%!{;P@4G@`S_cBhd#iyvOm;2>LksPu z(3>P-!ua{TkG)5A=EnvI`Y-1I7NJ3D9(=a`7A*qKy7bAn12Ye$4WuqfWf(eaNuv)WS2_NVA;dk{VUbb|Ow_55;T$s9&{(49L|KkX%*Q2RFWE6@*pE ze8s(QJ|>hq*VD<%>U`HzIGs^&K{UyHDd&&a%eXsvvf0ywm=d04Ntm8F2fPYE?<+=6 z*UQYhY&Mfv7f4W%C#**6jm3*IO6P#d$g@fTM!GY$l9TdN3(H?ViF`~}qZ>9=9V5U~ z-Zw&WaY^7+=XB@Zt^%_s2@7O%kK$j#a8o+qp(ENJLI^UXXQJr1R+X;8=N0dmTl+B|ACv^7q+NWT5q!a40v`V{IcS|1He=r?v z$?92zz~6+jebygrGz;&cYN`7 z!u-mZa-^*0!;s3FY`(1D*rmcvdXH5PVggjQqz+(>^5r8sE!b81wrv(0e6tiTUB-=<6c-1ut~bLpU!;KAJ#4b%)QWH7@XJI4 z`yd=RZb7zIZX!O4@NSqK+A!INViW-y#$UQOZw#(nj=e|!VWa}bjCqmllq9FscCyG7 z+|;^s;OM=y{H;7dRy<}dsi>HO5@@Nh;h4!D3dRgY4lZ4rHWh&SW{(LU?R~MScN|aTtg`~U(Ons!IK{{GzU6Nmwi%lfJ5+gCbt+R zXv(QD>NUf5wM`uxlBaR>T5E`H(6-!b@0S$pI63yWuf0rhSm4?~4g>r&S+8?BI^s zrfZ!ZHS)=3f+|blavVMEN_Nd`#sd-?wHYaGKuO6Q77;i`?DCB?@ODkMkeUC;&(SQk zgp_Ev4#?=UMY*`IL^D)P-8+b%JdyeXvq1ZWBIz6W4d5#yk#x3>2ej zt7`EQlg?bum&vN33bWw>?q0$A;#W&^N5>o0+r+U0k_iR!ba(Fbsvo)zyp3H;a;|F6 z)v?lYh9wpxlJY>)5>nAbx3R5&o(Bb1Y#K!iI*iHI;5u3VYo3b;X|tHcbyk~2a12LZ z)9}evm1SH>`w#(Kf2mz`TDyR^Yv}ocspmcUOspq3`_3hmO#w4wGOz6Y;9ie3p^w?4 z0&In@sfSz23G;TA-c<+kd`^o<%nM`{NJKUDBzny++5x-%Ij0pOzUvbKHd_`v`l8Bm zct^>OqybtyRqgFnuD((T#oHycX;Gh87Q8xjSI|FOHBP|}OPeZ-XXeP%rUmSY1w4z) z{2Ji#9lpg|*LFb+K;TXeS`t^I3}vDhNy~TtUHBF3_sQJ)JMCsH4BWHik!AOj5oEgMve&ZeiqZ zDvQ^8xl_H9vuHpb2yNKGNMbj?FEzJ9cR{$W3`xakS$6xU?ziim&dv1C* zB(kP)54N%5F;RTV25rH^Yxyf^malaX8YH+9!jxx-e5Y-_!-sN)`I zY{Zj4n8IZx9n^!GjZ!tF3cq9><k%8`TBZ1-(c=H{hM zlh{#tle(m{r1g33Zx@~p*pjnCm8UL^Pk0vDDx1_{@5UYVEuaP4jZoQCTAWsa`0B@q zYY}{GBw}-jh}=}~EoygZJMG5UFYLCZx>HBhQ!3gF$y~k~;;~fB?1kZ~V%gxg+J_h) z?B6x9{wq4ot9^^~sG-EDZe}NQWKdQN`W!$d{8}R!Hp|TQ_1C%l7gcPAdLJPIk#0M4 zNA2wwrzH3?_iZh 44b86e8925c4 +Step 2/10 : ENV ORDS_HOME=/opt/oracle/ords RUN_FILE="runOrdsSSL.sh" + ---> Running in e22c5a03b869 +Removing intermediate container e22c5a03b869 + ---> 1421497abef8 +Step 3/10 : COPY $RUN_FILE $ORDS_HOME/ + ---> d96ac1477d2d +Step 4/10 : RUN yum -y install yum-utils bind-utils tree hostname openssl net-tools zip unzip tar wget vim-minimal which sudo expect procps && yum-config-manager --add-repo=http://yum.oracle.com/repo/OracleLinux/OL8/oracle/software/x86_64 && yum -y install java-11-openjdk-devel && yum -y install ords && yum -y install iproute && yum clean all + ---> Running in c08b8dac80a5 +Oracle Linux 8 BaseOS Latest (x86_64) 72 MB/s | 49 MB 00:00 +Oracle Linux 8 Application Stream (x86_64) 88 MB/s | 37 MB 00:00 +Last metadata expiration check: 0:00:07 ago on Mon 12 Sep 2022 03:23:32 PM UTC. +Package yum-utils-4.0.21-11.0.1.el8.noarch is already installed. +Package vim-minimal-2:8.0.1763-19.0.1.el8_6.2.x86_64 is already installed. +Package procps-ng-3.3.15-6.0.1.el8.x86_64 is already installed. +Dependencies resolved. +================================================================================ + Package Arch Version Repository Size +================================================================================ +Installing: + bind-utils x86_64 32:9.11.36-3.el8 ol8_appstream 452 k + expect x86_64 5.45.4-5.el8 ol8_baseos_latest 266 k + hostname x86_64 3.20-6.el8 ol8_baseos_latest 32 k + net-tools x86_64 2.0-0.52.20160912git.el8 ol8_baseos_latest 322 k + openssl x86_64 1:1.1.1k-7.el8_6 ol8_baseos_latest 709 k + sudo x86_64 1.8.29-8.el8 ol8_baseos_latest 925 k + tar x86_64 2:1.30-5.el8 ol8_baseos_latest 838 k + tree x86_64 1.7.0-15.el8 ol8_baseos_latest 59 k + unzip x86_64 6.0-46.0.1.el8 ol8_baseos_latest 196 k + wget x86_64 1.19.5-10.0.1.el8 ol8_appstream 734 k + which x86_64 2.21-17.el8 ol8_baseos_latest 49 k + zip x86_64 3.0-23.el8 ol8_baseos_latest 270 k +Upgrading: + openssl-libs x86_64 1:1.1.1k-7.el8_6 ol8_baseos_latest 1.5 M + vim-minimal x86_64 2:8.0.1763-19.0.1.el8_6.4 ol8_baseos_latest 575 k +Installing dependencies: + bind-libs x86_64 32:9.11.36-3.el8 ol8_appstream 175 k + bind-libs-lite x86_64 32:9.11.36-3.el8 ol8_appstream 1.2 M + bind-license noarch 32:9.11.36-3.el8 ol8_appstream 103 k + fstrm x86_64 0.6.1-2.el8 ol8_appstream 29 k + libmaxminddb x86_64 1.2.0-10.el8 ol8_appstream 33 k + libmetalink x86_64 0.1.3-7.el8 ol8_baseos_latest 32 k + protobuf-c x86_64 1.3.0-6.el8 ol8_appstream 37 k + python3-bind noarch 32:9.11.36-3.el8 ol8_appstream 150 k + python3-ply noarch 3.9-9.el8 ol8_baseos_latest 111 k + tcl x86_64 1:8.6.8-2.el8 ol8_baseos_latest 1.1 M + +Transaction Summary +================================================================================ +Install 22 Packages +Upgrade 2 Packages + +Total download size: 9.7 M +Downloading Packages: +(1/24): expect-5.45.4-5.el8.x86_64.rpm 158 kB/s | 266 kB 00:01 +(2/24): hostname-3.20-6.el8.x86_64.rpm 18 kB/s | 32 kB 00:01 +(3/24): libmetalink-0.1.3-7.el8.x86_64.rpm 18 kB/s | 32 kB 00:01 +(4/24): net-tools-2.0-0.52.20160912git.el8.x86_ 2.3 MB/s | 322 kB 00:00 +(5/24): openssl-1.1.1k-7.el8_6.x86_64.rpm 4.0 MB/s | 709 kB 00:00 +(6/24): python3-ply-3.9-9.el8.noarch.rpm 538 kB/s | 111 kB 00:00 +(7/24): sudo-1.8.29-8.el8.x86_64.rpm 5.0 MB/s | 925 kB 00:00 +(8/24): tar-1.30-5.el8.x86_64.rpm 4.2 MB/s | 838 kB 00:00 +(9/24): unzip-6.0-46.0.1.el8.x86_64.rpm 3.6 MB/s | 196 kB 00:00 +(10/24): tcl-8.6.8-2.el8.x86_64.rpm 4.1 MB/s | 1.1 MB 00:00 +(11/24): which-2.21-17.el8.x86_64.rpm 613 kB/s | 49 kB 00:00 +(12/24): tree-1.7.0-15.el8.x86_64.rpm 208 kB/s | 59 kB 00:00 +(13/24): bind-libs-9.11.36-3.el8.x86_64.rpm 1.3 MB/s | 175 kB 00:00 +(14/24): bind-license-9.11.36-3.el8.noarch.rpm 2.6 MB/s | 103 kB 00:00 +(15/24): bind-libs-lite-9.11.36-3.el8.x86_64.rp 6.8 MB/s | 1.2 MB 00:00 +(16/24): bind-utils-9.11.36-3.el8.x86_64.rpm 3.6 MB/s | 452 kB 00:00 +(17/24): zip-3.0-23.el8.x86_64.rpm 804 kB/s | 270 kB 00:00 +(18/24): libmaxminddb-1.2.0-10.el8.x86_64.rpm 529 kB/s | 33 kB 00:00 +(19/24): fstrm-0.6.1-2.el8.x86_64.rpm 161 kB/s | 29 kB 00:00 +(20/24): python3-bind-9.11.36-3.el8.noarch.rpm 2.0 MB/s | 150 kB 00:00 +(21/24): protobuf-c-1.3.0-6.el8.x86_64.rpm 351 kB/s | 37 kB 00:00 +(22/24): vim-minimal-8.0.1763-19.0.1.el8_6.4.x8 6.4 MB/s | 575 kB 00:00 +(23/24): wget-1.19.5-10.0.1.el8.x86_64.rpm 3.3 MB/s | 734 kB 00:00 +(24/24): openssl-libs-1.1.1k-7.el8_6.x86_64.rpm 6.8 MB/s | 1.5 MB 00:00 +-------------------------------------------------------------------------------- +Total 3.3 MB/s | 9.7 MB 00:02 +Running transaction check +Transaction check succeeded. +Running transaction test +Transaction test succeeded. +Running transaction + Preparing : 1/1 + Upgrading : openssl-libs-1:1.1.1k-7.el8_6.x86_64 1/26 + Running scriptlet: openssl-libs-1:1.1.1k-7.el8_6.x86_64 1/26 + Installing : protobuf-c-1.3.0-6.el8.x86_64 2/26 + Installing : libmaxminddb-1.2.0-10.el8.x86_64 3/26 + Running scriptlet: libmaxminddb-1.2.0-10.el8.x86_64 3/26 + Installing : fstrm-0.6.1-2.el8.x86_64 4/26 + Installing : bind-license-32:9.11.36-3.el8.noarch 5/26 + Installing : bind-libs-lite-32:9.11.36-3.el8.x86_64 6/26 + Installing : bind-libs-32:9.11.36-3.el8.x86_64 7/26 + Upgrading : vim-minimal-2:8.0.1763-19.0.1.el8_6.4.x86_64 8/26 + Installing : unzip-6.0-46.0.1.el8.x86_64 9/26 + Installing : tcl-1:8.6.8-2.el8.x86_64 10/26 + Running scriptlet: tcl-1:8.6.8-2.el8.x86_64 10/26 + Installing : python3-ply-3.9-9.el8.noarch 11/26 + Installing : python3-bind-32:9.11.36-3.el8.noarch 12/26 + Installing : libmetalink-0.1.3-7.el8.x86_64 13/26 + Installing : wget-1.19.5-10.0.1.el8.x86_64 14/26 + Running scriptlet: wget-1.19.5-10.0.1.el8.x86_64 14/26 + Installing : bind-utils-32:9.11.36-3.el8.x86_64 15/26 + Installing : expect-5.45.4-5.el8.x86_64 16/26 + Installing : zip-3.0-23.el8.x86_64 17/26 + Installing : sudo-1.8.29-8.el8.x86_64 18/26 + Running scriptlet: sudo-1.8.29-8.el8.x86_64 18/26 + Installing : openssl-1:1.1.1k-7.el8_6.x86_64 19/26 + Installing : which-2.21-17.el8.x86_64 20/26 + Installing : tree-1.7.0-15.el8.x86_64 21/26 + Installing : tar-2:1.30-5.el8.x86_64 22/26 + Running scriptlet: tar-2:1.30-5.el8.x86_64 22/26 + Installing : net-tools-2.0-0.52.20160912git.el8.x86_64 23/26 + Running scriptlet: net-tools-2.0-0.52.20160912git.el8.x86_64 23/26 + Installing : hostname-3.20-6.el8.x86_64 24/26 + Running scriptlet: hostname-3.20-6.el8.x86_64 24/26 + Cleanup : vim-minimal-2:8.0.1763-19.0.1.el8_6.2.x86_64 25/26 + Cleanup : openssl-libs-1:1.1.1k-6.el8_5.x86_64 26/26 + Running scriptlet: openssl-libs-1:1.1.1k-6.el8_5.x86_64 26/26 + Verifying : expect-5.45.4-5.el8.x86_64 1/26 + Verifying : hostname-3.20-6.el8.x86_64 2/26 + Verifying : libmetalink-0.1.3-7.el8.x86_64 3/26 + Verifying : net-tools-2.0-0.52.20160912git.el8.x86_64 4/26 + Verifying : openssl-1:1.1.1k-7.el8_6.x86_64 5/26 + Verifying : python3-ply-3.9-9.el8.noarch 6/26 + Verifying : sudo-1.8.29-8.el8.x86_64 7/26 + Verifying : tar-2:1.30-5.el8.x86_64 8/26 + Verifying : tcl-1:8.6.8-2.el8.x86_64 9/26 + Verifying : tree-1.7.0-15.el8.x86_64 10/26 + Verifying : unzip-6.0-46.0.1.el8.x86_64 11/26 + Verifying : which-2.21-17.el8.x86_64 12/26 + Verifying : zip-3.0-23.el8.x86_64 13/26 + Verifying : bind-libs-32:9.11.36-3.el8.x86_64 14/26 + Verifying : bind-libs-lite-32:9.11.36-3.el8.x86_64 15/26 + Verifying : bind-license-32:9.11.36-3.el8.noarch 16/26 + Verifying : bind-utils-32:9.11.36-3.el8.x86_64 17/26 + Verifying : fstrm-0.6.1-2.el8.x86_64 18/26 + Verifying : libmaxminddb-1.2.0-10.el8.x86_64 19/26 + Verifying : protobuf-c-1.3.0-6.el8.x86_64 20/26 + Verifying : python3-bind-32:9.11.36-3.el8.noarch 21/26 + Verifying : wget-1.19.5-10.0.1.el8.x86_64 22/26 + Verifying : openssl-libs-1:1.1.1k-7.el8_6.x86_64 23/26 + Verifying : openssl-libs-1:1.1.1k-6.el8_5.x86_64 24/26 + Verifying : vim-minimal-2:8.0.1763-19.0.1.el8_6.4.x86_64 25/26 + Verifying : vim-minimal-2:8.0.1763-19.0.1.el8_6.2.x86_64 26/26 + +Upgraded: + openssl-libs-1:1.1.1k-7.el8_6.x86_64 + vim-minimal-2:8.0.1763-19.0.1.el8_6.4.x86_64 +Installed: + bind-libs-32:9.11.36-3.el8.x86_64 + bind-libs-lite-32:9.11.36-3.el8.x86_64 + bind-license-32:9.11.36-3.el8.noarch + bind-utils-32:9.11.36-3.el8.x86_64 + expect-5.45.4-5.el8.x86_64 + fstrm-0.6.1-2.el8.x86_64 + hostname-3.20-6.el8.x86_64 + libmaxminddb-1.2.0-10.el8.x86_64 + libmetalink-0.1.3-7.el8.x86_64 + net-tools-2.0-0.52.20160912git.el8.x86_64 + openssl-1:1.1.1k-7.el8_6.x86_64 + protobuf-c-1.3.0-6.el8.x86_64 + python3-bind-32:9.11.36-3.el8.noarch + python3-ply-3.9-9.el8.noarch + sudo-1.8.29-8.el8.x86_64 + tar-2:1.30-5.el8.x86_64 + tcl-1:8.6.8-2.el8.x86_64 + tree-1.7.0-15.el8.x86_64 + unzip-6.0-46.0.1.el8.x86_64 + wget-1.19.5-10.0.1.el8.x86_64 + which-2.21-17.el8.x86_64 + zip-3.0-23.el8.x86_64 + +Complete! +Adding repo from: http://yum.oracle.com/repo/OracleLinux/OL8/oracle/software/x86_64 +created by dnf config-manager from http://yum.o 221 kB/s | 45 kB 00:00 +Dependencies resolved. +============================================================================================= + Package Arch Version Repository Size +============================================================================================= +Installing: + java-11-openjdk-devel x86_64 1:11.0.16.1.1-1.el8_6 ol8_appstream 3.4 M +Installing dependencies: + alsa-lib x86_64 1.2.6.1-3.el8 ol8_appstream 491 k + avahi-libs x86_64 0.7-20.el8 ol8_baseos_latest 62 k + copy-jdk-configs noarch 4.0-2.el8 ol8_appstream 30 k + crypto-policies-scripts noarch 20211116-1.gitae470d6.el8 ol8_baseos_latest 83 k + cups-libs x86_64 1:2.2.6-45.el8_6.2 ol8_baseos_latest 434 k + giflib x86_64 5.1.4-3.el8 ol8_appstream 51 k + graphite2 x86_64 1.3.10-10.el8 ol8_appstream 122 k + harfbuzz x86_64 1.7.5-3.el8 ol8_appstream 295 k + java-11-openjdk x86_64 1:11.0.16.1.1-1.el8_6 ol8_appstream 272 k + java-11-openjdk-headless x86_64 1:11.0.16.1.1-1.el8_6 ol8_appstream 40 M + javapackages-filesystem noarch 5.3.0-1.module+el8+5136+7ff78f74 ol8_appstream 30 k + lcms2 x86_64 2.9-2.el8 ol8_appstream 164 k + libX11 x86_64 1.6.8-5.el8 ol8_appstream 611 k + libX11-common noarch 1.6.8-5.el8 ol8_appstream 158 k + libXau x86_64 1.0.9-3.el8 ol8_appstream 37 k + libXcomposite x86_64 0.4.4-14.el8 ol8_appstream 28 k + libXext x86_64 1.3.4-1.el8 ol8_appstream 45 k + libXi x86_64 1.7.10-1.el8 ol8_appstream 49 k + libXrender x86_64 0.9.10-7.el8 ol8_appstream 33 k + libXtst x86_64 1.2.3-7.el8 ol8_appstream 22 k + libfontenc x86_64 1.1.3-8.el8 ol8_appstream 37 k + libjpeg-turbo x86_64 1.5.3-12.el8 ol8_appstream 157 k + libpkgconf x86_64 1.4.2-1.el8 ol8_baseos_latest 35 k + libxcb x86_64 1.13.1-1.el8 ol8_appstream 231 k + lksctp-tools x86_64 1.0.18-3.el8 ol8_baseos_latest 100 k + lua x86_64 5.3.4-12.el8 ol8_appstream 192 k + nspr x86_64 4.32.0-1.el8_4 ol8_appstream 142 k + nss x86_64 3.67.0-7.el8_5 ol8_appstream 741 k + nss-softokn x86_64 3.67.0-7.el8_5 ol8_appstream 487 k + nss-softokn-freebl x86_64 3.67.0-7.el8_5 ol8_appstream 395 k + nss-sysinit x86_64 3.67.0-7.el8_5 ol8_appstream 73 k + nss-util x86_64 3.67.0-7.el8_5 ol8_appstream 137 k + pkgconf x86_64 1.4.2-1.el8 ol8_baseos_latest 38 k + pkgconf-m4 noarch 1.4.2-1.el8 ol8_baseos_latest 17 k + pkgconf-pkg-config x86_64 1.4.2-1.el8 ol8_baseos_latest 15 k + ttmkfdir x86_64 3.0.9-54.el8 ol8_appstream 62 k + tzdata-java noarch 2022c-1.el8 ol8_appstream 186 k + xorg-x11-font-utils x86_64 1:7.5-41.el8 ol8_appstream 104 k + xorg-x11-fonts-Type1 noarch 7.5-19.el8 ol8_appstream 522 k +Enabling module streams: + javapackages-runtime 201801 + +Transaction Summary +============================================================================================= +Install 40 Packages + +Total download size: 50 M +Installed size: 194 M +Downloading Packages: +(1/40): crypto-policies-scripts-20211116-1.gita 1.3 MB/s | 83 kB 00:00 +(2/40): avahi-libs-0.7-20.el8.x86_64.rpm 952 kB/s | 62 kB 00:00 +(3/40): libpkgconf-1.4.2-1.el8.x86_64.rpm 2.2 MB/s | 35 kB 00:00 +(4/40): cups-libs-2.2.6-45.el8_6.2.x86_64.rpm 4.9 MB/s | 434 kB 00:00 +(5/40): lksctp-tools-1.0.18-3.el8.x86_64.rpm 3.9 MB/s | 100 kB 00:00 +(6/40): pkgconf-1.4.2-1.el8.x86_64.rpm 2.3 MB/s | 38 kB 00:00 +(7/40): pkgconf-m4-1.4.2-1.el8.noarch.rpm 1.1 MB/s | 17 kB 00:00 +(8/40): pkgconf-pkg-config-1.4.2-1.el8.x86_64.r 1.1 MB/s | 15 kB 00:00 +(9/40): copy-jdk-configs-4.0-2.el8.noarch.rpm 1.8 MB/s | 30 kB 00:00 +(10/40): giflib-5.1.4-3.el8.x86_64.rpm 3.0 MB/s | 51 kB 00:00 +(11/40): alsa-lib-1.2.6.1-3.el8.x86_64.rpm 12 MB/s | 491 kB 00:00 +(12/40): graphite2-1.3.10-10.el8.x86_64.rpm 5.9 MB/s | 122 kB 00:00 +(13/40): harfbuzz-1.7.5-3.el8.x86_64.rpm 13 MB/s | 295 kB 00:00 +(14/40): java-11-openjdk-11.0.16.1.1-1.el8_6.x8 15 MB/s | 272 kB 00:00 +(15/40): javapackages-filesystem-5.3.0-1.module 2.1 MB/s | 30 kB 00:00 +(16/40): lcms2-2.9-2.el8.x86_64.rpm 9.5 MB/s | 164 kB 00:00 +(17/40): libX11-1.6.8-5.el8.x86_64.rpm 24 MB/s | 611 kB 00:00 +(18/40): java-11-openjdk-devel-11.0.16.1.1-1.el 40 MB/s | 3.4 MB 00:00 +(19/40): libX11-common-1.6.8-5.el8.noarch.rpm 8.6 MB/s | 158 kB 00:00 +(20/40): libXau-1.0.9-3.el8.x86_64.rpm 2.6 MB/s | 37 kB 00:00 +(21/40): libXcomposite-0.4.4-14.el8.x86_64.rpm 2.2 MB/s | 28 kB 00:00 +(22/40): libXext-1.3.4-1.el8.x86_64.rpm 2.7 MB/s | 45 kB 00:00 +(23/40): libXi-1.7.10-1.el8.x86_64.rpm 2.8 MB/s | 49 kB 00:00 +(24/40): libXrender-0.9.10-7.el8.x86_64.rpm 2.4 MB/s | 33 kB 00:00 +(25/40): libXtst-1.2.3-7.el8.x86_64.rpm 1.6 MB/s | 22 kB 00:00 +(26/40): libfontenc-1.1.3-8.el8.x86_64.rpm 2.7 MB/s | 37 kB 00:00 +(27/40): libjpeg-turbo-1.5.3-12.el8.x86_64.rpm 9.6 MB/s | 157 kB 00:00 +(28/40): libxcb-1.13.1-1.el8.x86_64.rpm 13 MB/s | 231 kB 00:00 +(29/40): lua-5.3.4-12.el8.x86_64.rpm 11 MB/s | 192 kB 00:00 +(30/40): nspr-4.32.0-1.el8_4.x86_64.rpm 9.2 MB/s | 142 kB 00:00 +(31/40): nss-3.67.0-7.el8_5.x86_64.rpm 31 MB/s | 741 kB 00:00 +(32/40): nss-softokn-3.67.0-7.el8_5.x86_64.rpm 24 MB/s | 487 kB 00:00 +(33/40): nss-softokn-freebl-3.67.0-7.el8_5.x86_ 18 MB/s | 395 kB 00:00 +(34/40): nss-sysinit-3.67.0-7.el8_5.x86_64.rpm 4.3 MB/s | 73 kB 00:00 +(35/40): nss-util-3.67.0-7.el8_5.x86_64.rpm 8.7 MB/s | 137 kB 00:00 +(36/40): ttmkfdir-3.0.9-54.el8.x86_64.rpm 4.0 MB/s | 62 kB 00:00 +(37/40): tzdata-java-2022c-1.el8.noarch.rpm 12 MB/s | 186 kB 00:00 +(38/40): xorg-x11-font-utils-7.5-41.el8.x86_64. 6.0 MB/s | 104 kB 00:00 +(39/40): xorg-x11-fonts-Type1-7.5-19.el8.noarch 23 MB/s | 522 kB 00:00 +(40/40): java-11-openjdk-headless-11.0.16.1.1-1 73 MB/s | 40 MB 00:00 +-------------------------------------------------------------------------------- +Total 71 MB/s | 50 MB 00:00 +Running transaction check +Transaction check succeeded. +Running transaction test +Transaction test succeeded. +Running transaction + Running scriptlet: copy-jdk-configs-4.0-2.el8.noarch 1/1 + Running scriptlet: java-11-openjdk-headless-1:11.0.16.1.1-1.el8_6.x86_6 1/1 + Preparing : 1/1 + Installing : nspr-4.32.0-1.el8_4.x86_64 1/40 + Running scriptlet: nspr-4.32.0-1.el8_4.x86_64 1/40 + Installing : nss-util-3.67.0-7.el8_5.x86_64 2/40 + Installing : libjpeg-turbo-1.5.3-12.el8.x86_64 3/40 + Installing : nss-softokn-freebl-3.67.0-7.el8_5.x86_64 4/40 + Installing : nss-softokn-3.67.0-7.el8_5.x86_64 5/40 + Installing : tzdata-java-2022c-1.el8.noarch 6/40 + Installing : ttmkfdir-3.0.9-54.el8.x86_64 7/40 + Installing : lua-5.3.4-12.el8.x86_64 8/40 + Installing : copy-jdk-configs-4.0-2.el8.noarch 9/40 + Installing : libfontenc-1.1.3-8.el8.x86_64 10/40 + Installing : libXau-1.0.9-3.el8.x86_64 11/40 + Installing : libxcb-1.13.1-1.el8.x86_64 12/40 + Installing : libX11-common-1.6.8-5.el8.noarch 13/40 + Installing : libX11-1.6.8-5.el8.x86_64 14/40 + Installing : libXext-1.3.4-1.el8.x86_64 15/40 + Installing : libXi-1.7.10-1.el8.x86_64 16/40 + Installing : libXtst-1.2.3-7.el8.x86_64 17/40 + Installing : libXcomposite-0.4.4-14.el8.x86_64 18/40 + Installing : libXrender-0.9.10-7.el8.x86_64 19/40 + Installing : lcms2-2.9-2.el8.x86_64 20/40 + Running scriptlet: lcms2-2.9-2.el8.x86_64 20/40 + Installing : javapackages-filesystem-5.3.0-1.module+el8+5136+7f 21/40 + Installing : graphite2-1.3.10-10.el8.x86_64 22/40 + Installing : harfbuzz-1.7.5-3.el8.x86_64 23/40 + Running scriptlet: harfbuzz-1.7.5-3.el8.x86_64 23/40 + Installing : giflib-5.1.4-3.el8.x86_64 24/40 + Installing : alsa-lib-1.2.6.1-3.el8.x86_64 25/40 + Running scriptlet: alsa-lib-1.2.6.1-3.el8.x86_64 25/40 + Installing : pkgconf-m4-1.4.2-1.el8.noarch 26/40 + Installing : lksctp-tools-1.0.18-3.el8.x86_64 27/40 + Running scriptlet: lksctp-tools-1.0.18-3.el8.x86_64 27/40 + Installing : libpkgconf-1.4.2-1.el8.x86_64 28/40 + Installing : pkgconf-1.4.2-1.el8.x86_64 29/40 + Installing : pkgconf-pkg-config-1.4.2-1.el8.x86_64 30/40 + Installing : xorg-x11-font-utils-1:7.5-41.el8.x86_64 31/40 + Installing : xorg-x11-fonts-Type1-7.5-19.el8.noarch 32/40 + Running scriptlet: xorg-x11-fonts-Type1-7.5-19.el8.noarch 32/40 + Installing : crypto-policies-scripts-20211116-1.gitae470d6.el8. 33/40 + Installing : nss-sysinit-3.67.0-7.el8_5.x86_64 34/40 + Installing : nss-3.67.0-7.el8_5.x86_64 35/40 + Installing : avahi-libs-0.7-20.el8.x86_64 36/40 + Installing : cups-libs-1:2.2.6-45.el8_6.2.x86_64 37/40 + Installing : java-11-openjdk-headless-1:11.0.16.1.1-1.el8_6.x86 38/40 + Running scriptlet: java-11-openjdk-headless-1:11.0.16.1.1-1.el8_6.x86 38/40 + Installing : java-11-openjdk-1:11.0.16.1.1-1.el8_6.x86_64 39/40 + Running scriptlet: java-11-openjdk-1:11.0.16.1.1-1.el8_6.x86_64 39/40 + Installing : java-11-openjdk-devel-1:11.0.16.1.1-1.el8_6.x86_64 40/40 + Running scriptlet: java-11-openjdk-devel-1:11.0.16.1.1-1.el8_6.x86_64 40/40 + Running scriptlet: copy-jdk-configs-4.0-2.el8.noarch 40/40 + Running scriptlet: crypto-policies-scripts-20211116-1.gitae470d6.el8. 40/40 + Running scriptlet: nss-3.67.0-7.el8_5.x86_64 40/40 + Running scriptlet: java-11-openjdk-headless-1:11.0.16.1.1-1.el8_6.x86 40/40 + Running scriptlet: java-11-openjdk-1:11.0.16.1.1-1.el8_6.x86_64 40/40 + Running scriptlet: java-11-openjdk-devel-1:11.0.16.1.1-1.el8_6.x86_64 40/40 + Verifying : avahi-libs-0.7-20.el8.x86_64 1/40 + Verifying : crypto-policies-scripts-20211116-1.gitae470d6.el8. 2/40 + Verifying : cups-libs-1:2.2.6-45.el8_6.2.x86_64 3/40 + Verifying : libpkgconf-1.4.2-1.el8.x86_64 4/40 + Verifying : lksctp-tools-1.0.18-3.el8.x86_64 5/40 + Verifying : pkgconf-1.4.2-1.el8.x86_64 6/40 + Verifying : pkgconf-m4-1.4.2-1.el8.noarch 7/40 + Verifying : pkgconf-pkg-config-1.4.2-1.el8.x86_64 8/40 + Verifying : alsa-lib-1.2.6.1-3.el8.x86_64 9/40 + Verifying : copy-jdk-configs-4.0-2.el8.noarch 10/40 + Verifying : giflib-5.1.4-3.el8.x86_64 11/40 + Verifying : graphite2-1.3.10-10.el8.x86_64 12/40 + Verifying : harfbuzz-1.7.5-3.el8.x86_64 13/40 + Verifying : java-11-openjdk-1:11.0.16.1.1-1.el8_6.x86_64 14/40 + Verifying : java-11-openjdk-devel-1:11.0.16.1.1-1.el8_6.x86_64 15/40 + Verifying : java-11-openjdk-headless-1:11.0.16.1.1-1.el8_6.x86 16/40 + Verifying : javapackages-filesystem-5.3.0-1.module+el8+5136+7f 17/40 + Verifying : lcms2-2.9-2.el8.x86_64 18/40 + Verifying : libX11-1.6.8-5.el8.x86_64 19/40 + Verifying : libX11-common-1.6.8-5.el8.noarch 20/40 + Verifying : libXau-1.0.9-3.el8.x86_64 21/40 + Verifying : libXcomposite-0.4.4-14.el8.x86_64 22/40 + Verifying : libXext-1.3.4-1.el8.x86_64 23/40 + Verifying : libXi-1.7.10-1.el8.x86_64 24/40 + Verifying : libXrender-0.9.10-7.el8.x86_64 25/40 + Verifying : libXtst-1.2.3-7.el8.x86_64 26/40 + Verifying : libfontenc-1.1.3-8.el8.x86_64 27/40 + Verifying : libjpeg-turbo-1.5.3-12.el8.x86_64 28/40 + Verifying : libxcb-1.13.1-1.el8.x86_64 29/40 + Verifying : lua-5.3.4-12.el8.x86_64 30/40 + Verifying : nspr-4.32.0-1.el8_4.x86_64 31/40 + Verifying : nss-3.67.0-7.el8_5.x86_64 32/40 + Verifying : nss-softokn-3.67.0-7.el8_5.x86_64 33/40 + Verifying : nss-softokn-freebl-3.67.0-7.el8_5.x86_64 34/40 + Verifying : nss-sysinit-3.67.0-7.el8_5.x86_64 35/40 + Verifying : nss-util-3.67.0-7.el8_5.x86_64 36/40 + Verifying : ttmkfdir-3.0.9-54.el8.x86_64 37/40 + Verifying : tzdata-java-2022c-1.el8.noarch 38/40 + Verifying : xorg-x11-font-utils-1:7.5-41.el8.x86_64 39/40 + Verifying : xorg-x11-fonts-Type1-7.5-19.el8.noarch 40/40 + +Installed: + alsa-lib-1.2.6.1-3.el8.x86_64 + avahi-libs-0.7-20.el8.x86_64 + copy-jdk-configs-4.0-2.el8.noarch + crypto-policies-scripts-20211116-1.gitae470d6.el8.noarch + cups-libs-1:2.2.6-45.el8_6.2.x86_64 + giflib-5.1.4-3.el8.x86_64 + graphite2-1.3.10-10.el8.x86_64 + harfbuzz-1.7.5-3.el8.x86_64 + java-11-openjdk-1:11.0.16.1.1-1.el8_6.x86_64 + java-11-openjdk-devel-1:11.0.16.1.1-1.el8_6.x86_64 + java-11-openjdk-headless-1:11.0.16.1.1-1.el8_6.x86_64 + javapackages-filesystem-5.3.0-1.module+el8+5136+7ff78f74.noarch + lcms2-2.9-2.el8.x86_64 + libX11-1.6.8-5.el8.x86_64 + libX11-common-1.6.8-5.el8.noarch + libXau-1.0.9-3.el8.x86_64 + libXcomposite-0.4.4-14.el8.x86_64 + libXext-1.3.4-1.el8.x86_64 + libXi-1.7.10-1.el8.x86_64 + libXrender-0.9.10-7.el8.x86_64 + libXtst-1.2.3-7.el8.x86_64 + libfontenc-1.1.3-8.el8.x86_64 + libjpeg-turbo-1.5.3-12.el8.x86_64 + libpkgconf-1.4.2-1.el8.x86_64 + libxcb-1.13.1-1.el8.x86_64 + lksctp-tools-1.0.18-3.el8.x86_64 + lua-5.3.4-12.el8.x86_64 + nspr-4.32.0-1.el8_4.x86_64 + nss-3.67.0-7.el8_5.x86_64 + nss-softokn-3.67.0-7.el8_5.x86_64 + nss-softokn-freebl-3.67.0-7.el8_5.x86_64 + nss-sysinit-3.67.0-7.el8_5.x86_64 + nss-util-3.67.0-7.el8_5.x86_64 + pkgconf-1.4.2-1.el8.x86_64 + pkgconf-m4-1.4.2-1.el8.noarch + pkgconf-pkg-config-1.4.2-1.el8.x86_64 + ttmkfdir-3.0.9-54.el8.x86_64 + tzdata-java-2022c-1.el8.noarch + xorg-x11-font-utils-1:7.5-41.el8.x86_64 + xorg-x11-fonts-Type1-7.5-19.el8.noarch + +Complete! +Last metadata expiration check: 0:00:10 ago on Mon 12 Sep 2022 03:23:49 PM UTC. +Dependencies resolved. +============================================================================================== + Package + Arch Version Repository Size +============================================================================================== +Installing: + ords noarch 22.2.1-2.el8 yum.oracle.com_repo_OracleLinux_OL8_oracle_software_x86_64 83 M +Installing dependencies: + lsof x86_64 4.93.2-1.el8 ol8_baseos_latest 253 k + +Transaction Summary +============================================================================================== +Install 2 Packages + +Total download size: 83 M +Installed size: 87 M +Downloading Packages: +(1/2): lsof-4.93.2-1.el8.x86_64.rpm 3.0 MB/s | 253 kB 00:00 +(2/2): ords-22.2.1-2.el8.noarch.rpm 56 MB/s | 83 MB 00:01 +-------------------------------------------------------------------------------- +Total 56 MB/s | 83 MB 00:01 +Running transaction check +Transaction check succeeded. +Running transaction test +Transaction test succeeded. +Running transaction + Preparing : 1/1 + Installing : lsof-4.93.2-1.el8.x86_64 1/2 + Running scriptlet: ords-22.2.1-2.el8.noarch 2/2 + Installing : ords-22.2.1-2.el8.noarch 2/2 + Running scriptlet: ords-22.2.1-2.el8.noarch 2/2 +INFO: Before starting ORDS service, run the below command as user oracle: + ords --config /etc/ords/config install + + Verifying : lsof-4.93.2-1.el8.x86_64 1/2 + Verifying : ords-22.2.1-2.el8.noarch 2/2 + +Installed: + lsof-4.93.2-1.el8.x86_64 ords-22.2.1-2.el8.noarch + +Complete! +Last metadata expiration check: 0:00:15 ago on Mon 12 Sep 2022 03:23:49 PM UTC. +Package iproute-5.15.0-4.el8.x86_64 is already installed. +Dependencies resolved. +Nothing to do. +Complete! +24 files removed +Removing intermediate container c08b8dac80a5 + ---> bb1a717f3e6e +Step 5/10 : RUN mkdir -p $ORDS_HOME/doc_root && mkdir -p $ORDS_HOME/error && mkdir -p $ORDS_HOME/secrets && chmod ug+x $ORDS_HOME/*.sh && groupadd -g 54322 dba && usermod -u 54321 -d /home/oracle -g dba -m -s /bin/bash oracle && chown -R oracle:dba $ORDS_HOME && echo "oracle ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + ---> Running in 0103c070f4b6 +Removing intermediate container 0103c070f4b6 + ---> 089d06d9b198 +Step 6/10 : USER oracle + ---> Running in 51b1846c8c6f +Removing intermediate container 51b1846c8c6f + ---> 6c7b115954a4 +Step 7/10 : WORKDIR /home/oracle + ---> Running in 5862e2bc8df9 +Removing intermediate container 5862e2bc8df9 + ---> 28543543a88c +Step 8/10 : VOLUME ["$ORDS_HOME/config/ords"] + ---> Running in 465398d6f2bb +Removing intermediate container 465398d6f2bb + ---> 4037eb7f2f12 +Step 9/10 : EXPOSE 8888 + ---> Running in 2813ab5473f6 +Removing intermediate container 2813ab5473f6 + ---> 3410f1be2fff +Step 10/10 : CMD $ORDS_HOME/$RUN_FILE + ---> Running in 0a9a72408177 +Removing intermediate container 0a9a72408177 + ---> 2ef5dc95701b +Successfully built 2ef5dc95701b +Successfully tagged oracle/ords-dboper:22.2.1 + diff --git a/docs/multitenant/provisioning/ords_image.md b/docs/multitenant/provisioning/ords_image.md new file mode 100644 index 00000000..21abcbab --- /dev/null +++ b/docs/multitenant/provisioning/ords_image.md @@ -0,0 +1,64 @@ + + +# Build ORDS Docker Image + +In the below steps, we are building an ORDS Docker Image for ORDS Software. The image built can be later pushed to a local repository to be used later for a deployment. + +**NOTE:** It is assumed that before this step, you have followed the [prerequisite](./../README.md#prerequsites-to-manage-pdb-life-cycle-using-oracle-db-operator-on-prem-database-controller) steps. + +1. Clone the software using git: +```sh + git clone git@orahub.oci.oraclecorp.com:rac-docker-dev/oracle-database-operator.git + cd oracle-database-operator/ords/ +``` + +2. Login to the registry: container-registry.oracle.com + +**NOTE:** To login to this registry, you will need to the URL https://container-registry.oracle.com , Sign in, then click on "Java" and then accept the agreement. + +```bash +docker login container-registry.oracle.com +``` + +3. Login to a repo where you want to push your docker image (if needed) to pull during deployment in your environment. + +```bash +docker login +``` + +4. Build the docker image by using below command: + +```bash +docker build -t oracle/ords-dboper:ords-latest . +``` + +5. Check the docker image details using: + +```bash +docker images +``` + +6. Tag and push the image to your docker repository. + +NOTE: We have the repo as `phx.ocir.io//oracle/ords:latest`. Please change as per your environment. + +```bash +docker tag oracle/ords-dboper:ords-latest phx.ocir.io//oracle/ords:latest +docker push phx.ocir.io//oracle/ords:latest +``` + +7. Verify the image pushed to your docker repository. + +You can refer to below sample output for above steps as well. + +8. Create a Kubernetes Secret for your docker repository to pull the image during deployment using the below command: + +```bash +kubectl create secret generic container-registry-secret --from-file=.dockerconfigjson=./.docker/config.json --type=kubernetes.io/dockerconfigjson -n oracle-database-operator-system +``` + +This Kubernetes secret will be provided in the .yaml file against the parameter `ordsImagePullSecret` to pull the ORDS Docker Image from your docker repository (if its a private repository). + +## Sample Output + +[Here](./ords_image.log) is the sample output for docker image created for ORDS latest version diff --git a/docs/onpremdb/provisioning/pdb.log b/docs/multitenant/provisioning/pdb.log similarity index 100% rename from docs/onpremdb/provisioning/pdb.log rename to docs/multitenant/provisioning/pdb.log diff --git a/docs/onpremdb/provisioning/pdb.yaml b/docs/multitenant/provisioning/pdb.yaml similarity index 100% rename from docs/onpremdb/provisioning/pdb.yaml rename to docs/multitenant/provisioning/pdb.yaml diff --git a/docs/onpremdb/provisioning/pdb_crd_resource.md b/docs/multitenant/provisioning/pdb_crd_resource.md similarity index 100% rename from docs/onpremdb/provisioning/pdb_crd_resource.md rename to docs/multitenant/provisioning/pdb_crd_resource.md diff --git a/docs/onpremdb/provisioning/pdb_secret.log b/docs/multitenant/provisioning/pdb_secret.log similarity index 100% rename from docs/onpremdb/provisioning/pdb_secret.log rename to docs/multitenant/provisioning/pdb_secret.log diff --git a/docs/onpremdb/provisioning/pdb_secret.yaml b/docs/multitenant/provisioning/pdb_secret.yaml similarity index 100% rename from docs/onpremdb/provisioning/pdb_secret.yaml rename to docs/multitenant/provisioning/pdb_secret.yaml diff --git a/docs/onpremdb/provisioning/plug_pdb.log b/docs/multitenant/provisioning/plug_pdb.log similarity index 100% rename from docs/onpremdb/provisioning/plug_pdb.log rename to docs/multitenant/provisioning/plug_pdb.log diff --git a/docs/onpremdb/provisioning/plug_pdb.md b/docs/multitenant/provisioning/plug_pdb.md similarity index 100% rename from docs/onpremdb/provisioning/plug_pdb.md rename to docs/multitenant/provisioning/plug_pdb.md diff --git a/docs/onpremdb/provisioning/plug_pdb.yaml b/docs/multitenant/provisioning/plug_pdb.yaml similarity index 100% rename from docs/onpremdb/provisioning/plug_pdb.yaml rename to docs/multitenant/provisioning/plug_pdb.yaml diff --git a/docs/onpremdb/provisioning/unplug_pdb.log b/docs/multitenant/provisioning/unplug_pdb.log similarity index 100% rename from docs/onpremdb/provisioning/unplug_pdb.log rename to docs/multitenant/provisioning/unplug_pdb.log diff --git a/docs/onpremdb/provisioning/unplug_pdb.md b/docs/multitenant/provisioning/unplug_pdb.md similarity index 100% rename from docs/onpremdb/provisioning/unplug_pdb.md rename to docs/multitenant/provisioning/unplug_pdb.md diff --git a/docs/onpremdb/provisioning/unplug_pdb.yaml b/docs/multitenant/provisioning/unplug_pdb.yaml similarity index 100% rename from docs/onpremdb/provisioning/unplug_pdb.yaml rename to docs/multitenant/provisioning/unplug_pdb.yaml diff --git a/docs/onpremdb/provisioning/validation_error.md b/docs/multitenant/provisioning/validation_error.md similarity index 100% rename from docs/onpremdb/provisioning/validation_error.md rename to docs/multitenant/provisioning/validation_error.md diff --git a/docs/onpremdb/provisioning/ords_image.log b/docs/onpremdb/provisioning/ords_image.log deleted file mode 100644 index 7cce536a..00000000 --- a/docs/onpremdb/provisioning/ords_image.log +++ /dev/null @@ -1,263 +0,0 @@ --- Clone the software using git - -% git clone git@orahub.oci.oraclecorp.com:rac-docker-dev/oracle-database-operator.git -Cloning into 'oracle-database-operator'... -Use of the Oracle network and applications is intended solely for Oracle's authorized users. The use of these resources by Oracle employees and contractors is subject to company policies, including the Code of Conduct, Acceptable Use Policy and Information Protection Policy; access may be monitored and logged, to the extent permitted by law, in accordance with Oracle policies. Unauthorized use may result in termination of your access, disciplinary action and/or civil and criminal penalties. -remote: Enumerating objects: 11244, done. -remote: Counting objects: 100% (506/506), done. -remote: Compressing objects: 100% (205/205), done. -remote: Total 11244 (delta 305), reused 493 (delta 296), pack-reused 10738 -Receiving objects: 100% (11244/11244), 4.11 MiB | 397.00 KiB/s, done. -Resolving deltas: 100% (7510/7510), done. -% cd oracle-database-operator/ords/ - - --- Download the ORDS Version 21.4.3 software and copy that to the current location: - -% ls -rlt -total 79968 --rw-rw-rw- 1 root root 81861484 Jun 26 04:07 ords-21.4.3.117.0405.zip --rw-r--r-- 1 root root 266 Jun 26 04:13 standalone.properties.tmpl --rw-r--r-- 1 root root 639 Jun 26 04:13 setupwebuser.sh --rw-r--r-- 1 root root 3096 Jun 26 04:13 runOrds.sh --rw-r--r-- 1 root root 427 Jun 26 04:13 ords_params.properties.tmpl --rw-r--r-- 1 root root 2230 Jun 26 04:13 Dockerfile --rw-r--r-- 1 root root 123 Jun 26 04:13 cdbadmin.properties.tmpl - - --- Login to repository: login container-registry.oracle.com -% docker login container-registry.oracle.com -Username: -Password: -WARNING! Your password will be stored unencrypted in /root/.docker/config.json. -Configure a credential helper to remove this warning. See -https://docs.docker.com/engine/reference/commandline/login/#credentials-store - -Login Succeeded - - --- Login to a repository where you want to push the ORDS docker image: -% docker login phx.ocir.io -Username (/>): -Password: -WARNING! Your password will be stored unencrypted in /root/.docker/config.json. -Configure a credential helper to remove this warning. See -https://docs.docker.com/engine/reference/commandline/login/#credentials-store - -Login Succeeded - - --- Build the ORDS Docker Image: -[root@docker-test-server ords]# docker build -t oracle/ords-dboper:ords-21.4.3 . -Sending build context to Docker daemon 81.88MB -Step 1/10 : FROM container-registry.oracle.com/java/jdk:latest - ---> b94ea1fc0f64 -Step 2/10 : ENV ORDS_HOME=/opt/oracle/ords INSTALL_FILE=ords*.zip CONFIG_PROPS="ords_params.properties.tmpl" STANDALONE_PROPS="standalone.properties.tmpl" CDBADMIN_PROPS="cdbadmin.properties.tmpl" SETUP_WEBUSER="setupwebuser.sh" RUN_FILE="runOrds.sh" - ---> Running in baf28954867a -Removing intermediate container baf28954867a - ---> 40d169e44c21 -Step 3/10 : COPY $INSTALL_FILE $CONFIG_PROPS $STANDALONE_PROPS $CDBADMIN_PROPS $RUN_FILE $SETUP_WEBUSER $ORDS_HOME/ - ---> 1d6a27f6ce13 -Step 4/10 : RUN yum -y install bind-utils net-tools zip unzip tar wget vim-minimal which sudo expect procps && yum clean all - ---> Running in e0879fd25edc -Oracle Linux 8 BaseOS Latest (x86_64) 74 MB/s | 46 MB 00:00 -Oracle Linux 8 Application Stream (x86_64) 66 MB/s | 36 MB 00:00 -Last metadata expiration check: 0:00:15 ago on Mon 27 Jun 2022 04:34:12 PM UTC. -Package vim-minimal-2:8.0.1763-16.0.2.el8_5.13.x86_64 is already installed. -Package procps-ng-3.3.15-6.0.1.el8.x86_64 is already installed. -Dependencies resolved. -================================================================================ - Package Arch Version Repository Size -================================================================================ -Installing: - bind-utils x86_64 32:9.11.36-3.el8 ol8_appstream 452 k - expect x86_64 5.45.4-5.el8 ol8_baseos_latest 266 k - net-tools x86_64 2.0-0.52.20160912git.el8 ol8_baseos_latest 322 k - sudo x86_64 1.8.29-8.el8 ol8_baseos_latest 925 k - tar x86_64 2:1.30-5.el8 ol8_baseos_latest 838 k - unzip x86_64 6.0-46.el8 ol8_baseos_latest 196 k - wget x86_64 1.19.5-10.0.1.el8 ol8_appstream 734 k - which x86_64 2.21-17.el8 ol8_baseos_latest 49 k - zip x86_64 3.0-23.el8 ol8_baseos_latest 270 k -Installing dependencies: - bind-libs x86_64 32:9.11.36-3.el8 ol8_appstream 175 k - bind-libs-lite x86_64 32:9.11.36-3.el8 ol8_appstream 1.2 M - bind-license noarch 32:9.11.36-3.el8 ol8_appstream 103 k - fstrm x86_64 0.6.1-2.el8 ol8_appstream 29 k - libmaxminddb x86_64 1.2.0-10.el8 ol8_appstream 33 k - libmetalink x86_64 0.1.3-7.el8 ol8_baseos_latest 32 k - protobuf-c x86_64 1.3.0-6.el8 ol8_appstream 37 k - python3-bind noarch 32:9.11.36-3.el8 ol8_appstream 150 k - python3-ply noarch 3.9-9.el8 ol8_baseos_latest 111 k - tcl x86_64 1:8.6.8-2.el8 ol8_baseos_latest 1.1 M - -Transaction Summary -================================================================================ -Install 19 Packages - -Total download size: 6.9 M -Installed size: 21 M -Downloading Packages: -(1/19): libmetalink-0.1.3-7.el8.x86_64.rpm 683 kB/s | 32 kB 00:00 -(2/19): net-tools-2.0-0.52.20160912git.el8.x86_ 5.7 MB/s | 322 kB 00:00 -(3/19): expect-5.45.4-5.el8.x86_64.rpm 4.3 MB/s | 266 kB 00:00 -(4/19): python3-ply-3.9-9.el8.noarch.rpm 5.4 MB/s | 111 kB 00:00 -(5/19): sudo-1.8.29-8.el8.x86_64.rpm 23 MB/s | 925 kB 00:00 -(6/19): unzip-6.0-46.el8.x86_64.rpm 12 MB/s | 196 kB 00:00 -(7/19): tar-1.30-5.el8.x86_64.rpm 14 MB/s | 838 kB 00:00 -(8/19): which-2.21-17.el8.x86_64.rpm 3.9 MB/s | 49 kB 00:00 -(9/19): tcl-8.6.8-2.el8.x86_64.rpm 16 MB/s | 1.1 MB 00:00 -(10/19): zip-3.0-23.el8.x86_64.rpm 12 MB/s | 270 kB 00:00 -(11/19): bind-libs-9.11.36-3.el8.x86_64.rpm 7.2 MB/s | 175 kB 00:00 -(12/19): bind-license-9.11.36-3.el8.noarch.rpm 9.6 MB/s | 103 kB 00:00 -(13/19): fstrm-0.6.1-2.el8.x86_64.rpm 4.8 MB/s | 29 kB 00:00 -(14/19): libmaxminddb-1.2.0-10.el8.x86_64.rpm 4.6 MB/s | 33 kB 00:00 -(15/19): bind-utils-9.11.36-3.el8.x86_64.rpm 17 MB/s | 452 kB 00:00 -(16/19): protobuf-c-1.3.0-6.el8.x86_64.rpm 3.4 MB/s | 37 kB 00:00 -(17/19): python3-bind-9.11.36-3.el8.noarch.rpm 16 MB/s | 150 kB 00:00 -(18/19): bind-libs-lite-9.11.36-3.el8.x86_64.rp 19 MB/s | 1.2 MB 00:00 -(19/19): wget-1.19.5-10.0.1.el8.x86_64.rpm 24 MB/s | 734 kB 00:00 --------------------------------------------------------------------------------- -Total 31 MB/s | 6.9 MB 00:00 -Running transaction check -Transaction check succeeded. -Running transaction test -Transaction test succeeded. -Running transaction - Preparing : 1/1 - Installing : protobuf-c-1.3.0-6.el8.x86_64 1/19 - Installing : libmaxminddb-1.2.0-10.el8.x86_64 2/19 - Running scriptlet: libmaxminddb-1.2.0-10.el8.x86_64 2/19 - Installing : fstrm-0.6.1-2.el8.x86_64 3/19 - Installing : bind-license-32:9.11.36-3.el8.noarch 4/19 - Installing : bind-libs-lite-32:9.11.36-3.el8.x86_64 5/19 - Installing : bind-libs-32:9.11.36-3.el8.x86_64 6/19 - Installing : unzip-6.0-46.el8.x86_64 7/19 - Installing : tcl-1:8.6.8-2.el8.x86_64 8/19 - Running scriptlet: tcl-1:8.6.8-2.el8.x86_64 8/19 - Installing : python3-ply-3.9-9.el8.noarch 9/19 - Installing : python3-bind-32:9.11.36-3.el8.noarch 10/19 - Installing : libmetalink-0.1.3-7.el8.x86_64 11/19 - Installing : wget-1.19.5-10.0.1.el8.x86_64 12/19 - Running scriptlet: wget-1.19.5-10.0.1.el8.x86_64 12/19 - Installing : bind-utils-32:9.11.36-3.el8.x86_64 13/19 - Installing : expect-5.45.4-5.el8.x86_64 14/19 - Installing : zip-3.0-23.el8.x86_64 15/19 - Installing : which-2.21-17.el8.x86_64 16/19 - Installing : tar-2:1.30-5.el8.x86_64 17/19 - Running scriptlet: tar-2:1.30-5.el8.x86_64 17/19 - Installing : sudo-1.8.29-8.el8.x86_64 18/19 - Running scriptlet: sudo-1.8.29-8.el8.x86_64 18/19 - Installing : net-tools-2.0-0.52.20160912git.el8.x86_64 19/19 - Running scriptlet: net-tools-2.0-0.52.20160912git.el8.x86_64 19/19 - Verifying : expect-5.45.4-5.el8.x86_64 1/19 - Verifying : libmetalink-0.1.3-7.el8.x86_64 2/19 - Verifying : net-tools-2.0-0.52.20160912git.el8.x86_64 3/19 - Verifying : python3-ply-3.9-9.el8.noarch 4/19 - Verifying : sudo-1.8.29-8.el8.x86_64 5/19 - Verifying : tar-2:1.30-5.el8.x86_64 6/19 - Verifying : tcl-1:8.6.8-2.el8.x86_64 7/19 - Verifying : unzip-6.0-46.el8.x86_64 8/19 - Verifying : which-2.21-17.el8.x86_64 9/19 - Verifying : zip-3.0-23.el8.x86_64 10/19 - Verifying : bind-libs-32:9.11.36-3.el8.x86_64 11/19 - Verifying : bind-libs-lite-32:9.11.36-3.el8.x86_64 12/19 - Verifying : bind-license-32:9.11.36-3.el8.noarch 13/19 - Verifying : bind-utils-32:9.11.36-3.el8.x86_64 14/19 - Verifying : fstrm-0.6.1-2.el8.x86_64 15/19 - Verifying : libmaxminddb-1.2.0-10.el8.x86_64 16/19 - Verifying : protobuf-c-1.3.0-6.el8.x86_64 17/19 - Verifying : python3-bind-32:9.11.36-3.el8.noarch 18/19 - Verifying : wget-1.19.5-10.0.1.el8.x86_64 19/19 - -Installed: - bind-libs-32:9.11.36-3.el8.x86_64 - bind-libs-lite-32:9.11.36-3.el8.x86_64 - bind-license-32:9.11.36-3.el8.noarch - bind-utils-32:9.11.36-3.el8.x86_64 - expect-5.45.4-5.el8.x86_64 - fstrm-0.6.1-2.el8.x86_64 - libmaxminddb-1.2.0-10.el8.x86_64 - libmetalink-0.1.3-7.el8.x86_64 - net-tools-2.0-0.52.20160912git.el8.x86_64 - protobuf-c-1.3.0-6.el8.x86_64 - python3-bind-32:9.11.36-3.el8.noarch - python3-ply-3.9-9.el8.noarch - sudo-1.8.29-8.el8.x86_64 - tar-2:1.30-5.el8.x86_64 - tcl-1:8.6.8-2.el8.x86_64 - unzip-6.0-46.el8.x86_64 - wget-1.19.5-10.0.1.el8.x86_64 - which-2.21-17.el8.x86_64 - zip-3.0-23.el8.x86_64 - -Complete! -17 files removed -Removing intermediate container e0879fd25edc - ---> 824de67afe8c -Step 5/10 : RUN mkdir -p $ORDS_HOME/doc_root && chmod ug+x $ORDS_HOME/*.sh && groupadd -g 54322 dba && useradd -u 54321 -d /home/oracle -g dba -m -s /bin/bash oracle && cd $ORDS_HOME && jar -xf $INSTALL_FILE && rm $INSTALL_FILE && mkdir -p $ORDS_HOME/config/ords && mkdir -p $ORDS_HOME/secrets && java -jar $ORDS_HOME/ords.war configdir $ORDS_HOME/config && chown -R oracle:dba $ORDS_HOME - ---> Running in 623f0b8b31a0 -2022-06-27T16:34:43.462Z INFO Set config.dir to /opt/oracle/ords/config in: /opt/oracle/ords/ords.war -Removing intermediate container 623f0b8b31a0 - ---> b5b4693d5ecf -Step 6/10 : USER oracle - ---> Running in dd481ff2fc2d -Removing intermediate container dd481ff2fc2d - ---> e4d3603d79ec -Step 7/10 : WORKDIR /home/oracle - ---> Running in 5aba61a54f70 -Removing intermediate container 5aba61a54f70 - ---> bf0501caf083 -Step 8/10 : VOLUME ["$ORDS_HOME/config/ords"] - ---> Running in 26e4dbd9c1ed -Removing intermediate container 26e4dbd9c1ed - ---> dc4134ac42aa -Step 9/10 : EXPOSE 8888 - ---> Running in a5fbea15a1ee -Removing intermediate container a5fbea15a1ee - ---> cf615843f041 -Step 10/10 : CMD $ORDS_HOME/$RUN_FILE - ---> Running in 444eaf7568e8 -Removing intermediate container 444eaf7568e8 - ---> 7c95be92a842 -Successfully built 7c95be92a842 -Successfully tagged oracle/ords-dboper:ords-21.4.3 - - - --- Check the docker image details: - -docker images -REPOSITORY TAG IMAGE ID CREATED SIZE -oracle/ords-dboper ords-21.4.3 7c95be92a842 44 seconds ago 771M - - --- Tag and push the image to your docker repository --- NOTE: We have used the repo as "phx.ocir.io//oracle/ords" -% docker tag oracle/ords-dboper:ords-21.4.3 phx.ocir.io//oracle/ords:21.4.3 -% docker push phx.ocir.io//oracle/ords:21.4.3 -The push refers to repository [phx.ocir.io//oracle/ords] -98222b4c695a: Pushed -d8fd7c53f02f: Pushed -2023d4f36fb3: Pushed -a14158bad0d3: Layer already exists -0c1b321d1256: Layer already exists -74a1be7b5642: Layer already exists -21.4.3: digest: sha256:0f75cba13605944d2ea970c38330b65bd81509aa2cfc63d63f454769d61c1d5b size: 1589 - - --- Verify the image pushed to your docker repository. - -For example, in above example, you can check the image on the cloud console with signature "sha256:0f75cba13605944d2ea970c38330b65bd81509aa2cfc63d63f454769d61c1d5b" - - --- Create a Kubernetes Secret for the docker repository (if its private) - -% kubectl create secret generic container-registry-secret --from-file=.dockerconfigjson=./.docker/config.json --type=kubernetes.io/dockerconfigjson -n oracle-database-operator-system -secret/container-registry-secret created - -% kubectl get secret -n oracle-database-operator-system -NAME TYPE DATA AGE -container-registry-secret kubernetes.io/dockerconfigjson 1 26s -default-token-x86d6 kubernetes.io/service-account-token 3 5d16h -webhook-server-cert kubernetes.io/tls 3 5d16h diff --git a/docs/onpremdb/provisioning/ords_image.md b/docs/onpremdb/provisioning/ords_image.md deleted file mode 100644 index 4399f3fb..00000000 --- a/docs/onpremdb/provisioning/ords_image.md +++ /dev/null @@ -1,68 +0,0 @@ -# Build ORDS Docker Image - -In the below steps, we are building an ORDS Docker Image for ORDS Software. - -The image built can be later pushed to a local repository to be used later for a deployment. - -**NOTE:** It is assumed that before this step, you have followed the [prerequisite](./../README.md#prerequsites-to-manage-pdb-life-cycle-using-oracle-db-operator-on-prem-database-controller) steps. - -1. Clone the software using git: -```sh -[root@test-server oracle-database-operator]# git clone git@orahub.oci.oraclecorp.com:rac-docker-dev/oracle-database-operator.git -[root@test-server oracle-database-operator]# cd oracle-database-operator/ords/ -``` - -2. Download the ORDS Software for required ORDS version. For Example: For ORDS Version 21.4.3, use this [link](https://www.oracle.com/tools/ords/ords-downloads-2143.html) - -3. Copy the downloaded software .zip file to the current location. - -4. Login to the registry: container-registry.oracle.com - -**NOTE:** To login to this registry, you will need to the URL https://container-registry.oracle.com , Sign in, then click on "Java" and then accept the agreement. - -```sh -docker login container-registry.oracle.com -``` - -5. Login to a repo where you want to push your docker image (if needed) to pull during deployment in your environment. - -```sh -docker login -``` - -6. Build the docker image by using below command: - -```sh -docker build -t oracle/ords-dboper:ords-21.4.3 . -``` - -7. Check the docker image details using: - -```sh -docker images -``` - -8. Tag and push the image to your docker repository. - -NOTE: We have the repo as `phx.ocir.io//oracle/ords:21.4.3`. Please change as per your environment. - -```sh -docker tag oracle/ords-dboper:ords-21.4.3 phx.ocir.io//oracle/ords:21.4.3 -docker push phx.ocir.io//oracle/ords:21.4.3 -``` - -9. Verify the image pushed to your docker repository. - -You can refer to below sample output for above steps as well. - -10. Create a Kubernetes Secret for your docker repository to pull the image during deployment using the below command: - -```sh -kubectl create secret generic container-registry-secret --from-file=.dockerconfigjson=./.docker/config.json --type=kubernetes.io/dockerconfigjson -n oracle-database-operator-system -``` - -This Kubernetes secret will be provided in the .yaml file against the parameter `ordsImagePullSecret` to pull the ORDS Docker Image from your docker repository (if its a private repository). - -## Sample Output - -[Here](./ords_image.log) is the sample output for docker image created for ORDS version 21.4.3 diff --git a/oracle-database-operator.yaml b/oracle-database-operator.yaml index 5e516801..9180474e 100644 --- a/oracle-database-operator.yaml +++ b/oracle-database-operator.yaml @@ -651,6 +651,8 @@ spec: spec: description: CDBSpec defines the desired state of CDB properties: + TestVariable: + type: string cdbAdminPwd: description: Password for the CDB Administrator to manage PDB lifecycle properties: @@ -688,6 +690,38 @@ spec: cdbName: description: Name of the CDB type: string + cdbTlsCrt: + properties: + secret: + description: CDBSecret defines the secretName + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - secret + type: object + cdbTlsKey: + properties: + secret: + description: CDBSecret defines the secretName + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - secret + type: object dbPort: description: DB server port type: integer @@ -1282,7 +1316,7 @@ spec: - additionalPrinterColumns: - description: The connect string to be used jsonPath: .status.connString - name: Connect String + name: Connect_String type: string - description: Name of the CDB jsonPath: .spec.cdbName @@ -1416,6 +1450,54 @@ spec: - OPEN - CLOSE type: string + pdbTlsCat: + properties: + secret: + description: PDBSecret defines the secretName + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - secret + type: object + pdbTlsCrt: + properties: + secret: + description: PDBSecret defines the secretName + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - secret + type: object + pdbTlsKey: + properties: + secret: + description: PDBSecret defines the secretName + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - secret + type: object reuseTempFile: description: Whether to reuse temp file type: boolean diff --git a/ords/Dockerfile b/ords/Dockerfile index 1d137cab..14823269 100644 --- a/ords/Dockerfile +++ b/ords/Dockerfile @@ -1,58 +1,42 @@ -# LICENSE UPL 1.0 +#LICENSE UPL 1.0 # # Copyright (c) 1982-2017 Oracle and/or its affiliates. All rights reserved. # # ORACLE DOCKERFILES PROJECT # -------------------------- -# This is the Dockerfile for Oracle Rest Data Services +# This is the Dockerfile for Oracle Rest Data Services 22.2 # -# REQUIRED FILES TO BUILD THIS IMAGE -# ---------------------------------- -# (1) ords.3.0.10.165.06.53.zip -# Download Oracle Rest Data Services from -# http://www.oracle.com/technetwork/developer-tools/rest-data-services/downloads/index.html -# -# HOW TO BUILD THIS IMAGE -# ----------------------- -# Put the downloaded file in the same directory as this Dockerfile -# Run: -# $ docker build -t oracle/restdataservices:3.0.10 . -# -# Pull base image -# --------------- -FROM container-registry.oracle.com/java/jdk:latest +FROM container-registry.oracle.com/java/jdk:latest # Environment variables required for this build (do NOT change) # ------------------------------------------------------------- ENV ORDS_HOME=/opt/oracle/ords \ - INSTALL_FILE=ords*.zip \ - CONFIG_PROPS="ords_params.properties.tmpl" \ - STANDALONE_PROPS="standalone.properties.tmpl" \ - CDBADMIN_PROPS="cdbadmin.properties.tmpl" \ - SETUP_WEBUSER="setupwebuser.sh" \ - RUN_FILE="runOrds.sh" + RUN_FILE="runOrdsSSL.sh" + +#RUN_FILE_NOSSL="runOrdsNOSSL.sh" # Copy binaries # ------------- -COPY $INSTALL_FILE $CONFIG_PROPS $STANDALONE_PROPS $CDBADMIN_PROPS $RUN_FILE $SETUP_WEBUSER $ORDS_HOME/ +COPY $RUN_FILE $ORDS_HOME +#COPY $RUN_FILE_NOSSL $ORDS_HOME/ -RUN yum -y install bind-utils net-tools zip unzip tar wget vim-minimal which sudo expect procps && \ +RUN yum -y install yum-utils bind-utils tree hostname openssl net-tools zip unzip tar wget vim-minimal which sudo expect procps && \ + yum-config-manager --add-repo=http://yum.oracle.com/repo/OracleLinux/OL8/oracle/software/x86_64 && \ + yum -y install java-11-openjdk-devel && \ + yum -y install ords && \ + yum -y install iproute && \ yum clean all # Setup filesystem and oracle user -# Adjust file permissions, go to /opt/oracle as user 'oracle' to proceed with ORDS installation # ------------------------------------------------------------ -RUN mkdir -p $ORDS_HOME/doc_root && \ - chmod ug+x $ORDS_HOME/*.sh && \ - groupadd -g 54322 dba && \ - useradd -u 54321 -d /home/oracle -g dba -m -s /bin/bash oracle && \ - cd $ORDS_HOME && \ - jar -xf $INSTALL_FILE && \ - rm $INSTALL_FILE && \ - mkdir -p $ORDS_HOME/config/ords && \ - mkdir -p $ORDS_HOME/secrets && \ - java -jar $ORDS_HOME/ords.war configdir $ORDS_HOME/config && \ - chown -R oracle:dba $ORDS_HOME +RUN mkdir -p $ORDS_HOME/doc_root && \ + mkdir -p $ORDS_HOME/error && \ + mkdir -p $ORDS_HOME/secrets && \ + chmod ug+x $ORDS_HOME/*.sh && \ + groupadd -g 54322 dba && \ + usermod -u 54321 -d /home/oracle -g dba -m -s /bin/bash oracle && \ + chown -R oracle:dba $ORDS_HOME && \ + echo "oracle ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers # Finalize setup # ------------------- @@ -62,5 +46,8 @@ WORKDIR /home/oracle VOLUME ["$ORDS_HOME/config/ords"] EXPOSE 8888 -# Define default command to start Oracle Database. +# Define default command to start Ords Services CMD $ORDS_HOME/$RUN_FILE + +## ONLY FOR DEVELOPMENT STAGE +#CMD ["/usr/sbin/init"] diff --git a/ords/cdbadmin.properties.tmpl b/ords/cdbadmin.properties.tmpl deleted file mode 100644 index 34bc56fd..00000000 --- a/ords/cdbadmin.properties.tmpl +++ /dev/null @@ -1,3 +0,0 @@ -database.api.admin.enabled=true -db.cdb.adminUser=###CDBADMIN_USER### as SYSDBA -db.cdb.adminUser.password=###CDBADMIN_PWD### \ No newline at end of file diff --git a/ords/ords_params.properties.tmpl b/ords/ords_params.properties.tmpl deleted file mode 100644 index d2aaf33b..00000000 --- a/ords/ords_params.properties.tmpl +++ /dev/null @@ -1,16 +0,0 @@ -db.hostname=###ORACLE_HOST### -db.port=###ORACLE_PORT### -db.servicename=###ORACLE_SERVICE### -database.api.admin.enabled=true -database.api.enabled=true -feature.sdw=true -plsql.gateway.add=false -restEnabledSql.active=true -rest.services.apex.add=false -rest.services.ords.add=true -standalone.http.port=8888 -standalone.mode=false -standalone.use.https=true -sys.user=SYS -sys.password=###ORACLE_PWD### -user.public.password=###ORDS_PWD### \ No newline at end of file diff --git a/ords/runOrds.sh b/ords/runOrds.sh deleted file mode 100644 index 60bab949..00000000 --- a/ords/runOrds.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash -# Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. -# -# Since: June, 2017 -# Author: gerald.venzl@oracle.com -# Description: Setup and runs Oracle Rest Data Services. -# -# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. -# -# Copyright (c) 2014-2017 Oracle and/or its affiliates. All rights reserved. -# -# MODIFIED (DD-Mon-YY) -# gdbhat 16-Aug-21 - Added CDB Admin properties. Commented out ORACLE_PWD check - -function setupOrds() { - - # Check whether the Oracle DB password has been specified - #if [ "$ORACLE_PWD" == "" ]; then - # echo "Error: No ORACLE_PWD specified!" - # echo "Please specify Oracle DB password using the ORACLE_PWD environment variable." - # exit 1; - #fi; - - # Defaults - ORACLE_SERVICE=${ORACLE_SERVICE:-"ORCLPDB1"} - ORACLE_HOST=${ORACLE_HOST:-"localhost"} - ORACLE_PORT=${ORACLE_PORT:-"1521"} - APEXI=${APEXI:-"$ORDS_HOME/doc_root/i"} - ORDS_PORT=${ORDS_PORT:-"8888"} - - ORDS_PWD=`cat $ORDS_HOME/secrets/$ORDS_PWD_KEY` - ORACLE_PWD=`cat $ORDS_HOME/secrets/$ORACLE_PWD_KEY` - CDBADMIN_USER=`cat $ORDS_HOME/secrets/$CDBADMIN_USER_KEY` - CDBADMIN_PWD=`cat $ORDS_HOME/secrets/$CDBADMIN_PWD_KEY` - - # Make standalone dir - mkdir -p $ORDS_HOME/config/ords/standalone - - # Copy template files - cp $ORDS_HOME/$CONFIG_PROPS $ORDS_HOME/params/ords_params.properties - cp $ORDS_HOME/$STANDALONE_PROPS $ORDS_HOME/config/ords/standalone/standalone.properties - cp $ORDS_HOME/$CDBADMIN_PROPS $ORDS_HOME/cdbadmin.properties - - # Replace DB related variables (ords_params.properties) - sed -i -e "s|###ORACLE_SERVICE###|$ORACLE_SERVICE|g" $ORDS_HOME/params/ords_params.properties - sed -i -e "s|###ORACLE_HOST###|$ORACLE_HOST|g" $ORDS_HOME/params/ords_params.properties - sed -i -e "s|###ORACLE_PORT###|$ORACLE_PORT|g" $ORDS_HOME/params/ords_params.properties - sed -i -e "s|###ORDS_PWD###|$ORDS_PWD|g" $ORDS_HOME/params/ords_params.properties - sed -i -e "s|###ORACLE_PWD###|$ORACLE_PWD|g" $ORDS_HOME/params/ords_params.properties - - # Replace standalone runtime variables (standalone.properties) - sed -i -e "s|###PORT###|$ORDS_PORT|g" $ORDS_HOME/config/ords/standalone/standalone.properties - sed -i -e "s|###DOC_ROOT###|$ORDS_HOME/doc_root|g" $ORDS_HOME/config/ords/standalone/standalone.properties - sed -i -e "s|###APEXI###|$APEXI|g" $ORDS_HOME/config/ords/standalone/standalone.properties - - # Replace CDB Admin runtime variables (cdbadmin.properties) - sed -i -e "s|###CDBADMIN_USER###|$CDBADMIN_USER|g" $ORDS_HOME/cdbadmin.properties - sed -i -e "s|###CDBADMIN_PWD###|$CDBADMIN_PWD|g" $ORDS_HOME/cdbadmin.properties - - # Start ORDS setup - java -jar $ORDS_HOME/ords.war install simple 2>&1 | tee /tmp/ords_install.log - - # Setup Web Server User - $ORDS_HOME/$SETUP_WEBUSER -} - -############# MAIN ################ - -# Check whether ords is already setup -if [ ! -f $ORDS_HOME/config/ords/standalone/standalone.properties ]; then - setupOrds; - java -jar $ORDS_HOME/ords.war set-properties --conf apex_pu $ORDS_HOME/cdbadmin.properties -fi; - -java -jar $ORDS_HOME/ords.war standalone diff --git a/ords/runOrdsSSL.sh b/ords/runOrdsSSL.sh new file mode 100644 index 00000000..ac357637 --- /dev/null +++ b/ords/runOrdsSSL.sh @@ -0,0 +1,193 @@ +#!/bin/bash +# +# Since: June, 2022 +# Author: matteo.malvezzi@oracle.com +# Description: Setup and runs Oracle Rest Data Services 22.2. +# +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS HEADER. +# +# Copyright (c) 2014-2017 Oracle and/or its affiliates. All rights reserved. +# +# MODIFIED (DD-Mon-YY) +# mmalvezz 25-Jun-22 - Initial version + +export ORDS=/usr/local/bin/ords +export ETCFILE=/etc/ords.conf +export CONFIG=`cat /etc/ords.conf |grep -v ^#|grep ORDS_CONFIG| awk -F= '{print $2}'` +export export _JAVA_OPTIONS="-Xms1126M -Xmx1126M" +export ERRORFOLDER=/opt/oracle/ords/error +export KEYSTORE=~/keystore +export OPENSSL=/usr/bin/openssl +export PASSFILE=${KEYSTORE}/PASSWORD +export HN=`hostname` +#export KEY=${KEYSTORE}/${HN}-key.der +#export CERTIFICATE=${KEYSTORE}/${HN}.der +export KEY=$ORDS_HOME/secrets/$TLSKEY +export CERTIFICATE=$ORDS_HOME/secrets/$TLSCRT + +export CUSTOMURL="jdbc:oracle:thin:@(DESCRIPTION = (ADDRESS = (PROTOCOL = TCP)(HOST = racnode1)(PORT = 1521)) (CONNECT_DATA = (SERVER = DEDICATED) (SERVICE_NAME = TESTORDS)))" +echo $CUSTOMURL + +function SetParameter() { + ##ords config info <--- Use this command to get the list + + $ORDS --config ${CONFIG} config set security.requestValidationFunction false + $ORDS --config ${CONFIG} config set jdbc.MaxLimit 100 + $ORDS --config ${CONFIG} config set jdbc.InitialLimit 50 + $ORDS --config ${CONFIG} config set error.externalPath ${ERRORFOLDER} + $ORDS --config ${CONFIG} config set standalone.access.log /home/oracle + $ORDS --config ${CONFIG} config set standalone.https.port 8888 + $ORDS --config ${CONFIG} config set standalone.https.cert ${CERTIFICATE} + $ORDS --config ${CONFIG} config set standalone.https.cert.key ${KEY} + $ORDS --config ${CONFIG} config set restEnabledSql.active true + $ORDS --config ${CONFIG} config set security.verifySSL true + $ORDS --config ${CONFIG} config set database.api.enabled true + $ORDS --config ${CONFIG} config set plsql.gateway.mode false + $ORDS --config ${CONFIG} config set database.api.management.services.disabled false + $ORDS --config ${CONFIG} config set misc.pagination.maxRows 1000 + $ORDS --config ${CONFIG} config set db.cdb.adminUser "${CDBADMIN_USER:-C##DBAPI_CDB_ADMIN} AS SYSDBA" + $ORDS --config ${CONFIG} config secret --password-stdin db.cdb.adminUser.password << EOF +${CDBADMIN_PWD:-WElcome_12##} +EOF + +## $ORDS --config ${CONFIG} config set db.username "SYS AS SYSDBA" +## $ORDS --config ${CONFIG} config secret --password-stdin db.password <$PASSFILE +welcome1 +EOF + +## $JAVA_HOME/bin/keytool -genkey -keyalg RSA -alias selfsigned -keystore keystore.jks \ +## -dname "CN=${HN}, OU=Example Department, O=Example Company, L=Birmingham, ST=West Midlands, C=GB" \ +## -storepass welcome1 -validity 3600 -keysize 2048 -keypass welcome1 +## +## +## $JAVA_HOME/bin/keytool -importkeystore -srckeystore keystore.jks -srcalias selfsigned -srcstorepass welcome1 \ +## -destkeystore keystore.p12 -deststoretype PKCS12 -deststorepass welcome1 -destkeypass welcome1 +## +## +## ${OPENSSL} pkcs12 -in ${KEYSTORE}/keystore.p12 -nodes -nocerts -out ${KEYSTORE}/${HN}-key.pem -passin file:${PASSFILE} +## ${OPENSSL} pkcs12 -in ${KEYSTORE}/keystore.p12 -nokeys -out ${KEYSTORE}/${HN}.pem -passin file:${PASSFILE} +## ${OPENSSL} pkcs8 -topk8 -inform PEM -outform DER -in ${HN}-key.pem -out ${HN}-key.der -nocrypt +## ${OPENSSL} x509 -inform PEM -outform DER -in ${HN}.pem -out ${HN}.der + + + + + + + + +rm $PASSFILE +ls -ltr $KEYSTORE + + + +} + + +function setupOrds() { + +echo "====================================================" +echo CONFIG=$CONFIG + +export ORDS_LOGS=/tmp + + [ -f $ORDS_HOME/secrets/$WEBSERVER_USER_KEY ] && + { + WEBSERVER_USER=`cat $ORDS_HOME/secrets/$WEBSERVER_USER_KEY` + } + + [ -f $ORDS_HOME/secrets/$WEBSERVER_PASSWORD_KEY ] && + { + WEBSERVER_PASSWORD=`cat $ORDS_HOME/secrets/$WEBSERVER_PASSWORD_KEY` + } + + [ -f $ORDS_HOME/secrets/$CDBADMIN_USER_KEY ] && + { + CDBADMIN_USER=`cat $ORDS_HOME/secrets/$CDBADMIN_USER_KEY` + } + + [ -f $ORDS_HOME/secrets/$CDBADMIN_PWD_KEY ] && + { + CDBADMIN_PWD=`cat $ORDS_HOME/secrets/$CDBADMIN_PWD_KEY` + } + + [ -f $ORDS_HOME/secrets/$ORACLE_PWD_KEY ] && + { + SYSDBA_PASSWORD=`cat $ORDS_HOME/secrets/$ORACLE_PWD_KEY` + } + + [ -f $ORDS_HOME/secrets/$ORACLE_PWD_KEY ] && + { + ORDS_PASSWORD=`cat $ORDS_HOME/secrets/$ORDS_PWD_KEY` + } + +setupHTTPS; + +SetParameter; + +$ORDS --config ${CONFIG} install \ + --admin-user ${SYSDBA_USER:-"SYS AS SYSDBA"} \ + --db-hostname ${ORACLE_HOST:-racnode1} \ + --db-port ${ORACLE_PORT:-1521} \ + --db-servicename ${ORACLE_SERVICE:-TESTORDS} \ + --feature-db-api true \ + --feature-rest-enabled-sql true \ + --log-folder ${ORDS_LOGS} \ + --proxy-user \ + --password-stdin < Date: Fri, 16 Sep 2022 10:29:26 +0000 Subject: [PATCH 26/37] Reducing number of events, explain about tcpsCertRenewInterval in readme --- .../sidb/singleinstancedatabase_tcps.yaml | 12 ---------- .../singleinstancedatabase_controller.go | 24 +++++++++---------- docs/sidb/README.md | 22 +++++++++-------- 3 files changed, 24 insertions(+), 34 deletions(-) diff --git a/config/samples/sidb/singleinstancedatabase_tcps.yaml b/config/samples/sidb/singleinstancedatabase_tcps.yaml index 8089c85c..85ceec73 100644 --- a/config/samples/sidb/singleinstancedatabase_tcps.yaml +++ b/config/samples/sidb/singleinstancedatabase_tcps.yaml @@ -3,18 +3,6 @@ # Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl. # -apiVersion: v1 -kind: Secret -metadata: - name: db-admin-secret - namespace: default -type: Opaque -stringData: - # Specify your DB password here - oracle_pwd: - ---- - apiVersion: database.oracle.com/v1alpha1 kind: SingleInstanceDatabase metadata: diff --git a/controllers/database/singleinstancedatabase_controller.go b/controllers/database/singleinstancedatabase_controller.go index 7e432188..77d40f90 100644 --- a/controllers/database/singleinstancedatabase_controller.go +++ b/controllers/database/singleinstancedatabase_controller.go @@ -674,7 +674,7 @@ func (r *SingleInstanceDatabaseReconciler) instantiatePodSpec(m *dbapi.SingleIns if m.Spec.ReadinessCheckPeriod > 0 { return int32(m.Spec.ReadinessCheckPeriod) } - return 30 + return 60 }(), }, @@ -1796,15 +1796,13 @@ func (r *SingleInstanceDatabaseReconciler) validateDBReadiness(m *dbapi.SingleIn return requeueY, readyPod, err } if readyPod.Name == "" { - eventReason := "Database Pending" - eventMsg := "status of database is not ready, retrying..." m.Status.Status = dbcommons.StatusPending if ok, _ := dbcommons.IsAnyPodWithStatus(available, corev1.PodFailed); ok { - eventReason = "Database Failed" - eventMsg = "pod creation failed" + eventReason := "Database Failed" + eventMsg := "pod creation failed" + r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) } else if ok, runningPod := dbcommons.IsAnyPodWithStatus(available, corev1.PodRunning); ok { - eventReason = "Database Creating" - eventMsg = "database creation in progress..." + r.Log.Info("Database Creating...", "Name", m.Name) m.Status.Status = dbcommons.StatusCreating if m.Spec.CloneFrom != "" { // Required since clone creates the datafiles under primary database SID folder @@ -1823,18 +1821,20 @@ func (r *SingleInstanceDatabaseReconciler) validateDBReadiness(m *dbapi.SingleIn r.Log.Info("GetCheckpointFileCMD Output : \n" + out) if out != "" { - eventReason = "Database Unhealthy" - eventMsg = "datafiles exists" + eventReason := "Database Unhealthy" + eventMsg := "datafiles exists" + r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) m.Status.DatafilesCreated = "true" m.Status.Status = dbcommons.StatusNotReady r.updateORDSStatus(m, ctx, req) } + } else { + r.Log.Info("Database Pending...", "Name", m.Name) } - r.Recorder.Eventf(m, corev1.EventTypeNormal, eventReason, eventMsg) - r.Log.Info(eventMsg) + // As No pod is ready now , turn on mode when pod is ready . so requeue the request - return requeueY, readyPod, errors.New(eventMsg) + return requeueY, readyPod, errors.New("no pod is ready currently") } if m.Status.DatafilesPatched != "true" { eventReason := "Datapatch Pending" diff --git a/docs/sidb/README.md b/docs/sidb/README.md index 3a7ba085..1535d14c 100644 --- a/docs/sidb/README.md +++ b/docs/sidb/README.md @@ -535,15 +535,9 @@ true ``` The following steps are required to connect the Database using TCPS: -- You need to download the wallet from the Persistent Volume(PV) attached with the database pod. You can use the following command to get the list of pod: +- You need to download the wallet from the Persistent Volume (PV) attached with the database pod. The location of the wallet inside the pod is as `/opt/oracle/oradata/clientWallet/$ORACLE_SID`. **Let us assume the `ORACLE_SID` is `ORCL1`, and singleinstance database resource name is `sidb-sample` for the upcoming example command**. You can copy the wallet to the destination directory by the following command: ```bash - kubectl get po - NAME READY STATUS RESTARTS AGE - sidb-sample-gaqoe 1/1 Running 0 3d14h - ``` -- The location of the wallet inside the pod is as `/opt/oracle/oradata/clientWallet/$ORACLE_SID`. **Let us assume the `ORACLE_SID` is `ORCL1` for the upcoming example commands**. The sample command to download the wallet is as follows: - ```bash - kubectl cp sidb-sample-gaqoe:/opt/oracle/oradata/clientWallet/ORCL1 + kubectl cp $(kubectl get pods -l app=sidb-sample -o=jsonpath='{.items[0].metadata.name}'):/opt/oracle/oradata/clientWallet/ORCL1 ``` - This wallet includes the sample `tnsnames.ora` and `sqlnet.ora` files. All the TNS entries for the database (corresponding to the CDB and PDB) resides in `tnsnames.ora` file. You need to go inside the downloaded wallet directory and set the `TNS_ADMIN` environment variable to point to the current directory as follows: ```bash @@ -553,13 +547,20 @@ The following steps are required to connect the Database using TCPS: After this, you can connect using SQL\*Plus using the following sample commands: ```bash sqlplus sys@ORCL1 as sysdba - - sqlplus system@ORCL1 + ``` +- Alternatively, you can use the following SQL\*Plus command to connect using TCPS without setting TNS_ADMIN environment variable: + ```bash + sqlplus sys@tcps://?wallet_location= + ``` + Here, TCPS connect string can be found by using the following command: + ```bash + kubectl get singleinstancedatabase sidb-sample -o "jsonpath={.status.TcpsConnectString}" ``` **NOTE:** - Only database server authentication is supported (no mTLS). - When TCPS is enabled, a self-signed certificate is generated and stored inside the wallets. For users' convenience, a client-side wallet is generated and stored at `/opt/oracle/oradata/clientWallet/$ORACLE_SID` location in the pod. - The self-signed certificate used with TCPS has validity for 2 years. After the certificate is expired, it will be renewed by the `OraOperator` automatically. You need to download the wallet again after the auto-renewal. +- You can set the certificate renew interval with the help of `tcpsCertRenewInterval` field in the **[config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml)** file. The minimum accepted value is 1m, and the maximum value is 26280h (3 years). The certificates used with TCPS will automatically be renewed after this interval. If this field is omitted/commented in the yaml file, the certificates will not be renewed automatically. ### Specifying Custom Ports As mentioned in the section [Setup Database with LoadBalancer](#setup-database-with-loadbalancer), there are two kubernetes services possible for the database: NodePort and LoadBalancer. You can specify which port to use with these services by editing the `listenerPort` and `tcpsListenerPort` fields of the [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file. @@ -574,6 +575,7 @@ In case of `NodePort` service, `listenerPort`, and `tcpsListenerPort` will be th - `listenerPort` and `tcpsListenerPort` can not have same values. - `tcpsListenerPort` will come into effect only when TCPS connections are enabled (i.e. `enableTCPS` field is set in [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file). - If TCPS connections are enabled, and `listenerPort` is commented/removed in the [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file, only TCPS endpoint will be exposed. +- If LoadBalancer is enabled, and either `listenerPort` or `tcpsListenerPort` is changed, then it takes some time to complete the work requests (drain existing backend sets and create new ones). SingleInstanceDatabase and LoadBalancer remains in the healthy state, but, you can check the progress of the work requests by logging into the OCI console and checking the corresponding LoadBalancer. ## OracleRestDataService Resource From 4d45c025a4af01de6721815856ce6d3ee0e2ad89 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Fri, 16 Sep 2022 11:28:39 +0000 Subject: [PATCH 27/37] Set min tcpsCertRenewInterval to 5m and some readme changes --- .../singleinstancedatabase_webhook.go | 4 +- .../crd/bases/database.oracle.com_cdbs.yaml | 34 +++++++++++++ .../crd/bases/database.oracle.com_pdbs.yaml | 50 ++++++++++++++++++- .../samples/sidb/singleinstancedatabase.yaml | 2 +- .../sidb/singleinstancedatabase_tcps.yaml | 2 +- docs/sidb/README.md | 4 +- 6 files changed, 89 insertions(+), 7 deletions(-) diff --git a/apis/database/v1alpha1/singleinstancedatabase_webhook.go b/apis/database/v1alpha1/singleinstancedatabase_webhook.go index e2671082..cc72cbfc 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_webhook.go +++ b/apis/database/v1alpha1/singleinstancedatabase_webhook.go @@ -271,11 +271,11 @@ func (r *SingleInstanceDatabase) ValidateCreate() error { "Please provide valid string to parse the tcpsCertRenewInterval.")) } maxLimit, _ := time.ParseDuration("26280h") - minLimit, _ := time.ParseDuration("1m") + minLimit, _ := time.ParseDuration("5m") if duration > maxLimit || duration < minLimit { allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("tcpsCertRenewInterval"), r.Spec.TcpsCertRenewInterval, - "Please specify tcpsCertRenewInterval in the range: 1m to 26280h")) + "Please specify tcpsCertRenewInterval in the range: 5m to 26280h")) } } if len(allErrs) == 0 { diff --git a/config/crd/bases/database.oracle.com_cdbs.yaml b/config/crd/bases/database.oracle.com_cdbs.yaml index e50888d0..bf64810b 100644 --- a/config/crd/bases/database.oracle.com_cdbs.yaml +++ b/config/crd/bases/database.oracle.com_cdbs.yaml @@ -65,6 +65,8 @@ spec: spec: description: CDBSpec defines the desired state of CDB properties: + TestVariable: + type: string cdbAdminPwd: description: Password for the CDB Administrator to manage PDB lifecycle properties: @@ -103,6 +105,38 @@ spec: cdbName: description: Name of the CDB type: string + cdbTlsCrt: + properties: + secret: + description: CDBSecret defines the secretName + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - secret + type: object + cdbTlsKey: + properties: + secret: + description: CDBSecret defines the secretName + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - secret + type: object dbPort: description: DB server port type: integer diff --git a/config/crd/bases/database.oracle.com_pdbs.yaml b/config/crd/bases/database.oracle.com_pdbs.yaml index 113b2b4a..1bc8d905 100644 --- a/config/crd/bases/database.oracle.com_pdbs.yaml +++ b/config/crd/bases/database.oracle.com_pdbs.yaml @@ -19,7 +19,7 @@ spec: - additionalPrinterColumns: - description: The connect string to be used jsonPath: .status.connString - name: Connect String + name: Connect_String type: string - description: Name of the CDB jsonPath: .spec.cdbName @@ -166,6 +166,54 @@ spec: - OPEN - CLOSE type: string + pdbTlsCat: + properties: + secret: + description: PDBSecret defines the secretName + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - secret + type: object + pdbTlsCrt: + properties: + secret: + description: PDBSecret defines the secretName + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - secret + type: object + pdbTlsKey: + properties: + secret: + description: PDBSecret defines the secretName + properties: + key: + type: string + secretName: + type: string + required: + - key + - secretName + type: object + required: + - secret + type: object reuseTempFile: description: Whether to reuse temp file type: boolean diff --git a/config/samples/sidb/singleinstancedatabase.yaml b/config/samples/sidb/singleinstancedatabase.yaml index dbfbe8e9..80123b69 100644 --- a/config/samples/sidb/singleinstancedatabase.yaml +++ b/config/samples/sidb/singleinstancedatabase.yaml @@ -49,7 +49,7 @@ spec: ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled. ## tcpsCertRenewInterval can be in hours(h), minutes(m) and seconds(s); e.g. 17520h, 8760h etc. - ## Maximum value is 26280h (3 years), Minimum value is 1m; Default value is 17520h (2 years) + ## Maximum value is 26280h (3 years), Minimum value is 5m; Default value is 17520h (2 years) ## If this field is commented out/removed from the yaml, it will disable the auto-renewal feature for TCPS certificate tcpsCertRenewInterval: 17520h diff --git a/config/samples/sidb/singleinstancedatabase_tcps.yaml b/config/samples/sidb/singleinstancedatabase_tcps.yaml index 85ceec73..08ba9ee9 100644 --- a/config/samples/sidb/singleinstancedatabase_tcps.yaml +++ b/config/samples/sidb/singleinstancedatabase_tcps.yaml @@ -36,7 +36,7 @@ spec: ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled. ## tcpsCertRenewInterval can be in hours(h), minutes(m) and seconds(s); e.g. 17520h, 8760h etc. - ## Maximum value is 26280h (3 years), Minimum value is 1m; Default value is 17520h (2 years) + ## Maximum value is 26280h (3 years), Minimum value is 5m; Default value is 17520h (2 years) ## If this field is commented out/removed from the yaml, it will disable the auto-renewal feature for TCPS certificate tcpsCertRenewInterval: 17520h diff --git a/docs/sidb/README.md b/docs/sidb/README.md index 1535d14c..043aec31 100644 --- a/docs/sidb/README.md +++ b/docs/sidb/README.md @@ -560,7 +560,7 @@ The following steps are required to connect the Database using TCPS: - Only database server authentication is supported (no mTLS). - When TCPS is enabled, a self-signed certificate is generated and stored inside the wallets. For users' convenience, a client-side wallet is generated and stored at `/opt/oracle/oradata/clientWallet/$ORACLE_SID` location in the pod. - The self-signed certificate used with TCPS has validity for 2 years. After the certificate is expired, it will be renewed by the `OraOperator` automatically. You need to download the wallet again after the auto-renewal. -- You can set the certificate renew interval with the help of `tcpsCertRenewInterval` field in the **[config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml)** file. The minimum accepted value is 1m, and the maximum value is 26280h (3 years). The certificates used with TCPS will automatically be renewed after this interval. If this field is omitted/commented in the yaml file, the certificates will not be renewed automatically. +- You can set the certificate renew interval with the help of `tcpsCertRenewInterval` field in the **[config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml)** file. The minimum accepted value is 5m, and the maximum value is 26280h (3 years). The certificates used with TCPS will automatically be renewed after this interval. If this field is omitted/commented in the yaml file, the certificates will not be renewed automatically. ### Specifying Custom Ports As mentioned in the section [Setup Database with LoadBalancer](#setup-database-with-loadbalancer), there are two kubernetes services possible for the database: NodePort and LoadBalancer. You can specify which port to use with these services by editing the `listenerPort` and `tcpsListenerPort` fields of the [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file. @@ -575,7 +575,7 @@ In case of `NodePort` service, `listenerPort`, and `tcpsListenerPort` will be th - `listenerPort` and `tcpsListenerPort` can not have same values. - `tcpsListenerPort` will come into effect only when TCPS connections are enabled (i.e. `enableTCPS` field is set in [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file). - If TCPS connections are enabled, and `listenerPort` is commented/removed in the [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file, only TCPS endpoint will be exposed. -- If LoadBalancer is enabled, and either `listenerPort` or `tcpsListenerPort` is changed, then it takes some time to complete the work requests (drain existing backend sets and create new ones). SingleInstanceDatabase and LoadBalancer remains in the healthy state, but, you can check the progress of the work requests by logging into the OCI console and checking the corresponding LoadBalancer. +- If LoadBalancer is enabled, and either `listenerPort` or `tcpsListenerPort` is changed, then it takes some time to complete the work requests (drain existing backend sets and create new ones). In this time, the database connectivity is broken. Although, SingleInstanceDatabase and LoadBalancer remain in the healthy state, you can check the progress of the work requests by logging into the cloud provider's console and checking the corresponding LoadBalancer. ## OracleRestDataService Resource From 2f70630d855f760cb8baf174f4c059b5d31fb712 Mon Sep 17 00:00:00 2001 From: matteo malvezzi Date: Mon, 19 Sep 2022 09:15:52 +0200 Subject: [PATCH 28/37] Correnct Dockerfile typo --- ords/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ords/Dockerfile b/ords/Dockerfile index 14823269..d4e16b6c 100644 --- a/ords/Dockerfile +++ b/ords/Dockerfile @@ -10,7 +10,7 @@ FROM container-registry.oracle.com/java/jdk:latest # Environment variables required for this build (do NOT change) # ------------------------------------------------------------- -ENV ORDS_HOME=/opt/oracle/ords \ +ENV ORDS_HOME=/opt/oracle/ords/ \ RUN_FILE="runOrdsSSL.sh" #RUN_FILE_NOSSL="runOrdsNOSSL.sh" @@ -18,7 +18,7 @@ ENV ORDS_HOME=/opt/oracle/ords \ # Copy binaries # ------------- COPY $RUN_FILE $ORDS_HOME -#COPY $RUN_FILE_NOSSL $ORDS_HOME/ +#COPY $RUN_FILE_NOSSL $ORDS_HOME RUN yum -y install yum-utils bind-utils tree hostname openssl net-tools zip unzip tar wget vim-minimal which sudo expect procps && \ yum-config-manager --add-repo=http://yum.oracle.com/repo/OracleLinux/OL8/oracle/software/x86_64 && \ From c5a4da5fb9e554425e10e16c2af43b74d77b87e8 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Wed, 21 Sep 2022 12:23:13 +0000 Subject: [PATCH 29/37] TCPS connections bugfixes --- .../singleinstancedatabase_webhook.go | 8 +++---- commons/database/constants.go | 6 ++--- .../samples/sidb/singleinstancedatabase.yaml | 2 +- .../sidb/singleinstancedatabase_tcps.yaml | 2 +- docs/sidb/README.md | 24 +++++-------------- 5 files changed, 15 insertions(+), 27 deletions(-) diff --git a/apis/database/v1alpha1/singleinstancedatabase_webhook.go b/apis/database/v1alpha1/singleinstancedatabase_webhook.go index cc72cbfc..1006c68e 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_webhook.go +++ b/apis/database/v1alpha1/singleinstancedatabase_webhook.go @@ -250,10 +250,10 @@ func (r *SingleInstanceDatabase) ValidateCreate() error { } } else { // LoadBalancer Service is expected. - if r.Spec.EnableTCPS && r.Spec.TcpsListenerPort == 0 && r.Spec.ListenerPort == 1522 { + if r.Spec.EnableTCPS && r.Spec.TcpsListenerPort == 0 && r.Spec.ListenerPort == int(dbcommons.CONTAINER_TCPS_PORT) { allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("listenerPort"), r.Spec.ListenerPort, - "listenerPort can not be 1522 as the default port for tcpsListenerPort is 1522.")) + "listenerPort can not be 2484 as the default port for tcpsListenerPort is 2484.")) } } if r.Spec.EnableTCPS && r.Spec.ListenerPort != 0 && r.Spec.TcpsListenerPort != 0 && r.Spec.ListenerPort == r.Spec.TcpsListenerPort { @@ -271,11 +271,11 @@ func (r *SingleInstanceDatabase) ValidateCreate() error { "Please provide valid string to parse the tcpsCertRenewInterval.")) } maxLimit, _ := time.ParseDuration("26280h") - minLimit, _ := time.ParseDuration("5m") + minLimit, _ := time.ParseDuration("24h") if duration > maxLimit || duration < minLimit { allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("tcpsCertRenewInterval"), r.Spec.TcpsCertRenewInterval, - "Please specify tcpsCertRenewInterval in the range: 5m to 26280h")) + "Please specify tcpsCertRenewInterval in the range: 24h to 26280h")) } } if len(allErrs) == 0 { diff --git a/commons/database/constants.go b/commons/database/constants.go index 9df367b0..a1d7bfc8 100644 --- a/commons/database/constants.go +++ b/commons/database/constants.go @@ -40,7 +40,7 @@ package commons const CONTAINER_LISTENER_PORT int32 = 1521 -const CONTAINER_TCPS_PORT int32 = 1522 +const CONTAINER_TCPS_PORT int32 = 2484 const ORACLE_UID int64 = 54321 @@ -513,7 +513,7 @@ const LsnrPort string = "\"name\": \"listener\", \"protocol\": \"TCP\", \"port\" const LsnrNodePort string = "\"name\": \"listener\", \"protocol\": \"TCP\", \"port\": 1521, \"nodePort\": %d" // Payload section for TCPS port -const TcpsPort string = "\"name\": \"listener-tcps\", \"protocol\": \"TCP\", \"port\": %d, \"targetPort\": 1522" +const TcpsPort string = "\"name\": \"listener-tcps\", \"protocol\": \"TCP\", \"port\": %d, \"targetPort\": 2484" // Payload section for TCPS node port -const TcpsNodePort string = "\"name\": \"listener-tcps\", \"protocol\": \"TCP\", \"port\": 1522, \"nodePort\": %d" +const TcpsNodePort string = "\"name\": \"listener-tcps\", \"protocol\": \"TCP\", \"port\": 2484, \"nodePort\": %d" diff --git a/config/samples/sidb/singleinstancedatabase.yaml b/config/samples/sidb/singleinstancedatabase.yaml index 80123b69..0fafd46e 100644 --- a/config/samples/sidb/singleinstancedatabase.yaml +++ b/config/samples/sidb/singleinstancedatabase.yaml @@ -49,7 +49,7 @@ spec: ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled. ## tcpsCertRenewInterval can be in hours(h), minutes(m) and seconds(s); e.g. 17520h, 8760h etc. - ## Maximum value is 26280h (3 years), Minimum value is 5m; Default value is 17520h (2 years) + ## Maximum value is 26280h (3 years), Minimum value is 24h; Default value is 17520h (2 years) ## If this field is commented out/removed from the yaml, it will disable the auto-renewal feature for TCPS certificate tcpsCertRenewInterval: 17520h diff --git a/config/samples/sidb/singleinstancedatabase_tcps.yaml b/config/samples/sidb/singleinstancedatabase_tcps.yaml index 08ba9ee9..b4a64ba8 100644 --- a/config/samples/sidb/singleinstancedatabase_tcps.yaml +++ b/config/samples/sidb/singleinstancedatabase_tcps.yaml @@ -36,7 +36,7 @@ spec: ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled. ## tcpsCertRenewInterval can be in hours(h), minutes(m) and seconds(s); e.g. 17520h, 8760h etc. - ## Maximum value is 26280h (3 years), Minimum value is 5m; Default value is 17520h (2 years) + ## Maximum value is 26280h (3 years), Minimum value is 24h; Default value is 17520h (2 years) ## If this field is commented out/removed from the yaml, it will disable the auto-renewal feature for TCPS certificate tcpsCertRenewInterval: 17520h diff --git a/docs/sidb/README.md b/docs/sidb/README.md index 043aec31..1e9f02c3 100644 --- a/docs/sidb/README.md +++ b/docs/sidb/README.md @@ -520,14 +520,6 @@ Alternatively, you can use the following command: ```bash kubectl patch --type=merge singleinstancedatabases.database.oracle.com sidb-sample -p '{"spec": {"enableTCPS": true}}' ``` - -When TCPS connections are enabled, a Kubernetes event is published notifying the same. This event can be seen by any one of the following commands: -```bash -kubectl describe singleinstancedatabases.database.oracle.com sidb-sample - -kubectl get events -``` - Once TCPS connections are enabled, the database connect string will change accordingly. The TCPS connections status can also be queried by the following command: ```bash kubectl get singleinstancedatabase sidb-sample -o "jsonpath={.status.isTcpsEnabled}" @@ -548,26 +540,22 @@ The following steps are required to connect the Database using TCPS: ```bash sqlplus sys@ORCL1 as sysdba ``` -- Alternatively, you can use the following SQL\*Plus command to connect using TCPS without setting TNS_ADMIN environment variable: - ```bash - sqlplus sys@tcps://?wallet_location= - ``` - Here, TCPS connect string can be found by using the following command: - ```bash - kubectl get singleinstancedatabase sidb-sample -o "jsonpath={.status.TcpsConnectString}" - ``` **NOTE:** - Only database server authentication is supported (no mTLS). - When TCPS is enabled, a self-signed certificate is generated and stored inside the wallets. For users' convenience, a client-side wallet is generated and stored at `/opt/oracle/oradata/clientWallet/$ORACLE_SID` location in the pod. - The self-signed certificate used with TCPS has validity for 2 years. After the certificate is expired, it will be renewed by the `OraOperator` automatically. You need to download the wallet again after the auto-renewal. -- You can set the certificate renew interval with the help of `tcpsCertRenewInterval` field in the **[config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml)** file. The minimum accepted value is 5m, and the maximum value is 26280h (3 years). The certificates used with TCPS will automatically be renewed after this interval. If this field is omitted/commented in the yaml file, the certificates will not be renewed automatically. +- You can set the certificate renew interval with the help of `tcpsCertRenewInterval` field in the **[config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml)** file. The minimum accepted value is 24h, and the maximum value is 26280h (3 years). The certificates used with TCPS will automatically be renewed after this interval. If this field is omitted/commented in the yaml file, the certificates will not be renewed automatically. +- When the certificate gets created/renewed, the `.status.certCreationTimestamp` status variable gets updated accordingly. You can see this timestamp by using the following command: + ```bash + kubectl get singleinstancedatabase sidb-sample -o "jsonpath={.status.certCreationTimestamp}" + ``` ### Specifying Custom Ports As mentioned in the section [Setup Database with LoadBalancer](#setup-database-with-loadbalancer), there are two kubernetes services possible for the database: NodePort and LoadBalancer. You can specify which port to use with these services by editing the `listenerPort` and `tcpsListenerPort` fields of the [config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml) file. `listenerPort` is intended for normal database connections. Similarly, `tcpsListenerPort` is intended for TCPS database connections. -If the `LoadBalancer` is enabled, the `listenerPort`, and `tcpsListenerPort` will be the opened ports on the Load Balancer for normal and TCPS database connections respectively. +If the `LoadBalancer` is enabled, the `listenerPort`, and `tcpsListenerPort` will be the opened ports on the Load Balancer for normal and TCPS database connections respectively. The default values of `listenerPort` and `tcpsListenerPort` are 1521 and 2484 respectively when the `LoadBalancer` is enabled. In case of `NodePort` service, `listenerPort`, and `tcpsListenerPort` will be the opened ports on the Kubernetes nodes for for normal and TCPS database connections respectively. In this case, the allowed range for the `listenerPort`, and `tcpsListenerPort` is 30000-32767. From fe853c3c2f2ab07553b2f7b33d6054376949c507 Mon Sep 17 00:00:00 2001 From: matteo malvezzi Date: Mon, 26 Sep 2022 16:34:30 +0200 Subject: [PATCH 30/37] fix 34510825 + check of cert/key/ca input param --- apis/database/v1alpha1/cdb_types.go | 1 - apis/database/v1alpha1/cdb_webhook.go | 15 +- apis/database/v1alpha1/pdb_webhook.go | 15 + controllers/database/cdb_controller.go | 6 +- cscope.out | 2060 ++++++++++++++++++++++++ cscope.tmplst | 1 + 6 files changed, 2087 insertions(+), 11 deletions(-) create mode 100644 cscope.out create mode 100644 cscope.tmplst diff --git a/apis/database/v1alpha1/cdb_types.go b/apis/database/v1alpha1/cdb_types.go index 0687cb7f..9762c1cc 100644 --- a/apis/database/v1alpha1/cdb_types.go +++ b/apis/database/v1alpha1/cdb_types.go @@ -52,7 +52,6 @@ type CDBSpec struct { // Name of the CDB Service ServiceName string `json:"serviceName,omitempty"` - TestVariable string `json:"TestVariable,omitempty"` // Password for the CDB System Administrator SysAdminPwd CDBSysAdminPassword `json:"sysAdminPwd,omitempty"` diff --git a/apis/database/v1alpha1/cdb_webhook.go b/apis/database/v1alpha1/cdb_webhook.go index 5ca7f9b8..e38d037b 100644 --- a/apis/database/v1alpha1/cdb_webhook.go +++ b/apis/database/v1alpha1/cdb_webhook.go @@ -92,11 +92,16 @@ func (r *CDB) ValidateCreate() error { allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("serviceName"), "Please specify CDB Service name")) } - - if r.Spec.TestVariable == "" { - allErrs = append(allErrs, - field.Required(field.NewPath("spec").Child("TestVariable"), "Please specify CDB testvarable")) - } + + if reflect.ValueOf(r.Spec.CDBTlsKey).IsZero() { + allErrs = append(allErrs, + field.Required(field.NewPath("spec").Child("cdbTlsKey"), "Please specify CDB Tls key(secret)")) + } + + if reflect.ValueOf(r.Spec.CDBTlsCrt).IsZero() { + allErrs = append(allErrs, + field.Required(field.NewPath("spec").Child("cdbTlsCrt"), "Please specify CDB Tls Certificate(secret)")) + } if r.Spec.SCANName == "" { allErrs = append(allErrs, diff --git a/apis/database/v1alpha1/pdb_webhook.go b/apis/database/v1alpha1/pdb_webhook.go index 1fb2f4cf..0e78790a 100644 --- a/apis/database/v1alpha1/pdb_webhook.go +++ b/apis/database/v1alpha1/pdb_webhook.go @@ -147,6 +147,21 @@ func (r *PDB) validateAction(allErrs *field.ErrorList) { pdblog.Info("Valdiating PDB Resource Action : " + action) + if reflect.ValueOf(r.Spec.PDBTlsKey).IsZero() { + *allErrs = append(*allErrs, + field.Required(field.NewPath("spec").Child("pdbTlsKey"), "Please specify PDB Tls Key(secret)")) + } + + if reflect.ValueOf(r.Spec.PDBTlsCrt).IsZero() { + *allErrs = append(*allErrs, + field.Required(field.NewPath("spec").Child("pdbTlsCrt"), "Please specify PDB Tls Certificate(secret)")) + } + + if reflect.ValueOf(r.Spec.PDBTlsCat).IsZero() { + *allErrs = append(*allErrs, + field.Required(field.NewPath("spec").Child("pdbTlsCat"), "Please specify PDB Tls Certificate Authority(secret)")) + } + switch action { case "CREATE": if reflect.ValueOf(r.Spec.AdminName).IsZero() { diff --git a/controllers/database/cdb_controller.go b/controllers/database/cdb_controller.go index 5c9bc3ee..02c04142 100644 --- a/controllers/database/cdb_controller.go +++ b/controllers/database/cdb_controller.go @@ -286,7 +286,7 @@ func (r *CDBReconciler) validateORDSPods(ctx context.Context, req ctrl.Request, if strings.Contains(out, "HTTP/1.1 200 OK") || strings.Contains(strings.ToUpper(err.Error()), "HTTP/1.1 200 OK") || strings.Contains(out, "HTTP/2") || strings.Contains(strings.ToUpper(err.Error()), " HTTP/2") { readyPods++ - } else if strings.Contains(out, "HTTP/1.1 404 Not Found") || strings.Contains(strings.ToUpper(err.Error()), "HTTP/1.1 404 NOT FOUND") { + } else if strings.Contains(out, "HTTP/1.1 404 Not Found") || strings.Contains(strings.ToUpper(err.Error()), "HTTP/1.1 404 NOT FOUND") || strings.Contains(strings.ToUpper(err.Error()), "HTTP/2 404") || strings.Contains(strings.ToUpper(err.Error()), "Failed to connect to localhost") { // Check if DB connection parameters are correct getORDSInstallStatus := " grep -q 'Failed to' /tmp/ords_install.log; echo $?;" out, _ := dbcommons.ExecCommand(r, r.Config, pod.Name, pod.Namespace, "", ctx, req, false, "bash", "-c", getORDSInstallStatus) @@ -449,10 +449,6 @@ func (r *CDBReconciler) createPodSpec(cdb *dbapi.CDB) corev1.PodSpec { Name: "ORACLE_HOST", Value: cdb.Spec.DBServer, }, - { - Name: "TESTVAR", - Value: cdb.Spec.TestVariable, - }, { Name: "TLSCRT", Value: cdb.Spec.CDBTlsCrt.Secret.Key, diff --git a/cscope.out b/cscope.out new file mode 100644 index 00000000..fbb75c7b --- /dev/null +++ b/cscope.out @@ -0,0 +1,2060 @@ +cscope 15 $HOME/WORKDIR/Ords22_operator/oracle-database-operator 0000008991 + @Makefile + +1 # +#CÝyrighˆ( +c +è2022, +O¿þe + +ªd +/ +Ü + +™s + +affžŸ‹s +. + +2 #Liûn£d +und” + +the + +Univ”§l + +P”missive + +Liûn£ + +v + 1.0 +as + +shown + +© + +h‰p +: + +4 #Cu¼’ˆ +O³¿tÜ + +v”siÚ + + +5 + gVERSION + ?= 0.0.1 + +6 #DeçuÉ +bundË + +image + +g + + +7 +BUNDLE_IMG + ?ð +cÚŒÞËr +- +bundË +: + $$ +( +VERSION +) + +9 + `iâeq + ( + `$ +( +Üigš + +CHANNELS +), +undefšed +) + +10 +BUNDLE_CHANNELS + :ð-- +chªÃls += + $$ +( +CHANNELS +) + +11 +’dif + + +12 + `iâeq + ( + `$ +( +Üigš + +DEFAULT_CHANNEL +), +undefšed +) + +13 +BUNDLE_DEFAULT_CHANNEL + :ð--- +chªÃl += + $$ +( +DEFAULT_CHANNEL +) + +14 +’dif + + +15 +BUNDLE_METADATA_OPTS + ?ð + `$ +( +BUNDLE_CHANNELS +è + $$ +( +BUNDLE_DEFAULT_CHANNEL +) + +17 #Imag +URL + +to + +u£ + +®l + +buždšg +/ +pushšg + +image + +rg‘s + + +18 +IMG + ?ð +cÚŒÞËr +: +Ï‹¡ + + +19 #Produû +CRDs + +th© + +wÜk + +back + +to + +Kub”Ës + 1.11 ( +no + +v”siÚ + +cÚv”siÚ +) + +20 #API +v”siÚ + +has + +to + +be + +v1 +Ø +u£ + + `deçuÉšg + ( +h‰ps +: + +21 +CRD_OPTIONS + ?= "crd:trivialVersions=true,preserveUnknownFields=false" + +22 #ENVTEST_K8S_VERSION +»ãrs + +to + +the + +v”siÚ + +of + +kubebužd” + +as£ts +Ø +be + +dowÆßded + +by + +’v‹¡ + +bš¬y +. + +23 +ENVTEST_K8S_VERSION + = 1.21 + +24 #O³¿tÜ +YAML + +fže + + +25 +OPERATOR_YAML += + `$$ +( +ba£Çme + $$( +pwd +)). +yaml + + +27 #G‘ +the + +cu¼’Žy + +u£d + +gÞªg + +š¡®l + + `·th + ( +š + +GOPATH +/ +bš +, +uÆess + +GOBIN + +is + +£t +) + +28 + `iãq + (, + $$ +( +sh–l + +go + +’v + +GOBIN +)) + +29 +GOBIN += + `$ +( +sh–l + +go + +’v + +GOPATH +)/ +bš + + +31 +GOBIN += + $$ +( +sh–l + +go + +’v + +GOBIN +) + +32 +’dif + + +34 #S‘tšg +SHELL + +to + +bash + +®lows + bash +commªds +Ø +be + +execu‹d + +by + +»ces +. + +35 #Thi  +is + +a + +»quœem’t + '£tup-’v‹¡.sh' +š + +the + +‹¡ + +rg‘ +. + +36 #O±iÚ  +¬e + +£t + +to + +ex™ + +wh’ + +a + +»ce + +lše + +ex™s + +nÚ +- +z”o + +Ü +‡ +ped + +commªd + +çžs +. + +37 +SHELL + = / +u¤ +/ +bš +/ +’v + +bash + - +o + +peçž + + +38 . +SHELLFLAGS + = - +ec + + +40 +®l +: +bužd + + +42 ##@ +Dev–Ým’t + + +44 +mªiã¡s +: +cÚŒÞËr +- +g’ + ## +G’”©e + +WebhookCÚfigu¿tiÚ +, +Clu¡”RÞe + +ªd + +Cu¡omResourûDefš™iÚ + +objeùs +. + +45 + `$ +( +CONTROLLER_GEN +è + $$ +( +CRD_OPTIONS +è +rbac +: +rÞeName += +mªag” +- +rÞe + +webhook + +·ths +="./..." +ouut +: +üd +: +¬tiçùs +: +cÚfig +=cÚfig/üd/ +ba£s + + +47 +g’”©e +: +cÚŒÞËr +- +g’ + ## +G’”©e + +code + +cÚššg + +D“pCÝy +, +D“pCÝyIÁo +, +ªd + +D“pCÝyObjeù + +m‘hod + +im¶em’tiÚs +. + +48 + $$ +( +CONTROLLER_GEN +è +objeù +: +h—d”Fže +="hack/bož”¶©e.go.txt" +·ths +="./..." + +50 +fmt +: ## +Run + +go + fmˆ +agaš¡ + +code +. + +51 +go + +fmt + ./... + +53 +v‘ +: ## +Run + +go + v‘ +agaš¡ + +code +. + +54 +go + +v‘ + ./... + +56 +TEST + ?ð./ +­is +/ +d©aba£ +/ +v1®pha1 + ./ +commÚs +/... ./ +cÚŒÞËrs +/... + +57 +‹¡ +: +mªiã¡s + +g’”©e + +fmt + +v‘ + +’v‹¡ + ## +Run + +un™ + +‹¡s +. + +58 +KUBEBUILDER_ASSETS +="$(sh–È$(ENVTESTèu£ $(ENVTEST_K8S_VERSIONè-°·th)" +go + +‹¡ + + `$ +( +TEST +è- +cov”´ofže + +cov” +. +out + + +60 +E2ETEST + ?ð./ +‹¡ +/ +e2e +/ + +61 +e2e +: +mªiã¡s + +g’”©e + +fmt + +v‘ + +’v‹¡ + ## +Run +ƒ2 +‹¡s +. + +62 +KUBEBUILDER_ASSETS +="$(sh–È$(ENVTESTèu£ $(ENVTEST_K8S_VERSIONè-°·th)" +gškgo + - +v + -- +timeout +=2 +h30m + -- +çž +- +ç¡ + + $$ +( +E2ETEST +) + +64 ##@ +Bužd + + +66 +bužd +: +g’”©e + +fmt + +v‘ + ## +Bužd + +mªag” + +bš¬y +. + +67 +go + +bužd + - +o + +bš +/ +mªag” + +maš +.go + +69 +run +: +mªiã¡s + +g’”©e + +fmt + +v‘ + ## +Run + +a + +cÚŒÞËr + +äom + +your + +ho¡ +. + +70 +go + +run + ./ +maš +.go + +72 +dock” +- +bužd +: +mªiã¡s + +g’”©e + +fmt + +v‘ + #‹¡ ## +Bužd + dock” +image + +w™h + +the + +mªag” +. +Di§bË +h +‹¡ + +but + +k“p +h +v®id©iÚs + +to + +çž + +ç¡ + + +73 +dock” + +bužd + -- +no +- +ÿche += +Œue + --bužd- +¬g + +h‰p_´oxy += +$ +{ +HTTP_PROXY +} --bužd-¬g h‰ps_´oxy=${HTTPS_PROXY} . -ˆ${IMG} + +75 #dock”- +bužd +- +´oxy +: +‹¡ + + +76 #dock” +bužd + --bužd- +¬g + +h‰p_´oxy += +$ +{h‰p_´oxy} --bužd-¬g +h‰ps_´oxy +=${h‰ps_´oxy} bužd . - +t + ${ +IMG +} + +78 +dock” +- +push +: ## +Push + dock” +image + +w™h + +the + +mªag” +. + +79 +dock” + +push + +$ +{ +IMG +} + +81 ##@ +D•loym’t + + +83 +š¡®l +: +mªiã¡s + +ku¡omize + ## +In¡®l + +CRDs + +što + +the + +K8s + +þu¡” + +¥ecif›d + +š + ~/. +kube +/ +cÚfig +. + +84 + $$ +( +KUSTOMIZE +è +bužd + +cÚfig +/ +üd + | +kubeùl + +­¶y + - +f + - + +86 +unš¡®l +: +mªiã¡s + +ku¡omize + ## +Unš¡®l + +CRDs + +äom + +the + +K8s + +þu¡” + +¥ecif›d + +š + ~/. +kube +/ +cÚfig +. + +87 + $$ +( +KUSTOMIZE +è +bužd + +cÚfig +/ +üd + | +kubeùl + +d–‘e + - +f + - + +89 +d•loy +: +mªiã¡s + +ku¡omize + ## +D•loy + +cÚŒÞËr + +to + +the + +K8s + +þu¡” + +¥ecif›d + +š + ~/. +kube +/ +cÚfig +. + +90 +cd + +cÚfig +/ +mªag” + && + $$ +( +KUSTOMIZE +è +ed™ + +£t + +image + +cÚŒÞËr += +$ +{ +IMG +} + +91 + $$ +( +KUSTOMIZE +è +bužd + +cÚfig +/ | +kubeùl + +­¶y + - +f + - + +94 #U£d +£d + +to + +»pos™iÚ + +the + +cÚŒÞËr +- +mªag” + +D•loym’t + +aá” +h +û¹ifiÿ‹ + +ü—tiÚ + +š +h +OPERATOR_YAML + + +95 +Ý”©Ü +- +yaml +: +mªiã¡s + +ku¡omize + + +96 +cd + +cÚfig +/ +mªag” + && + $$ +( +KUSTOMIZE +è +ed™ + +£t + +image + +cÚŒÞËr += +$ +{ +IMG +} + +97 + $$ +( +KUSTOMIZE +è +bužd + +cÚfig +/ > "${OPERATOR_YAML}" + +98 +£d + - +i +. +bak + - +e + '/^apiVersion:‡pps\/v1/,/---/d' "${OPERATOR_YAML}" + +99 ( +echo + --- && +£d + '/^apiVersion:‡pps\/v1/,/---/!d' "${OPERATOR_YAML}.bak") >> "${OPERATOR_YAML}" + +100 +rm + "${OPERATOR_YAML}.bak" + +102 +und•loy +: ## +Und•loy + +cÚŒÞËr + +äom + +the + +K8s + +þu¡” + +¥ecif›d + +š + ~/. +kube +/ +cÚfig +. + +103 + $$ +( +KUSTOMIZE +è +bužd + +cÚfig +/ | +kubeùl + +d–‘e + - +f + - + +105 ##@ +Bužd + +D•’d’c›s + + +107 ## +LoÿtiÚ + +to + +š¡®l + +d•’d’c›s +o + +108 +LOCALBIN + ?ð + `$ +( +sh–l + +pwd +)/ +bš + + +109 + $$ +( +LOCALBIN +): + +110 +mkdœ + - +p + + $$ +( +LOCALBIN +) + +112 ## +ToÞ + +Bš¬›s + + +113 +KUSTOMIZE + ?ð + `$ +( +LOCALBIN +)/ +ku¡omize + + +114 +CONTROLLER_GEN + ?ð + `$ +( +LOCALBIN +)/ +cÚŒÞËr +- +g’ + + +115 +ENVTEST + ?ð + `$ +( +LOCALBIN +)/ +£tup +- +’v‹¡ + + +117 ## +ToÞ + +V”siÚs + + +118 +KUSTOMIZE_VERSION + ?ð +v3 +.8.7 + +119 +CONTROLLER_TOOLS_VERSION + ?ð +v0 +.6.1 + +121 +KUSTOMIZE_INSTALL_SCRIPT + ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" + +122 . +PHONY +: +ku¡omize + + +123 +ku¡omize +: + $$ +( +KUSTOMIZE +è## +DowÆßd + +ku¡omize + +loÿÎy +  +Ãûs§ry +. + +124 + $$ +( +KUSTOMIZE +): + $$ +( +LOCALBIN +) + +125 +cu¾ + - +s + + `$ +( +KUSTOMIZE_INSTALL_SCRIPT +è| +bash + - -- $( +sub¡ + +v +,,$( +KUSTOMIZE_VERSION +)è$( +LOCALBIN +) + +127 . +PHONY +: +cÚŒÞËr +- +g’ + + +128 +cÚŒÞËr +- +g’ +: + $$ +( +CONTROLLER_GEN +è## +DowÆßd + +cÚŒÞËr +- +g’ + +loÿÎy +  +Ãûs§ry +. + +129 + $$ +( +CONTROLLER_GEN +): + $$ +( +LOCALBIN +) + +130 +GOBIN += + $$ +( +LOCALBIN +è +go + +š¡®l + +sigs +. +k8s +. +io +/ +cÚŒÞËr +- +toÞs +/ +cmd +/cÚŒÞËr- +g’ +@ + `$ +( +CONTROLLER_TOOLS_VERSION +) + +132 . +PHONY +: +’v‹¡ + + +133 +’v‹¡ +: + $$ +( +ENVTEST +è## +DowÆßd + +’v‹¡ +- +£tup + +loÿÎy +  +Ãûs§ry +. + +134 + $$ +( +ENVTEST +): + $$ +( +LOCALBIN +) + +135 +GOBIN += + $$ +( +LOCALBIN +è +go + +š¡®l + +sigs +. +k8s +. +io +/ +cÚŒÞËr +- +ruÁime +/ +toÞs +/ +£tup +- +’v‹¡ +@ +Ï‹¡ + + +138 . +PHONY +: +bundË + + +139 +bundË +: +mªiã¡s + +ku¡omize + ## +G’”©e + bundË mªiã¡  +ªd + +m‘ad©a +, +th’ + +v®id©e + +g’”©ed + +fžes +. + +140 +Ý”©Ü +- +sdk + +g’”©e + +ku¡omize + +mªiã¡s + - +q + + +141 +cd + +cÚfig +/ +mªag” + && + $$ +( +KUSTOMIZE +è +ed™ + +£t + +image + +cÚŒÞËr += + `$ +( +IMG +) + +142 + $$ +( +KUSTOMIZE +è +bužd + +cÚfig +/ +mªiã¡s + | +Ý”©Ü +- +sdk + +g’”©e + +bundË + - +q + -- +ov”wr™e + -- +v”siÚ + + `$ +( +VERSION +è + $$ +( +BUNDLE_METADATA_OPTS +) + +143 +Ý”©Ü +- +sdk + +bundË + +v®id©e + ./bundle + +145 . +PHONY +: +bundË +- +bužd + + +146 +bundË +- +bužd +: ## +Bužd + +the + bundË +image +. + +147 +dock” + +bužd + - +f + +bundË +. +Dock”fže + - +t + + `$ +( +BUNDLE_IMG +) . + +149 . +PHONY +: +bundË +- +push + + +150 +bundË +- +push +: ## +Push + +the + bundË +image +. + +151 + $$ +( +MAKE +è +dock” +- +push + +IMG += + `$ +( +BUNDLE_IMG +) + +153 . +PHONY +: +Ým + + +154 +OPM + = ./ +bš +/ +Ým + + +155 +Ým +: ## +DowÆßd + opm +loÿÎy +  +Ãûs§ry +. + +156 + `iãq + (, + `$ +( +wždÿrd + + $$ +( +OPM +))) + +157 + `iãq + (, + `$ +( +sh–l + +which + +Ým + 2>/ +dev +/ +nuÎ +)) + +159 +£t + - +e + ;\ + +160 +mkdœ + - +p + + `$ +( +dœ + $( +OPM +)) ;\ + +161 +OS += + `$ +( +sh–l + +go + +’v + +GOOS +è&& +ARCH +=$(sh–ÈgØ’v +GOARCH +) && \ + +162 +cu¾ + - +sSLo + + `$ +( +OPM +è +h‰ps +: + +163 +chmod + + +x + + `$ +( +OPM +) ;\ + +164 + } +} + +166 + gOPM + = + $$ +( +sh–l + +which + +Ým +) + +167 +’dif + + +168 +’dif + + +170 #A +comma +- +£·¿‹d + +li¡ + +of + +bundË + + `images + ( +e +. +g +. +make + +ÿlog +- +bužd + +BUNDLE_IMGS += +exam¶e +. +com +/ +Ý”©Ü +-bundË: +v0 +.1.0,example.com/operator-bundle:v0.2.0). + +171 #The£ +images + +MUST + +exi¡ + +š + +a + +»gi¡ry + +ªd + +be + +puÎ +- +abË +. + +172 +BUNDLE_IMGS + ?ð + $$ +( +BUNDLE_IMG +) + +174 #Th +image + +g + +giv’ + +to + +the + +»suÉšg + +ÿlog + + `image + ( +e +. +g +. +make + c©®og- +bužd + +CATALOG_IMG += +exam¶e +. +com +/ +Ý”©Ü +-ÿlog: +v0 +.2.0). + +175 +CATALOG_IMG + ?ð + `$ +( +IMAGE_TAG_BASE +)- +ÿlog +: + $v$ +( +VERSION +) + +177 #S‘ +CATALOG_BASE_IMG + +to + +ª + +exi¡šg + +ÿlog + +image + +g +Ø +add + +$BUNDLE_IMGS +Ø +th© + image. + +178 + `iâeq + ( + `$ +( +Üigš + +CATALOG_BASE_IMG +), +undefšed +) + +179 +FROM_INDEX_OPT + :ð-- +äom +- +šdex + + $$ +( +CATALOG_BASE_IMG +) + +180 +’dif + + +182 #Bužd +a + +ÿlog + +image + +by + +addšg + +bundË + +images + +to + +ª + +em±y + c©®og +usšg + +the + +Ý”©Ü + +·ckage + +mªag” + +toÞ +, 'opm'. + +183 #Thi  +»ce + +švokes + 'Ým' +š + '£mv”' +bundË + +add + +mode +. +FÜ + +mÜe + +šfÜm©iÚ + +Ú +‡dd +modes +, +£e +: + +185 . +PHONY +: +ÿlog +- +bužd + + +186 +ÿlog +- +bužd +: +Ým + ## +Bužd + +a + c©®og +image +. + +187 + $$ +( +OPM +è +šdex + +add + -- +cÚš” +- +toÞ + +dock” + -- +mode + +£mv” + -- +g + + `$ +( +CATALOG_IMG +è-- +bundËs + $( +BUNDLE_IMGS +è + $$ +( +FROM_INDEX_OPT +) + +189 #Push +the + +ÿlog + +image +. + +190 . +PHONY +: +ÿlog +- +push + + +191 +ÿlog +- +push +: ## +Push + +a + c©®og +image +. + +192 + $$ +( +MAKE +è +dock” +- +push + +IMG += + `$ +( +CATALOG_IMG +) + + @ +1 +. +1 +/usr/include +1 +9 +Makefile diff --git a/cscope.tmplst b/cscope.tmplst new file mode 100644 index 00000000..0937d485 --- /dev/null +++ b/cscope.tmplst @@ -0,0 +1 @@ +./Makefile From 8990cbf2057fc003a35286a37d187fd0f86d2060 Mon Sep 17 00:00:00 2001 From: Abhishek Kumar Date: Thu, 29 Sep 2022 16:13:28 +0000 Subject: [PATCH 31/37] Patching the svc while changing type from LB to NodePort, max cert validity 1yr --- .../singleinstancedatabase_webhook.go | 4 +- commons/database/constants.go | 4 +- .../crd/bases/database.oracle.com_cdbs.yaml | 2 - .../samples/sidb/singleinstancedatabase.yaml | 6 +- .../sidb/singleinstancedatabase_tcps.yaml | 6 +- .../singleinstancedatabase_controller.go | 84 +++++++------------ docs/sidb/README.md | 4 +- 7 files changed, 44 insertions(+), 66 deletions(-) diff --git a/apis/database/v1alpha1/singleinstancedatabase_webhook.go b/apis/database/v1alpha1/singleinstancedatabase_webhook.go index 1006c68e..6323550b 100644 --- a/apis/database/v1alpha1/singleinstancedatabase_webhook.go +++ b/apis/database/v1alpha1/singleinstancedatabase_webhook.go @@ -270,12 +270,12 @@ func (r *SingleInstanceDatabase) ValidateCreate() error { field.Invalid(field.NewPath("spec").Child("tcpsCertRenewInterval"), r.Spec.TcpsCertRenewInterval, "Please provide valid string to parse the tcpsCertRenewInterval.")) } - maxLimit, _ := time.ParseDuration("26280h") + maxLimit, _ := time.ParseDuration("8760h") minLimit, _ := time.ParseDuration("24h") if duration > maxLimit || duration < minLimit { allErrs = append(allErrs, field.Invalid(field.NewPath("spec").Child("tcpsCertRenewInterval"), r.Spec.TcpsCertRenewInterval, - "Please specify tcpsCertRenewInterval in the range: 24h to 26280h")) + "Please specify tcpsCertRenewInterval in the range: 24h to 8760h")) } } if len(allErrs) == 0 { diff --git a/commons/database/constants.go b/commons/database/constants.go index a1d7bfc8..76750ffb 100644 --- a/commons/database/constants.go +++ b/commons/database/constants.go @@ -501,10 +501,10 @@ const ClientWalletLocation string = "/opt/oracle/oradata/clientWallet/%s" // Service Patch Payloads // Three port payload: one OEM express, one TCP and one TCPS port -const ThreePortPayload string = "{\"spec\": { \"ports\": [{\"name\": \"xmldb\", \"port\": 5500, \"protocol\": \"TCP\"},{%s},{%s}]}}" +const ThreePortPayload string = "{\"spec\": { \"type\": \"%s\", \"ports\": [{\"name\": \"xmldb\", \"port\": 5500, \"protocol\": \"TCP\"},{%s},{%s}]}}" // Two port payload: one OEM express, one TCP/TCPS port -const TwoPortPayload string = "{\"spec\": { \"ports\": [{\"name\": \"xmldb\", \"port\": 5500, \"protocol\": \"TCP\"},{%s}]}}" +const TwoPortPayload string = "{\"spec\": { \"type\": \"%s\", \"ports\": [{\"name\": \"xmldb\", \"port\": 5500, \"protocol\": \"TCP\"},{%s}]}}" // Payload section for listener port const LsnrPort string = "\"name\": \"listener\", \"protocol\": \"TCP\", \"port\": %d, \"targetPort\": 1521" diff --git a/config/crd/bases/database.oracle.com_cdbs.yaml b/config/crd/bases/database.oracle.com_cdbs.yaml index bf64810b..26eec406 100644 --- a/config/crd/bases/database.oracle.com_cdbs.yaml +++ b/config/crd/bases/database.oracle.com_cdbs.yaml @@ -65,8 +65,6 @@ spec: spec: description: CDBSpec defines the desired state of CDB properties: - TestVariable: - type: string cdbAdminPwd: description: Password for the CDB Administrator to manage PDB lifecycle properties: diff --git a/config/samples/sidb/singleinstancedatabase.yaml b/config/samples/sidb/singleinstancedatabase.yaml index 0fafd46e..f836f6dd 100644 --- a/config/samples/sidb/singleinstancedatabase.yaml +++ b/config/samples/sidb/singleinstancedatabase.yaml @@ -48,10 +48,10 @@ spec: enableTCPS: false ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled. - ## tcpsCertRenewInterval can be in hours(h), minutes(m) and seconds(s); e.g. 17520h, 8760h etc. - ## Maximum value is 26280h (3 years), Minimum value is 24h; Default value is 17520h (2 years) + ## tcpsCertRenewInterval can be in hours(h), minutes(m) and seconds(s); e.g. 4380h, 8760h etc. + ## Maximum value is 8760h (1 year), Minimum value is 24h; Default value is 8760h (1 year) ## If this field is commented out/removed from the yaml, it will disable the auto-renewal feature for TCPS certificate - tcpsCertRenewInterval: 17520h + tcpsCertRenewInterval: 8760h ## NA if cloning from a SourceDB (cloneFrom is set) ## Specify both sgaSize and pgaSize (in MB) or dont specify both diff --git a/config/samples/sidb/singleinstancedatabase_tcps.yaml b/config/samples/sidb/singleinstancedatabase_tcps.yaml index b4a64ba8..2f9e4994 100644 --- a/config/samples/sidb/singleinstancedatabase_tcps.yaml +++ b/config/samples/sidb/singleinstancedatabase_tcps.yaml @@ -35,10 +35,10 @@ spec: enableTCPS: true ## TCPS Certificate Renewal Interval: The time after which TCPS certificate will be renewed if TCPS connections are enabled. - ## tcpsCertRenewInterval can be in hours(h), minutes(m) and seconds(s); e.g. 17520h, 8760h etc. - ## Maximum value is 26280h (3 years), Minimum value is 24h; Default value is 17520h (2 years) + ## tcpsCertRenewInterval can be in hours(h), minutes(m) and seconds(s); e.g. 4380h, 8760h etc. + ## Maximum value is 8760h (1 year), Minimum value is 24h; Default value is 8760h (1 year) ## If this field is commented out/removed from the yaml, it will disable the auto-renewal feature for TCPS certificate - tcpsCertRenewInterval: 17520h + tcpsCertRenewInterval: 8760h ## Database image details image: diff --git a/controllers/database/singleinstancedatabase_controller.go b/controllers/database/singleinstancedatabase_controller.go index 77d40f90..314b2c66 100644 --- a/controllers/database/singleinstancedatabase_controller.go +++ b/controllers/database/singleinstancedatabase_controller.go @@ -1126,57 +1126,37 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex } } - deleteSvc := false patchSvc := false - if extSvc.Spec.Type != extSvcType { - deleteSvc = true - } else { - // Conditions to determine whether to patch or not - if len(extSvc.Spec.Ports) != requiredPorts { - patchSvc = true - } - if (m.Spec.ListenerPort != 0 && svcPort != targetPorts[1]) || (m.Spec.EnableTCPS && m.Spec.TcpsListenerPort != 0 && tcpsSvcPort != targetPorts[len(targetPorts)-1]) { - patchSvc = true - } + // Conditions to determine whether to patch or not + if extSvc.Spec.Type != extSvcType || len(extSvc.Spec.Ports) != requiredPorts { + patchSvc = true + } - if m.Spec.LoadBalancer { - if m.Spec.EnableTCPS { - if m.Spec.TcpsListenerPort == 0 && tcpsSvcPort != targetPorts[len(targetPorts)-1] { - patchSvc = true - } - } else { - if m.Spec.ListenerPort == 0 && svcPort != targetPorts[1] { - patchSvc = true - } + if (m.Spec.ListenerPort != 0 && svcPort != targetPorts[1]) || (m.Spec.EnableTCPS && m.Spec.TcpsListenerPort != 0 && tcpsSvcPort != targetPorts[len(targetPorts)-1]) { + patchSvc = true + } + + if m.Spec.LoadBalancer { + if m.Spec.EnableTCPS { + if m.Spec.TcpsListenerPort == 0 && tcpsSvcPort != targetPorts[len(targetPorts)-1] { + patchSvc = true } } else { - if m.Spec.EnableTCPS { - if m.Spec.TcpsListenerPort == 0 && tcpsSvcPort != extSvc.Spec.Ports[len(targetPorts)-1].TargetPort.IntVal { - patchSvc = true - } - } else { - if m.Spec.ListenerPort == 0 && svcPort != extSvc.Spec.Ports[1].TargetPort.IntVal { - patchSvc = true - } + if m.Spec.ListenerPort == 0 && svcPort != targetPorts[1] { + patchSvc = true } } - } - - if deleteSvc { - // Deleting th service - log.Info("Deleting service", "name", extSvcName) - // Setting GracePeriodSeconds to 0 for instant deletion - delOpts := &client.DeleteOptions{} - var gracePeriod client.GracePeriodSeconds = 0 - gracePeriod.ApplyToDelete(delOpts) - - err := r.Delete(ctx, extSvc, delOpts) - if err != nil { - r.Log.Error(err, "Failed to delete service", "name", extSvcName) - return requeueN, err + } else { + if m.Spec.EnableTCPS { + if m.Spec.TcpsListenerPort == 0 && tcpsSvcPort != extSvc.Spec.Ports[len(targetPorts)-1].TargetPort.IntVal { + patchSvc = true + } + } else { + if m.Spec.ListenerPort == 0 && svcPort != extSvc.Spec.Ports[1].TargetPort.IntVal { + patchSvc = true + } } - isExtSvcFound = false } if patchSvc { @@ -1193,29 +1173,29 @@ func (r *SingleInstanceDatabaseReconciler) createOrReplaceSVC(ctx context.Contex if m.Spec.LoadBalancer { if m.Spec.EnableTCPS { if m.Spec.ListenerPort != 0 { - payload = fmt.Sprintf(dbcommons.ThreePortPayload, fmt.Sprintf(dbcommons.LsnrPort, svcPort), fmt.Sprintf(dbcommons.TcpsPort, tcpsSvcPort)) + payload = fmt.Sprintf(dbcommons.ThreePortPayload, extSvcType, fmt.Sprintf(dbcommons.LsnrPort, svcPort), fmt.Sprintf(dbcommons.TcpsPort, tcpsSvcPort)) } else { - payload = fmt.Sprintf(dbcommons.TwoPortPayload, fmt.Sprintf(dbcommons.TcpsPort, tcpsSvcPort)) + payload = fmt.Sprintf(dbcommons.TwoPortPayload, extSvcType, fmt.Sprintf(dbcommons.TcpsPort, tcpsSvcPort)) } } else { - payload = fmt.Sprintf(dbcommons.TwoPortPayload, fmt.Sprintf(dbcommons.LsnrPort, svcPort)) + payload = fmt.Sprintf(dbcommons.TwoPortPayload, extSvcType, fmt.Sprintf(dbcommons.LsnrPort, svcPort)) } } else { if m.Spec.EnableTCPS { if m.Spec.ListenerPort != 0 && m.Spec.TcpsListenerPort != 0 { - payload = fmt.Sprintf(dbcommons.ThreePortPayload, fmt.Sprintf(dbcommons.LsnrNodePort, svcPort), fmt.Sprintf(dbcommons.TcpsNodePort, tcpsSvcPort)) + payload = fmt.Sprintf(dbcommons.ThreePortPayload, extSvcType, fmt.Sprintf(dbcommons.LsnrNodePort, svcPort), fmt.Sprintf(dbcommons.TcpsNodePort, tcpsSvcPort)) } else if m.Spec.ListenerPort != 0 { - payload = fmt.Sprintf(dbcommons.ThreePortPayload, fmt.Sprintf(dbcommons.LsnrNodePort, svcPort), fmt.Sprintf(dbcommons.TcpsPort, tcpsSvcPort)) + payload = fmt.Sprintf(dbcommons.ThreePortPayload, extSvcType, fmt.Sprintf(dbcommons.LsnrNodePort, svcPort), fmt.Sprintf(dbcommons.TcpsPort, tcpsSvcPort)) } else if m.Spec.TcpsListenerPort != 0 { - payload = fmt.Sprintf(dbcommons.TwoPortPayload, fmt.Sprintf(dbcommons.TcpsNodePort, tcpsSvcPort)) + payload = fmt.Sprintf(dbcommons.TwoPortPayload, extSvcType, fmt.Sprintf(dbcommons.TcpsNodePort, tcpsSvcPort)) } else { - payload = fmt.Sprintf(dbcommons.TwoPortPayload, fmt.Sprintf(dbcommons.TcpsPort, tcpsSvcPort)) + payload = fmt.Sprintf(dbcommons.TwoPortPayload, extSvcType, fmt.Sprintf(dbcommons.TcpsPort, tcpsSvcPort)) } } else { if m.Spec.ListenerPort != 0 { - payload = fmt.Sprintf(dbcommons.TwoPortPayload, fmt.Sprintf(dbcommons.LsnrNodePort, svcPort)) + payload = fmt.Sprintf(dbcommons.TwoPortPayload, extSvcType, fmt.Sprintf(dbcommons.LsnrNodePort, svcPort)) } else { - payload = fmt.Sprintf(dbcommons.TwoPortPayload, fmt.Sprintf(dbcommons.LsnrPort, svcPort)) + payload = fmt.Sprintf(dbcommons.TwoPortPayload, extSvcType, fmt.Sprintf(dbcommons.LsnrPort, svcPort)) } } } diff --git a/docs/sidb/README.md b/docs/sidb/README.md index 1e9f02c3..33257d24 100644 --- a/docs/sidb/README.md +++ b/docs/sidb/README.md @@ -543,8 +543,8 @@ The following steps are required to connect the Database using TCPS: **NOTE:** - Only database server authentication is supported (no mTLS). - When TCPS is enabled, a self-signed certificate is generated and stored inside the wallets. For users' convenience, a client-side wallet is generated and stored at `/opt/oracle/oradata/clientWallet/$ORACLE_SID` location in the pod. -- The self-signed certificate used with TCPS has validity for 2 years. After the certificate is expired, it will be renewed by the `OraOperator` automatically. You need to download the wallet again after the auto-renewal. -- You can set the certificate renew interval with the help of `tcpsCertRenewInterval` field in the **[config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml)** file. The minimum accepted value is 24h, and the maximum value is 26280h (3 years). The certificates used with TCPS will automatically be renewed after this interval. If this field is omitted/commented in the yaml file, the certificates will not be renewed automatically. +- The self-signed certificate used with TCPS has validity for 1 year. After the certificate is expired, it will be renewed by the `OraOperator` automatically. You need to download the wallet again after the auto-renewal. +- You can set the certificate renew interval with the help of `tcpsCertRenewInterval` field in the **[config/samples/sidb/singleinstancedatabase.yaml](../../config/samples/sidb/singleinstancedatabase.yaml)** file. The minimum accepted value is 24h, and the maximum value is 8760h (1 year). The certificates used with TCPS will automatically be renewed after this interval. If this field is omitted/commented in the yaml file, the certificates will not be renewed automatically. - When the certificate gets created/renewed, the `.status.certCreationTimestamp` status variable gets updated accordingly. You can see this timestamp by using the following command: ```bash kubectl get singleinstancedatabase sidb-sample -o "jsonpath={.status.certCreationTimestamp}" From 0fc0c7448a3845141d5bfdecf37aed37250ab0b4 Mon Sep 17 00:00:00 2001 From: matteo malvezzi Date: Mon, 10 Oct 2022 09:06:03 +0200 Subject: [PATCH 32/37] bug 34677093 --- apis/database/v1alpha1/cdb_types.go | 3 - apis/database/v1alpha1/cdb_webhook.go | 5 +- .../crd/bases/database.oracle.com_cdbs.yaml | 7 - config/samples/multitenant/cdb.yaml | 3 +- controllers/DBserver | 33 + controllers/database/pdb_controller.go | 14 +- cscope.out | 2060 ----------------- cscope.tmplst | 1 - .../multitenant/provisioning/add_replica.yaml | 1 - docs/multitenant/provisioning/cdb.yaml | 1 - oracle-database-operator.yaml | 40 +- 11 files changed, 74 insertions(+), 2094 deletions(-) create mode 100644 controllers/DBserver delete mode 100644 cscope.out delete mode 100644 cscope.tmplst diff --git a/apis/database/v1alpha1/cdb_types.go b/apis/database/v1alpha1/cdb_types.go index 9762c1cc..a987718a 100644 --- a/apis/database/v1alpha1/cdb_types.go +++ b/apis/database/v1alpha1/cdb_types.go @@ -80,8 +80,6 @@ type CDBSpec struct { WebServerUser WebServerUser `json:"webServerUser,omitempty"` // Password for the Web Server User WebServerPwd WebServerPassword `json:"webServerPwd,omitempty"` - // SCAN Name - SCANName string `json:"scanName,omitempty"` // Name of the DB server DBServer string `json:"dbServer,omitempty"` // DB server port @@ -152,7 +150,6 @@ type CDBStatus struct { // +kubebuilder:printcolumn:JSONPath=".spec.cdbName",name="CDB Name",type="string",description="Name of the CDB" // +kubebuilder:printcolumn:JSONPath=".spec.dbServer",name="DB Server",type="string",description=" Name of the DB Server" // +kubebuilder:printcolumn:JSONPath=".spec.dbPort",name="DB Port",type="integer",description="DB server port" -// +kubebuilder:printcolumn:JSONPath=".spec.scanName",name="SCAN Name",type="string",description="SCAN Name" // +kubebuilder:printcolumn:JSONPath=".spec.replicas",name="Replicas",type="integer",description="Replicas" // +kubebuilder:printcolumn:JSONPath=".status.phase",name="Status",type="string",description="Status of the CDB Resource" // +kubebuilder:printcolumn:JSONPath=".status.msg",name="Message",type="string",description="Error message, if any" diff --git a/apis/database/v1alpha1/cdb_webhook.go b/apis/database/v1alpha1/cdb_webhook.go index e38d037b..c760875c 100644 --- a/apis/database/v1alpha1/cdb_webhook.go +++ b/apis/database/v1alpha1/cdb_webhook.go @@ -103,10 +103,11 @@ func (r *CDB) ValidateCreate() error { field.Required(field.NewPath("spec").Child("cdbTlsCrt"), "Please specify CDB Tls Certificate(secret)")) } - if r.Spec.SCANName == "" { + /*if r.Spec.SCANName == "" { allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("scanName"), "Please specify SCAN Name for CDB")) - } + }*/ + if r.Spec.DBServer == "" { allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("dbServer"), "Please specify Database Server Name or IP Address")) diff --git a/config/crd/bases/database.oracle.com_cdbs.yaml b/config/crd/bases/database.oracle.com_cdbs.yaml index 26eec406..ab2ab2ef 100644 --- a/config/crd/bases/database.oracle.com_cdbs.yaml +++ b/config/crd/bases/database.oracle.com_cdbs.yaml @@ -29,10 +29,6 @@ spec: jsonPath: .spec.dbPort name: DB Port type: integer - - description: SCAN Name - jsonPath: .spec.scanName - name: SCAN Name - type: string - description: Replicas jsonPath: .spec.replicas name: Replicas @@ -183,9 +179,6 @@ spec: replicas: description: Number of ORDS Containers to create type: integer - scanName: - description: SCAN Name - type: string serviceName: description: Name of the CDB Service type: string diff --git a/config/samples/multitenant/cdb.yaml b/config/samples/multitenant/cdb.yaml index 7b566cd7..e3513d12 100644 --- a/config/samples/multitenant/cdb.yaml +++ b/config/samples/multitenant/cdb.yaml @@ -9,7 +9,6 @@ metadata: namespace: oracle-database-operator-system spec: cdbName: "devcdb" - scanName: "devdb" dbServer: "172.17.0.4" dbPort: 1521 replicas: 1 @@ -41,4 +40,4 @@ spec: webServerPwd: secret: secretName: "cdb1-secret" - key: "webserver_pwd" \ No newline at end of file + key: "webserver_pwd" diff --git a/controllers/DBserver b/controllers/DBserver new file mode 100644 index 00000000..57c0f4fb --- /dev/null +++ b/controllers/DBserver @@ -0,0 +1,33 @@ +# This viminfo file was generated by Vim 7.4. +# You may edit it if you're careful! + +# Value of 'encoding' when this file was written +*encoding=utf-8 + + +# hlsearch on (H) or off (h): +~H +# Command Line History (newest to oldest): +:q! +:q + +# Search String History (newest to oldest): + +# Expression History (newest to oldest): + +# Input Line History (newest to oldest): + +# Input Line History (newest to oldest): + +# Registers: + +# File marks: +'0 1 0 ~/WORKDIR/Ords22_operator/oracle-database-operator/controllers/grep + +# Jumplist (newest first): +-' 1 0 ~/WORKDIR/Ords22_operator/oracle-database-operator/controllers/grep + +# History of marks within files (newest to oldest): + +> ~/WORKDIR/Ords22_operator/oracle-database-operator/controllers/grep + " 1 0 diff --git a/controllers/database/pdb_controller.go b/controllers/database/pdb_controller.go index 099cb921..07580725 100644 --- a/controllers/database/pdb_controller.go +++ b/controllers/database/pdb_controller.go @@ -444,10 +444,11 @@ func (r *PDBReconciler) callAPI(ctx context.Context, req ctrl.Request, pdb *dbap } caCert := secret.Data[pdb.Spec.PDBTlsCat.Secret.Key] - + /* r.Recorder.Eventf(pdb, corev1.EventTypeWarning, "ORDSINFO", string(rsaKeyPEM)) r.Recorder.Eventf(pdb, corev1.EventTypeWarning, "ORDSINFO", string(rsaCertPEM)) r.Recorder.Eventf(pdb, corev1.EventTypeWarning, "ORDSINFO", string(caCert)) + */ certificate, err := tls.X509KeyPair([]byte(rsaCertPEM), []byte(rsaKeyPEM)) if err != nil { @@ -501,7 +502,6 @@ func (r *PDBReconciler) callAPI(ctx context.Context, req ctrl.Request, pdb *dbap if action == "GET" { httpreq, err = http.NewRequest(action, url, nil) } else { - fmt.Println("payload:", payload) jsonValue, _ := json.Marshal(payload) httpreq, err = http.NewRequest(action, url, bytes.NewBuffer(jsonValue)) } @@ -641,7 +641,7 @@ func (r *PDBReconciler) createPDB(ctx context.Context, req ctrl.Request, pdb *db r.Recorder.Eventf(pdb, corev1.EventTypeNormal, "Created", "PDB '%s' created successfully", pdb.Spec.PDBName) - pdb.Status.ConnString = cdb.Spec.SCANName + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName log.Info("Created PDB Resource", "PDB Name", pdb.Spec.PDBName) r.getPDBState(ctx, req, pdb) return nil @@ -695,7 +695,7 @@ func (r *PDBReconciler) clonePDB(ctx context.Context, req ctrl.Request, pdb *dba r.Recorder.Eventf(pdb, corev1.EventTypeNormal, "Created", "PDB '%s' cloned successfully", pdb.Spec.PDBName) - pdb.Status.ConnString = cdb.Spec.SCANName + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName log.Info("Cloned PDB successfully", "Source PDB Name", pdb.Spec.SrcPDBName, "Clone PDB Name", pdb.Spec.PDBName) r.getPDBState(ctx, req, pdb) return nil @@ -763,7 +763,7 @@ func (r *PDBReconciler) plugPDB(ctx context.Context, req ctrl.Request, pdb *dbap r.Recorder.Eventf(pdb, corev1.EventTypeNormal, "Created", "PDB '%s' plugged successfully", pdb.Spec.PDBName) - pdb.Status.ConnString = cdb.Spec.SCANName + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName log.Info("Successfully plugged PDB", "PDB Name", pdb.Spec.PDBName) r.getPDBState(ctx, req, pdb) return nil @@ -883,7 +883,7 @@ func (r *PDBReconciler) modifyPDB(ctx context.Context, req ctrl.Request, pdb *db } r.Recorder.Eventf(pdb, corev1.EventTypeNormal, "Modified", "PDB '%s' modified successfully", pdb.Spec.PDBName) - pdb.Status.ConnString = cdb.Spec.SCANName + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName log.Info("Successfully modified PDB state", "PDB Name", pdb.Spec.PDBName) r.getPDBState(ctx, req, pdb) @@ -970,7 +970,7 @@ func (r *PDBReconciler) mapPDB(ctx context.Context, req ctrl.Request, pdb *dbapi pdb.Status.OpenMode = objmap["open_mode"].(string) pdb.Status.TotalSize = fmt.Sprintf("%.2f", totSizeInGB) + "G" - pdb.Status.ConnString = cdb.Spec.SCANName + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName log.Info("Successfully mapped PDB to Kubernetes resource", "PDB Name", pdb.Spec.PDBName) return nil diff --git a/cscope.out b/cscope.out deleted file mode 100644 index fbb75c7b..00000000 --- a/cscope.out +++ /dev/null @@ -1,2060 +0,0 @@ -cscope 15 $HOME/WORKDIR/Ords22_operator/oracle-database-operator 0000008991 - @Makefile - -1 # -#CÝyrighˆ( -c -è2022, -O¿þe - -ªd -/ -Ü - -™s - -affžŸ‹s -. - -2 #Liûn£d -und” - -the - -Univ”§l - -P”missive - -Liûn£ - -v - 1.0 -as - -shown - -© - -h‰p -: - -4 #Cu¼’ˆ -O³¿tÜ - -v”siÚ - - -5 - gVERSION - ?= 0.0.1 - -6 #DeçuÉ -bundË - -image - -g - - -7 -BUNDLE_IMG - ?ð -cÚŒÞËr -- -bundË -: - $$ -( -VERSION -) - -9 - `iâeq - ( - `$ -( -Üigš - -CHANNELS -), -undefšed -) - -10 -BUNDLE_CHANNELS - :ð-- -chªÃls -= - $$ -( -CHANNELS -) - -11 -’dif - - -12 - `iâeq - ( - `$ -( -Üigš - -DEFAULT_CHANNEL -), -undefšed -) - -13 -BUNDLE_DEFAULT_CHANNEL - :ð--- -chªÃl -= - $$ -( -DEFAULT_CHANNEL -) - -14 -’dif - - -15 -BUNDLE_METADATA_OPTS - ?ð - `$ -( -BUNDLE_CHANNELS -è - $$ -( -BUNDLE_DEFAULT_CHANNEL -) - -17 #Imag -URL - -to - -u£ - -®l - -buždšg -/ -pushšg - -image - -rg‘s - - -18 -IMG - ?ð -cÚŒÞËr -: -Ï‹¡ - - -19 #Produû -CRDs - -th© - -wÜk - -back - -to - -Kub”Ës - 1.11 ( -no - -v”siÚ - -cÚv”siÚ -) - -20 #API -v”siÚ - -has - -to - -be - -v1 -Ø -u£ - - `deçuÉšg - ( -h‰ps -: - -21 -CRD_OPTIONS - ?= "crd:trivialVersions=true,preserveUnknownFields=false" - -22 #ENVTEST_K8S_VERSION -»ãrs - -to - -the - -v”siÚ - -of - -kubebužd” - -as£ts -Ø -be - -dowÆßded - -by - -’v‹¡ - -bš¬y -. - -23 -ENVTEST_K8S_VERSION - = 1.21 - -24 #O³¿tÜ -YAML - -fže - - -25 -OPERATOR_YAML -= - `$$ -( -ba£Çme - $$( -pwd -)). -yaml - - -27 #G‘ -the - -cu¼’Žy - -u£d - -gÞªg - -š¡®l - - `·th - ( -š - -GOPATH -/ -bš -, -uÆess - -GOBIN - -is - -£t -) - -28 - `iãq - (, - $$ -( -sh–l - -go - -’v - -GOBIN -)) - -29 -GOBIN -= - `$ -( -sh–l - -go - -’v - -GOPATH -)/ -bš - - -31 -GOBIN -= - $$ -( -sh–l - -go - -’v - -GOBIN -) - -32 -’dif - - -34 #S‘tšg -SHELL - -to - -bash - -®lows - bash -commªds -Ø -be - -execu‹d - -by - -»ces -. - -35 #Thi  -is - -a - -»quœem’t - '£tup-’v‹¡.sh' -š - -the - -‹¡ - -rg‘ -. - -36 #O±iÚ  -¬e - -£t - -to - -ex™ - -wh’ - -a - -»ce - -lše - -ex™s - -nÚ -- -z”o - -Ü -‡ -ped - -commªd - -çžs -. - -37 -SHELL - = / -u¤ -/ -bš -/ -’v - -bash - - -o - -peçž - - -38 . -SHELLFLAGS - = - -ec - - -40 -®l -: -bužd - - -42 ##@ -Dev–Ým’t - - -44 -mªiã¡s -: -cÚŒÞËr -- -g’ - ## -G’”©e - -WebhookCÚfigu¿tiÚ -, -Clu¡”RÞe - -ªd - -Cu¡omResourûDefš™iÚ - -objeùs -. - -45 - `$ -( -CONTROLLER_GEN -è - $$ -( -CRD_OPTIONS -è -rbac -: -rÞeName -= -mªag” -- -rÞe - -webhook - -·ths -="./..." -ouut -: -üd -: -¬tiçùs -: -cÚfig -=cÚfig/üd/ -ba£s - - -47 -g’”©e -: -cÚŒÞËr -- -g’ - ## -G’”©e - -code - -cÚššg - -D“pCÝy -, -D“pCÝyIÁo -, -ªd - -D“pCÝyObjeù - -m‘hod - -im¶em’tiÚs -. - -48 - $$ -( -CONTROLLER_GEN -è -objeù -: -h—d”Fže -="hack/bož”¶©e.go.txt" -·ths -="./..." - -50 -fmt -: ## -Run - -go - fmˆ -agaš¡ - -code -. - -51 -go - -fmt - ./... - -53 -v‘ -: ## -Run - -go - v‘ -agaš¡ - -code -. - -54 -go - -v‘ - ./... - -56 -TEST - ?ð./ -­is -/ -d©aba£ -/ -v1®pha1 - ./ -commÚs -/... ./ -cÚŒÞËrs -/... - -57 -‹¡ -: -mªiã¡s - -g’”©e - -fmt - -v‘ - -’v‹¡ - ## -Run - -un™ - -‹¡s -. - -58 -KUBEBUILDER_ASSETS -="$(sh–È$(ENVTESTèu£ $(ENVTEST_K8S_VERSIONè-°·th)" -go - -‹¡ - - `$ -( -TEST -è- -cov”´ofže - -cov” -. -out - - -60 -E2ETEST - ?ð./ -‹¡ -/ -e2e -/ - -61 -e2e -: -mªiã¡s - -g’”©e - -fmt - -v‘ - -’v‹¡ - ## -Run -ƒ2 -‹¡s -. - -62 -KUBEBUILDER_ASSETS -="$(sh–È$(ENVTESTèu£ $(ENVTEST_K8S_VERSIONè-°·th)" -gškgo - - -v - -- -timeout -=2 -h30m - -- -çž -- -ç¡ - - $$ -( -E2ETEST -) - -64 ##@ -Bužd - - -66 -bužd -: -g’”©e - -fmt - -v‘ - ## -Bužd - -mªag” - -bš¬y -. - -67 -go - -bužd - - -o - -bš -/ -mªag” - -maš -.go - -69 -run -: -mªiã¡s - -g’”©e - -fmt - -v‘ - ## -Run - -a - -cÚŒÞËr - -äom - -your - -ho¡ -. - -70 -go - -run - ./ -maš -.go - -72 -dock” -- -bužd -: -mªiã¡s - -g’”©e - -fmt - -v‘ - #‹¡ ## -Bužd - dock” -image - -w™h - -the - -mªag” -. -Di§bË -h -‹¡ - -but - -k“p -h -v®id©iÚs - -to - -çž - -ç¡ - - -73 -dock” - -bužd - -- -no -- -ÿche -= -Œue - --bužd- -¬g - -h‰p_´oxy -= -$ -{ -HTTP_PROXY -} --bužd-¬g h‰ps_´oxy=${HTTPS_PROXY} . -ˆ${IMG} - -75 #dock”- -bužd -- -´oxy -: -‹¡ - - -76 #dock” -bužd - --bužd- -¬g - -h‰p_´oxy -= -$ -{h‰p_´oxy} --bužd-¬g -h‰ps_´oxy -=${h‰ps_´oxy} bužd . - -t - ${ -IMG -} - -78 -dock” -- -push -: ## -Push - dock” -image - -w™h - -the - -mªag” -. - -79 -dock” - -push - -$ -{ -IMG -} - -81 ##@ -D•loym’t - - -83 -š¡®l -: -mªiã¡s - -ku¡omize - ## -In¡®l - -CRDs - -što - -the - -K8s - -þu¡” - -¥ecif›d - -š - ~/. -kube -/ -cÚfig -. - -84 - $$ -( -KUSTOMIZE -è -bužd - -cÚfig -/ -üd - | -kubeùl - -­¶y - - -f - - - -86 -unš¡®l -: -mªiã¡s - -ku¡omize - ## -Unš¡®l - -CRDs - -äom - -the - -K8s - -þu¡” - -¥ecif›d - -š - ~/. -kube -/ -cÚfig -. - -87 - $$ -( -KUSTOMIZE -è -bužd - -cÚfig -/ -üd - | -kubeùl - -d–‘e - - -f - - - -89 -d•loy -: -mªiã¡s - -ku¡omize - ## -D•loy - -cÚŒÞËr - -to - -the - -K8s - -þu¡” - -¥ecif›d - -š - ~/. -kube -/ -cÚfig -. - -90 -cd - -cÚfig -/ -mªag” - && - $$ -( -KUSTOMIZE -è -ed™ - -£t - -image - -cÚŒÞËr -= -$ -{ -IMG -} - -91 - $$ -( -KUSTOMIZE -è -bužd - -cÚfig -/ | -kubeùl - -­¶y - - -f - - - -94 #U£d -£d - -to - -»pos™iÚ - -the - -cÚŒÞËr -- -mªag” - -D•loym’t - -aá” -h -û¹ifiÿ‹ - -ü—tiÚ - -š -h -OPERATOR_YAML - - -95 -Ý”©Ü -- -yaml -: -mªiã¡s - -ku¡omize - - -96 -cd - -cÚfig -/ -mªag” - && - $$ -( -KUSTOMIZE -è -ed™ - -£t - -image - -cÚŒÞËr -= -$ -{ -IMG -} - -97 - $$ -( -KUSTOMIZE -è -bužd - -cÚfig -/ > "${OPERATOR_YAML}" - -98 -£d - - -i -. -bak - - -e - '/^apiVersion:‡pps\/v1/,/---/d' "${OPERATOR_YAML}" - -99 ( -echo - --- && -£d - '/^apiVersion:‡pps\/v1/,/---/!d' "${OPERATOR_YAML}.bak") >> "${OPERATOR_YAML}" - -100 -rm - "${OPERATOR_YAML}.bak" - -102 -und•loy -: ## -Und•loy - -cÚŒÞËr - -äom - -the - -K8s - -þu¡” - -¥ecif›d - -š - ~/. -kube -/ -cÚfig -. - -103 - $$ -( -KUSTOMIZE -è -bužd - -cÚfig -/ | -kubeùl - -d–‘e - - -f - - - -105 ##@ -Bužd - -D•’d’c›s - - -107 ## -LoÿtiÚ - -to - -š¡®l - -d•’d’c›s -o - -108 -LOCALBIN - ?ð - `$ -( -sh–l - -pwd -)/ -bš - - -109 - $$ -( -LOCALBIN -): - -110 -mkdœ - - -p - - $$ -( -LOCALBIN -) - -112 ## -ToÞ - -Bš¬›s - - -113 -KUSTOMIZE - ?ð - `$ -( -LOCALBIN -)/ -ku¡omize - - -114 -CONTROLLER_GEN - ?ð - `$ -( -LOCALBIN -)/ -cÚŒÞËr -- -g’ - - -115 -ENVTEST - ?ð - `$ -( -LOCALBIN -)/ -£tup -- -’v‹¡ - - -117 ## -ToÞ - -V”siÚs - - -118 -KUSTOMIZE_VERSION - ?ð -v3 -.8.7 - -119 -CONTROLLER_TOOLS_VERSION - ?ð -v0 -.6.1 - -121 -KUSTOMIZE_INSTALL_SCRIPT - ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh" - -122 . -PHONY -: -ku¡omize - - -123 -ku¡omize -: - $$ -( -KUSTOMIZE -è## -DowÆßd - -ku¡omize - -loÿÎy -  -Ãûs§ry -. - -124 - $$ -( -KUSTOMIZE -): - $$ -( -LOCALBIN -) - -125 -cu¾ - - -s - - `$ -( -KUSTOMIZE_INSTALL_SCRIPT -è| -bash - - -- $( -sub¡ - -v -,,$( -KUSTOMIZE_VERSION -)è$( -LOCALBIN -) - -127 . -PHONY -: -cÚŒÞËr -- -g’ - - -128 -cÚŒÞËr -- -g’ -: - $$ -( -CONTROLLER_GEN -è## -DowÆßd - -cÚŒÞËr -- -g’ - -loÿÎy -  -Ãûs§ry -. - -129 - $$ -( -CONTROLLER_GEN -): - $$ -( -LOCALBIN -) - -130 -GOBIN -= - $$ -( -LOCALBIN -è -go - -š¡®l - -sigs -. -k8s -. -io -/ -cÚŒÞËr -- -toÞs -/ -cmd -/cÚŒÞËr- -g’ -@ - `$ -( -CONTROLLER_TOOLS_VERSION -) - -132 . -PHONY -: -’v‹¡ - - -133 -’v‹¡ -: - $$ -( -ENVTEST -è## -DowÆßd - -’v‹¡ -- -£tup - -loÿÎy -  -Ãûs§ry -. - -134 - $$ -( -ENVTEST -): - $$ -( -LOCALBIN -) - -135 -GOBIN -= - $$ -( -LOCALBIN -è -go - -š¡®l - -sigs -. -k8s -. -io -/ -cÚŒÞËr -- -ruÁime -/ -toÞs -/ -£tup -- -’v‹¡ -@ -Ï‹¡ - - -138 . -PHONY -: -bundË - - -139 -bundË -: -mªiã¡s - -ku¡omize - ## -G’”©e - bundË mªiã¡  -ªd - -m‘ad©a -, -th’ - -v®id©e - -g’”©ed - -fžes -. - -140 -Ý”©Ü -- -sdk - -g’”©e - -ku¡omize - -mªiã¡s - - -q - - -141 -cd - -cÚfig -/ -mªag” - && - $$ -( -KUSTOMIZE -è -ed™ - -£t - -image - -cÚŒÞËr -= - `$ -( -IMG -) - -142 - $$ -( -KUSTOMIZE -è -bužd - -cÚfig -/ -mªiã¡s - | -Ý”©Ü -- -sdk - -g’”©e - -bundË - - -q - -- -ov”wr™e - -- -v”siÚ - - `$ -( -VERSION -è - $$ -( -BUNDLE_METADATA_OPTS -) - -143 -Ý”©Ü -- -sdk - -bundË - -v®id©e - ./bundle - -145 . -PHONY -: -bundË -- -bužd - - -146 -bundË -- -bužd -: ## -Bužd - -the - bundË -image -. - -147 -dock” - -bužd - - -f - -bundË -. -Dock”fže - - -t - - `$ -( -BUNDLE_IMG -) . - -149 . -PHONY -: -bundË -- -push - - -150 -bundË -- -push -: ## -Push - -the - bundË -image -. - -151 - $$ -( -MAKE -è -dock” -- -push - -IMG -= - `$ -( -BUNDLE_IMG -) - -153 . -PHONY -: -Ým - - -154 -OPM - = ./ -bš -/ -Ým - - -155 -Ým -: ## -DowÆßd - opm -loÿÎy -  -Ãûs§ry -. - -156 - `iãq - (, - `$ -( -wždÿrd - - $$ -( -OPM -))) - -157 - `iãq - (, - `$ -( -sh–l - -which - -Ým - 2>/ -dev -/ -nuÎ -)) - -159 -£t - - -e - ;\ - -160 -mkdœ - - -p - - `$ -( -dœ - $( -OPM -)) ;\ - -161 -OS -= - `$ -( -sh–l - -go - -’v - -GOOS -è&& -ARCH -=$(sh–ÈgØ’v -GOARCH -) && \ - -162 -cu¾ - - -sSLo - - `$ -( -OPM -è -h‰ps -: - -163 -chmod - + -x - - `$ -( -OPM -) ;\ - -164 - } -} - -166 - gOPM - = - $$ -( -sh–l - -which - -Ým -) - -167 -’dif - - -168 -’dif - - -170 #A -comma -- -£·¿‹d - -li¡ - -of - -bundË - - `images - ( -e -. -g -. -make - -ÿlog -- -bužd - -BUNDLE_IMGS -= -exam¶e -. -com -/ -Ý”©Ü --bundË: -v0 -.1.0,example.com/operator-bundle:v0.2.0). - -171 #The£ -images - -MUST - -exi¡ - -š - -a - -»gi¡ry - -ªd - -be - -puÎ -- -abË -. - -172 -BUNDLE_IMGS - ?ð - $$ -( -BUNDLE_IMG -) - -174 #Th -image - -g - -giv’ - -to - -the - -»suÉšg - -ÿlog - - `image - ( -e -. -g -. -make - c©®og- -bužd - -CATALOG_IMG -= -exam¶e -. -com -/ -Ý”©Ü --ÿlog: -v0 -.2.0). - -175 -CATALOG_IMG - ?ð - `$ -( -IMAGE_TAG_BASE -)- -ÿlog -: - $v$ -( -VERSION -) - -177 #S‘ -CATALOG_BASE_IMG - -to - -ª - -exi¡šg - -ÿlog - -image - -g -Ø -add - -$BUNDLE_IMGS -Ø -th© - image. - -178 - `iâeq - ( - `$ -( -Üigš - -CATALOG_BASE_IMG -), -undefšed -) - -179 -FROM_INDEX_OPT - :ð-- -äom -- -šdex - - $$ -( -CATALOG_BASE_IMG -) - -180 -’dif - - -182 #Bužd -a - -ÿlog - -image - -by - -addšg - -bundË - -images - -to - -ª - -em±y - c©®og -usšg - -the - -Ý”©Ü - -·ckage - -mªag” - -toÞ -, 'opm'. - -183 #Thi  -»ce - -švokes - 'Ým' -š - '£mv”' -bundË - -add - -mode -. -FÜ - -mÜe - -šfÜm©iÚ - -Ú -‡dd -modes -, -£e -: - -185 . -PHONY -: -ÿlog -- -bužd - - -186 -ÿlog -- -bužd -: -Ým - ## -Bužd - -a - c©®og -image -. - -187 - $$ -( -OPM -è -šdex - -add - -- -cÚš” -- -toÞ - -dock” - -- -mode - -£mv” - -- -g - - `$ -( -CATALOG_IMG -è-- -bundËs - $( -BUNDLE_IMGS -è - $$ -( -FROM_INDEX_OPT -) - -189 #Push -the - -ÿlog - -image -. - -190 . -PHONY -: -ÿlog -- -push - - -191 -ÿlog -- -push -: ## -Push - -a - c©®og -image -. - -192 - $$ -( -MAKE -è -dock” -- -push - -IMG -= - `$ -( -CATALOG_IMG -) - - @ -1 -. -1 -/usr/include -1 -9 -Makefile diff --git a/cscope.tmplst b/cscope.tmplst deleted file mode 100644 index 0937d485..00000000 --- a/cscope.tmplst +++ /dev/null @@ -1 +0,0 @@ -./Makefile diff --git a/docs/multitenant/provisioning/add_replica.yaml b/docs/multitenant/provisioning/add_replica.yaml index 2bc54495..fac2d7ba 100644 --- a/docs/multitenant/provisioning/add_replica.yaml +++ b/docs/multitenant/provisioning/add_replica.yaml @@ -9,7 +9,6 @@ metadata: namespace: oracle-database-operator-system spec: cdbName: "goldcdb" - scanName: "goldhost-scan.lbsub52b3b1cae.okecluster.oraclevcn.com" dbServer: "goldhost1.lbsub52b3b1cae.okecluster.oraclevcn.com" ordsImage: phx.ocir.io//oracle/ords:21.4.3 dbPort: 1521 diff --git a/docs/multitenant/provisioning/cdb.yaml b/docs/multitenant/provisioning/cdb.yaml index 4282040e..8e25a763 100644 --- a/docs/multitenant/provisioning/cdb.yaml +++ b/docs/multitenant/provisioning/cdb.yaml @@ -9,7 +9,6 @@ metadata: namespace: oracle-database-operator-system spec: cdbName: "goldcdb" - scanName: "goldhost-scan.lbsub52b3b1cae.okecluster.oraclevcn.com" dbServer: "goldhost1.lbsub52b3b1cae.okecluster.oraclevcn.com" ordsImage: phx.ocir.io//oracle/ords:21.4.3 ordsImagePullSecret: "container-registry-secret" diff --git a/oracle-database-operator.yaml b/oracle-database-operator.yaml index 9180474e..95786f0b 100644 --- a/oracle-database-operator.yaml +++ b/oracle-database-operator.yaml @@ -619,10 +619,6 @@ spec: jsonPath: .spec.dbPort name: DB Port type: integer - - description: SCAN Name - jsonPath: .spec.scanName - name: SCAN Name - type: string - description: Replicas jsonPath: .spec.replicas name: Replicas @@ -651,8 +647,6 @@ spec: spec: description: CDBSpec defines the desired state of CDB properties: - TestVariable: - type: string cdbAdminPwd: description: Password for the CDB Administrator to manage PDB lifecycle properties: @@ -768,9 +762,6 @@ spec: replicas: description: Number of ORDS Containers to create type: integer - scanName: - description: SCAN Name - type: string serviceName: description: Name of the CDB Service type: string @@ -2031,10 +2022,17 @@ spec: - jsonPath: .status.connectString name: Connect Str type: string + - jsonPath: .status.tcpsConnectString + name: TCPS Connect Str + type: string - jsonPath: .status.pdbConnectString name: Pdb Connect Str priority: 1 type: string + - jsonPath: .status.tcpsPdbConnectString + name: TCPS Pdb Connect Str + priority: 1 + type: string - jsonPath: .status.oemExpressUrl name: Oem Express Url type: string @@ -2079,6 +2077,8 @@ spec: - enterprise - express type: string + enableTCPS: + type: boolean flashBack: type: boolean forceLog: @@ -2109,6 +2109,8 @@ spec: sgaTarget: type: integer type: object + listenerPort: + type: integer loadBalancer: type: boolean nodeSelector: @@ -2149,6 +2151,10 @@ spec: maxLength: 12 pattern: ^[a-zA-Z0-9]+$ type: string + tcpsCertRenewInterval: + type: string + tcpsListenerPort: + type: integer required: - image type: object @@ -2159,8 +2165,14 @@ spec: type: boolean archiveLog: type: string + certCreationTimestamp: + type: string + certRenewInterval: + type: string charset: type: string + clientWalletLoc: + type: string cloneFrom: type: string clusterConnectString: @@ -2241,6 +2253,9 @@ spec: type: integer initSgaSize: type: integer + isTcpsEnabled: + default: false + type: boolean nodes: items: type: string @@ -2286,7 +2301,12 @@ spec: type: object status: type: string + tcpsConnectString: + type: string + tcpsPdbConnectString: + type: string required: + - isTcpsEnabled - persistence type: object type: object @@ -3198,7 +3218,7 @@ spec: - --enable-leader-election command: - /manager - image: container-registry.oracle.com/database/operator:0.2.0 + image: lin.ocir.io/intsanjaysingh/mmalvezz/testppr/myoperator:latest imagePullPolicy: Always name: manager ports: From b4365d002eeef2244f6dbab9ca9421a5014ce675 Mon Sep 17 00:00:00 2001 From: mmalvezz Date: Wed, 19 Oct 2022 17:50:48 +0200 Subject: [PATCH 33/37] bug 34699326 + usecase01 --- apis/database/v1alpha1/cdb_types.go | 2 + apis/database/v1alpha1/cdb_webhook.go | 16 +- controllers/DBserver | 33 -- controllers/database/cdb_controller.go | 4 + controllers/database/pdb_controller.go | 36 +- docs/multitenant/usecase01/BuildImage.log | 487 +++++++++++++++ docs/multitenant/usecase01/ImagePush.log | 11 + docs/multitenant/usecase01/README.md | 555 ++++++++++++++++++ docs/multitenant/usecase01/cdb.log | 372 ++++++++++++ docs/multitenant/usecase01/cdb.yaml | 46 ++ .../usecase01/openssl_execution.log | 22 + docs/multitenant/usecase01/ordsconfig.log | 35 ++ docs/multitenant/usecase01/sync.sh | 5 + docs/multitenant/usecase01/testapi.log | 49 ++ oracle-database-operator.yaml | 6 + ords/runOrdsSSL.sh | 32 +- 16 files changed, 1662 insertions(+), 49 deletions(-) delete mode 100644 controllers/DBserver create mode 100644 docs/multitenant/usecase01/BuildImage.log create mode 100644 docs/multitenant/usecase01/ImagePush.log create mode 100644 docs/multitenant/usecase01/README.md create mode 100644 docs/multitenant/usecase01/cdb.log create mode 100644 docs/multitenant/usecase01/cdb.yaml create mode 100644 docs/multitenant/usecase01/openssl_execution.log create mode 100644 docs/multitenant/usecase01/ordsconfig.log create mode 100644 docs/multitenant/usecase01/sync.sh create mode 100644 docs/multitenant/usecase01/testapi.log diff --git a/apis/database/v1alpha1/cdb_types.go b/apis/database/v1alpha1/cdb_types.go index a987718a..b5f39707 100644 --- a/apis/database/v1alpha1/cdb_types.go +++ b/apis/database/v1alpha1/cdb_types.go @@ -86,6 +86,7 @@ type CDBSpec struct { DBPort int `json:"dbPort,omitempty"` // Node Selector for running the Pod NodeSelector map[string]string `json:"nodeSelector,omitempty"` + DBTnsurl string `json:"dbTnsurl,omitempty"` } // CDBSecret defines the secretName @@ -150,6 +151,7 @@ type CDBStatus struct { // +kubebuilder:printcolumn:JSONPath=".spec.cdbName",name="CDB Name",type="string",description="Name of the CDB" // +kubebuilder:printcolumn:JSONPath=".spec.dbServer",name="DB Server",type="string",description=" Name of the DB Server" // +kubebuilder:printcolumn:JSONPath=".spec.dbPort",name="DB Port",type="integer",description="DB server port" +// +kubebuilder:printcolumn:JSONPath=".spec.dbTnsurl",name="TNS STRING",type="string",description=" string of the tnsalias" // +kubebuilder:printcolumn:JSONPath=".spec.replicas",name="Replicas",type="integer",description="Replicas" // +kubebuilder:printcolumn:JSONPath=".status.phase",name="Status",type="string",description="Status of the CDB Resource" // +kubebuilder:printcolumn:JSONPath=".status.msg",name="Message",type="string",description="Error message, if any" diff --git a/apis/database/v1alpha1/cdb_webhook.go b/apis/database/v1alpha1/cdb_webhook.go index c760875c..25975092 100644 --- a/apis/database/v1alpha1/cdb_webhook.go +++ b/apis/database/v1alpha1/cdb_webhook.go @@ -88,7 +88,7 @@ func (r *CDB) ValidateCreate() error { var allErrs field.ErrorList - if r.Spec.ServiceName == "" { + if r.Spec.ServiceName == "" && r.Spec.DBServer != "" { allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("serviceName"), "Please specify CDB Service name")) } @@ -108,15 +108,21 @@ func (r *CDB) ValidateCreate() error { field.Required(field.NewPath("spec").Child("scanName"), "Please specify SCAN Name for CDB")) }*/ - if r.Spec.DBServer == "" { + if ((r.Spec.DBServer == "" && r.Spec.DBTnsurl == "") || (r.Spec.DBServer != "" && r.Spec.DBTnsurl != "")) { allErrs = append(allErrs, - field.Required(field.NewPath("spec").Child("dbServer"), "Please specify Database Server Name or IP Address")) + field.Required(field.NewPath("spec").Child("dbServer"), "Please specify Database Server Name/IP Address or tnsalias string")) } - if r.Spec.DBPort == 0 { + + if r.Spec.DBTnsurl != "" && ( r.Spec.DBServer != "" || r.Spec.DBPort != 0 || r.Spec.ServiceName != "" ) { + allErrs = append(allErrs, + field.Required(field.NewPath("spec").Child("dbServer"), "DBtnsurl is orthogonal to (DBServer,DBport,Services)")) + } + + if r.Spec.DBPort == 0 && r.Spec.DBServer != "" { allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("dbPort"), "Please specify DB Server Port")) } - if r.Spec.DBPort < 0 { + if r.Spec.DBPort < 0 && r.Spec.DBServer != "" { allErrs = append(allErrs, field.Required(field.NewPath("spec").Child("dbPort"), "Please specify a valid DB Server Port")) } diff --git a/controllers/DBserver b/controllers/DBserver deleted file mode 100644 index 57c0f4fb..00000000 --- a/controllers/DBserver +++ /dev/null @@ -1,33 +0,0 @@ -# This viminfo file was generated by Vim 7.4. -# You may edit it if you're careful! - -# Value of 'encoding' when this file was written -*encoding=utf-8 - - -# hlsearch on (H) or off (h): -~H -# Command Line History (newest to oldest): -:q! -:q - -# Search String History (newest to oldest): - -# Expression History (newest to oldest): - -# Input Line History (newest to oldest): - -# Input Line History (newest to oldest): - -# Registers: - -# File marks: -'0 1 0 ~/WORKDIR/Ords22_operator/oracle-database-operator/controllers/grep - -# Jumplist (newest first): --' 1 0 ~/WORKDIR/Ords22_operator/oracle-database-operator/controllers/grep - -# History of marks within files (newest to oldest): - -> ~/WORKDIR/Ords22_operator/oracle-database-operator/controllers/grep - " 1 0 diff --git a/controllers/database/cdb_controller.go b/controllers/database/cdb_controller.go index 02c04142..58eebf51 100644 --- a/controllers/database/cdb_controller.go +++ b/controllers/database/cdb_controller.go @@ -449,6 +449,10 @@ func (r *CDBReconciler) createPodSpec(cdb *dbapi.CDB) corev1.PodSpec { Name: "ORACLE_HOST", Value: cdb.Spec.DBServer, }, + { + Name: "DBTNSURL", + Value: cdb.Spec.DBTnsurl, + }, { Name: "TLSCRT", Value: cdb.Spec.CDBTlsCrt.Secret.Key, diff --git a/controllers/database/pdb_controller.go b/controllers/database/pdb_controller.go index 07580725..f9231a66 100644 --- a/controllers/database/pdb_controller.go +++ b/controllers/database/pdb_controller.go @@ -641,7 +641,12 @@ func (r *PDBReconciler) createPDB(ctx context.Context, req ctrl.Request, pdb *db r.Recorder.Eventf(pdb, corev1.EventTypeNormal, "Created", "PDB '%s' created successfully", pdb.Spec.PDBName) + if cdb.Spec.DBServer != "" { pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + } else { + pdb.Status.ConnString = cdb.Spec.DBTnsurl + } + log.Info("Created PDB Resource", "PDB Name", pdb.Spec.PDBName) r.getPDBState(ctx, req, pdb) return nil @@ -695,7 +700,13 @@ func (r *PDBReconciler) clonePDB(ctx context.Context, req ctrl.Request, pdb *dba r.Recorder.Eventf(pdb, corev1.EventTypeNormal, "Created", "PDB '%s' cloned successfully", pdb.Spec.PDBName) - pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + if cdb.Spec.DBServer != "" { + pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + } else { + pdb.Status.ConnString = cdb.Spec.DBTnsurl + } + + log.Info("Cloned PDB successfully", "Source PDB Name", pdb.Spec.SrcPDBName, "Clone PDB Name", pdb.Spec.PDBName) r.getPDBState(ctx, req, pdb) return nil @@ -763,7 +774,12 @@ func (r *PDBReconciler) plugPDB(ctx context.Context, req ctrl.Request, pdb *dbap r.Recorder.Eventf(pdb, corev1.EventTypeNormal, "Created", "PDB '%s' plugged successfully", pdb.Spec.PDBName) - pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + if cdb.Spec.DBServer != "" { + pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + } else { + pdb.Status.ConnString = cdb.Spec.DBTnsurl + } + log.Info("Successfully plugged PDB", "PDB Name", pdb.Spec.PDBName) r.getPDBState(ctx, req, pdb) return nil @@ -883,7 +899,13 @@ func (r *PDBReconciler) modifyPDB(ctx context.Context, req ctrl.Request, pdb *db } r.Recorder.Eventf(pdb, corev1.EventTypeNormal, "Modified", "PDB '%s' modified successfully", pdb.Spec.PDBName) - pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + + if cdb.Spec.DBServer != "" { + pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + } else { + pdb.Status.ConnString = cdb.Spec.DBTnsurl + } + log.Info("Successfully modified PDB state", "PDB Name", pdb.Spec.PDBName) r.getPDBState(ctx, req, pdb) @@ -970,7 +992,13 @@ func (r *PDBReconciler) mapPDB(ctx context.Context, req ctrl.Request, pdb *dbapi pdb.Status.OpenMode = objmap["open_mode"].(string) pdb.Status.TotalSize = fmt.Sprintf("%.2f", totSizeInGB) + "G" - pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + + if cdb.Spec.DBServer != "" { + pdb.Status.ConnString = cdb.Spec.DBServer + ":" + strconv.Itoa(cdb.Spec.DBPort) + "/" + pdb.Spec.PDBName + } else { + pdb.Status.ConnString = cdb.Spec.DBTnsurl + } + log.Info("Successfully mapped PDB to Kubernetes resource", "PDB Name", pdb.Spec.PDBName) return nil diff --git a/docs/multitenant/usecase01/BuildImage.log b/docs/multitenant/usecase01/BuildImage.log new file mode 100644 index 00000000..4ee2fa05 --- /dev/null +++ b/docs/multitenant/usecase01/BuildImage.log @@ -0,0 +1,487 @@ +/usr/bin/docker build -t oracle/ords-dboper:latest . +Sending build context to Docker daemon 92.38MB +Step 1/10 : FROM container-registry.oracle.com/java/jdk:latest +Trying to pull repository container-registry.oracle.com/java/jdk ... +latest: Pulling from container-registry.oracle.com/java/jdk +7cb069903b8a: Pull complete +a98ca67f4239: Pull complete +1b4060d1d804: Pull complete +Digest: sha256:8e7161bbd6a3a3beb77ee6f2d80c17ae4c80d88e0f5af667a19a0271c33f1b5e +Status: Downloaded newer image for container-registry.oracle.com/java/jdk:latest + ---> ad9ff1bbe92a +Step 2/10 : ENV ORDS_HOME=/opt/oracle/ords/ RUN_FILE="runOrdsSSL.sh" + ---> Running in e6f76deab66e +Removing intermediate container e6f76deab66e + ---> 0b26c489e4fd +Step 3/10 : COPY $RUN_FILE $ORDS_HOME + ---> ee472155adab +Step 4/10 : RUN yum -y install yum-utils bind-utils tree hostname openssl net-tools zip unzip tar wget vim-minimal which sudo expect procps && yum-config-manager --add-repo=http://yum.oracle.com/repo/OracleLinux/OL8/oracle/software/x86_64 && yum -y install java-11-openjdk-devel && yum -y install ords && yum -y install iproute && yum clean all + ---> Running in d38a69d2cc70 +Oracle Linux 8 BaseOS Latest (x86_64) 105 MB/s | 50 MB 00:00 +Oracle Linux 8 Application Stream (x86_64) 90 MB/s | 38 MB 00:00 +Last metadata expiration check: 0:00:07 ago on Mon 10 Oct 2022 04:06:15 PM UTC. +Package yum-utils-4.0.21-11.0.1.el8.noarch is already installed. +Package tar-2:1.30-5.el8.x86_64 is already installed. +Package vim-minimal-2:8.0.1763-19.0.1.el8_6.4.x86_64 is already installed. +Package procps-ng-3.3.15-6.0.1.el8.x86_64 is already installed. +Dependencies resolved. +================================================================================ + Package Arch Version Repository Size +================================================================================ +Installing: + bind-utils x86_64 32:9.11.36-3.el8_6.1 ol8_appstream 452 k + expect x86_64 5.45.4-5.el8 ol8_baseos_latest 266 k + hostname x86_64 3.20-6.el8 ol8_baseos_latest 32 k + net-tools x86_64 2.0-0.52.20160912git.el8 ol8_baseos_latest 322 k + openssl x86_64 1:1.1.1k-7.el8_6 ol8_baseos_latest 709 k + sudo x86_64 1.8.29-8.el8 ol8_baseos_latest 925 k + tree x86_64 1.7.0-15.el8 ol8_baseos_latest 59 k + unzip x86_64 6.0-46.0.1.el8 ol8_baseos_latest 196 k + wget x86_64 1.19.5-10.0.1.el8 ol8_appstream 734 k + which x86_64 2.21-17.el8 ol8_baseos_latest 49 k + zip x86_64 3.0-23.el8 ol8_baseos_latest 270 k +Installing dependencies: + bind-libs x86_64 32:9.11.36-3.el8_6.1 ol8_appstream 175 k + bind-libs-lite x86_64 32:9.11.36-3.el8_6.1 ol8_appstream 1.2 M + bind-license noarch 32:9.11.36-3.el8_6.1 ol8_appstream 103 k + fstrm x86_64 0.6.1-2.el8 ol8_appstream 29 k + libmaxminddb x86_64 1.2.0-10.el8 ol8_appstream 33 k + libmetalink x86_64 0.1.3-7.el8 ol8_baseos_latest 32 k + protobuf-c x86_64 1.3.0-6.el8 ol8_appstream 37 k + python3-bind noarch 32:9.11.36-3.el8_6.1 ol8_appstream 150 k + python3-ply noarch 3.9-9.el8 ol8_baseos_latest 111 k + tcl x86_64 1:8.6.8-2.el8 ol8_baseos_latest 1.1 M + +Transaction Summary +================================================================================ +Install 21 Packages + +Total download size: 6.9 M +Installed size: 20 M +Downloading Packages: +(1/21): hostname-3.20-6.el8.x86_64.rpm 555 kB/s | 32 kB 00:00 +(2/21): libmetalink-0.1.3-7.el8.x86_64.rpm 492 kB/s | 32 kB 00:00 +(3/21): expect-5.45.4-5.el8.x86_64.rpm 3.2 MB/s | 266 kB 00:00 +(4/21): python3-ply-3.9-9.el8.noarch.rpm 5.5 MB/s | 111 kB 00:00 +(5/21): net-tools-2.0-0.52.20160912git.el8.x86_ 6.7 MB/s | 322 kB 00:00 +(6/21): openssl-1.1.1k-7.el8_6.x86_64.rpm 12 MB/s | 709 kB 00:00 +(7/21): tree-1.7.0-15.el8.x86_64.rpm 4.1 MB/s | 59 kB 00:00 +(8/21): sudo-1.8.29-8.el8.x86_64.rpm 19 MB/s | 925 kB 00:00 +(9/21): which-2.21-17.el8.x86_64.rpm 2.5 MB/s | 49 kB 00:00 +(10/21): unzip-6.0-46.0.1.el8.x86_64.rpm 5.9 MB/s | 196 kB 00:00 +(11/21): tcl-8.6.8-2.el8.x86_64.rpm 15 MB/s | 1.1 MB 00:00 +(12/21): zip-3.0-23.el8.x86_64.rpm 15 MB/s | 270 kB 00:00 +(13/21): bind-libs-9.11.36-3.el8_6.1.x86_64.rpm 7.9 MB/s | 175 kB 00:00 +(14/21): bind-license-9.11.36-3.el8_6.1.noarch. 4.9 MB/s | 103 kB 00:00 +(15/21): bind-utils-9.11.36-3.el8_6.1.x86_64.rp 21 MB/s | 452 kB 00:00 +(16/21): bind-libs-lite-9.11.36-3.el8_6.1.x86_6 28 MB/s | 1.2 MB 00:00 +(17/21): libmaxminddb-1.2.0-10.el8.x86_64.rpm 1.8 MB/s | 33 kB 00:00 +(18/21): fstrm-0.6.1-2.el8.x86_64.rpm 1.0 MB/s | 29 kB 00:00 +(19/21): protobuf-c-1.3.0-6.el8.x86_64.rpm 1.4 MB/s | 37 kB 00:00 +(20/21): python3-bind-9.11.36-3.el8_6.1.noarch. 9.2 MB/s | 150 kB 00:00 +(21/21): wget-1.19.5-10.0.1.el8.x86_64.rpm 7.5 MB/s | 734 kB 00:00 +-------------------------------------------------------------------------------- +Total 20 MB/s | 6.9 MB 00:00 +Running transaction check +Transaction check succeeded. +Running transaction test +Transaction test succeeded. +Running transaction + Preparing : 1/1 + Installing : protobuf-c-1.3.0-6.el8.x86_64 1/21 + Installing : libmaxminddb-1.2.0-10.el8.x86_64 2/21 + Running scriptlet: libmaxminddb-1.2.0-10.el8.x86_64 2/21 + Installing : fstrm-0.6.1-2.el8.x86_64 3/21 + Installing : bind-license-32:9.11.36-3.el8_6.1.noarch 4/21 + Installing : bind-libs-lite-32:9.11.36-3.el8_6.1.x86_64 5/21 + Installing : bind-libs-32:9.11.36-3.el8_6.1.x86_64 6/21 + Installing : unzip-6.0-46.0.1.el8.x86_64 7/21 + Installing : tcl-1:8.6.8-2.el8.x86_64 8/21 + Running scriptlet: tcl-1:8.6.8-2.el8.x86_64 8/21 + Installing : python3-ply-3.9-9.el8.noarch 9/21 + Installing : python3-bind-32:9.11.36-3.el8_6.1.noarch 10/21 + Installing : libmetalink-0.1.3-7.el8.x86_64 11/21 + Installing : wget-1.19.5-10.0.1.el8.x86_64 12/21 + Running scriptlet: wget-1.19.5-10.0.1.el8.x86_64 12/21 + Installing : bind-utils-32:9.11.36-3.el8_6.1.x86_64 13/21 + Installing : expect-5.45.4-5.el8.x86_64 14/21 + Installing : zip-3.0-23.el8.x86_64 15/21 + Installing : which-2.21-17.el8.x86_64 16/21 + Installing : tree-1.7.0-15.el8.x86_64 17/21 + Installing : sudo-1.8.29-8.el8.x86_64 18/21 + Running scriptlet: sudo-1.8.29-8.el8.x86_64 18/21 + Installing : openssl-1:1.1.1k-7.el8_6.x86_64 19/21 + Installing : net-tools-2.0-0.52.20160912git.el8.x86_64 20/21 + Running scriptlet: net-tools-2.0-0.52.20160912git.el8.x86_64 20/21 + Installing : hostname-3.20-6.el8.x86_64 21/21 + Running scriptlet: hostname-3.20-6.el8.x86_64 21/21 + Verifying : expect-5.45.4-5.el8.x86_64 1/21 + Verifying : hostname-3.20-6.el8.x86_64 2/21 + Verifying : libmetalink-0.1.3-7.el8.x86_64 3/21 + Verifying : net-tools-2.0-0.52.20160912git.el8.x86_64 4/21 + Verifying : openssl-1:1.1.1k-7.el8_6.x86_64 5/21 + Verifying : python3-ply-3.9-9.el8.noarch 6/21 + Verifying : sudo-1.8.29-8.el8.x86_64 7/21 + Verifying : tcl-1:8.6.8-2.el8.x86_64 8/21 + Verifying : tree-1.7.0-15.el8.x86_64 9/21 + Verifying : unzip-6.0-46.0.1.el8.x86_64 10/21 + Verifying : which-2.21-17.el8.x86_64 11/21 + Verifying : zip-3.0-23.el8.x86_64 12/21 + Verifying : bind-libs-32:9.11.36-3.el8_6.1.x86_64 13/21 + Verifying : bind-libs-lite-32:9.11.36-3.el8_6.1.x86_64 14/21 + Verifying : bind-license-32:9.11.36-3.el8_6.1.noarch 15/21 + Verifying : bind-utils-32:9.11.36-3.el8_6.1.x86_64 16/21 + Verifying : fstrm-0.6.1-2.el8.x86_64 17/21 + Verifying : libmaxminddb-1.2.0-10.el8.x86_64 18/21 + Verifying : protobuf-c-1.3.0-6.el8.x86_64 19/21 + Verifying : python3-bind-32:9.11.36-3.el8_6.1.noarch 20/21 + Verifying : wget-1.19.5-10.0.1.el8.x86_64 21/21 + +Installed: + bind-libs-32:9.11.36-3.el8_6.1.x86_64 + bind-libs-lite-32:9.11.36-3.el8_6.1.x86_64 + bind-license-32:9.11.36-3.el8_6.1.noarch + bind-utils-32:9.11.36-3.el8_6.1.x86_64 + expect-5.45.4-5.el8.x86_64 + fstrm-0.6.1-2.el8.x86_64 + hostname-3.20-6.el8.x86_64 + libmaxminddb-1.2.0-10.el8.x86_64 + libmetalink-0.1.3-7.el8.x86_64 + net-tools-2.0-0.52.20160912git.el8.x86_64 + openssl-1:1.1.1k-7.el8_6.x86_64 + protobuf-c-1.3.0-6.el8.x86_64 + python3-bind-32:9.11.36-3.el8_6.1.noarch + python3-ply-3.9-9.el8.noarch + sudo-1.8.29-8.el8.x86_64 + tcl-1:8.6.8-2.el8.x86_64 + tree-1.7.0-15.el8.x86_64 + unzip-6.0-46.0.1.el8.x86_64 + wget-1.19.5-10.0.1.el8.x86_64 + which-2.21-17.el8.x86_64 + zip-3.0-23.el8.x86_64 + +Complete! +Adding repo from: http://yum.oracle.com/repo/OracleLinux/OL8/oracle/software/x86_64 +created by dnf config-manager from http://yum.o 194 kB/s | 49 kB 00:00 +Dependencies resolved. +============================================================================================= + Package Arch Version Repository Size +============================================================================================= +Installing: + java-11-openjdk-devel x86_64 1:11.0.16.1.1-1.el8_6 ol8_appstream 3.4 M +Installing dependencies: + alsa-lib x86_64 1.2.6.1-3.el8 ol8_appstream 491 k + avahi-libs x86_64 0.7-20.el8 ol8_baseos_latest 62 k + copy-jdk-configs noarch 4.0-2.el8 ol8_appstream 30 k + crypto-policies-scripts noarch 20211116-1.gitae470d6.el8 ol8_baseos_latest 83 k + cups-libs x86_64 1:2.2.6-45.el8_6.2 ol8_baseos_latest 434 k + giflib x86_64 5.1.4-3.el8 ol8_appstream 51 k + graphite2 x86_64 1.3.10-10.el8 ol8_appstream 122 k + harfbuzz x86_64 1.7.5-3.el8 ol8_appstream 295 k + java-11-openjdk x86_64 1:11.0.16.1.1-1.el8_6 ol8_appstream 272 k + java-11-openjdk-headless x86_64 1:11.0.16.1.1-1.el8_6 ol8_appstream 40 M + javapackages-filesystem noarch 5.3.0-1.module+el8+5136+7ff78f74 ol8_appstream 30 k + lcms2 x86_64 2.9-2.el8 ol8_appstream 164 k + libX11 x86_64 1.6.8-5.el8 ol8_appstream 611 k + libX11-common noarch 1.6.8-5.el8 ol8_appstream 158 k + libXau x86_64 1.0.9-3.el8 ol8_appstream 37 k + libXcomposite x86_64 0.4.4-14.el8 ol8_appstream 28 k + libXext x86_64 1.3.4-1.el8 ol8_appstream 45 k + libXi x86_64 1.7.10-1.el8 ol8_appstream 49 k + libXrender x86_64 0.9.10-7.el8 ol8_appstream 33 k + libXtst x86_64 1.2.3-7.el8 ol8_appstream 22 k + libfontenc x86_64 1.1.3-8.el8 ol8_appstream 37 k + libjpeg-turbo x86_64 1.5.3-12.el8 ol8_appstream 157 k + libpkgconf x86_64 1.4.2-1.el8 ol8_baseos_latest 35 k + libxcb x86_64 1.13.1-1.el8 ol8_appstream 231 k + lksctp-tools x86_64 1.0.18-3.el8 ol8_baseos_latest 100 k + lua x86_64 5.3.4-12.el8 ol8_appstream 192 k + nspr x86_64 4.34.0-3.el8_6 ol8_appstream 143 k + nss x86_64 3.79.0-10.el8_6 ol8_appstream 747 k + nss-softokn x86_64 3.79.0-10.el8_6 ol8_appstream 1.2 M + nss-softokn-freebl x86_64 3.79.0-10.el8_6 ol8_appstream 398 k + nss-sysinit x86_64 3.79.0-10.el8_6 ol8_appstream 74 k + nss-util x86_64 3.79.0-10.el8_6 ol8_appstream 138 k + pkgconf x86_64 1.4.2-1.el8 ol8_baseos_latest 38 k + pkgconf-m4 noarch 1.4.2-1.el8 ol8_baseos_latest 17 k + pkgconf-pkg-config x86_64 1.4.2-1.el8 ol8_baseos_latest 15 k + ttmkfdir x86_64 3.0.9-54.el8 ol8_appstream 62 k + tzdata-java noarch 2022d-1.el8 ol8_appstream 186 k + xorg-x11-font-utils x86_64 1:7.5-41.el8 ol8_appstream 104 k + xorg-x11-fonts-Type1 noarch 7.5-19.el8 ol8_appstream 522 k +Enabling module streams: + javapackages-runtime 201801 + +Transaction Summary +============================================================================================= +Install 40 Packages + +Total download size: 50 M +Installed size: 196 M +Downloading Packages: +(1/40): crypto-policies-scripts-20211116-1.gita 1.3 MB/s | 83 kB 00:00 +(2/40): avahi-libs-0.7-20.el8.x86_64.rpm 879 kB/s | 62 kB 00:00 +(3/40): libpkgconf-1.4.2-1.el8.x86_64.rpm 2.0 MB/s | 35 kB 00:00 +(4/40): cups-libs-2.2.6-45.el8_6.2.x86_64.rpm 4.5 MB/s | 434 kB 00:00 +(5/40): lksctp-tools-1.0.18-3.el8.x86_64.rpm 3.7 MB/s | 100 kB 00:00 +(6/40): pkgconf-1.4.2-1.el8.x86_64.rpm 2.2 MB/s | 38 kB 00:00 +(7/40): pkgconf-m4-1.4.2-1.el8.noarch.rpm 1.2 MB/s | 17 kB 00:00 +(8/40): pkgconf-pkg-config-1.4.2-1.el8.x86_64.r 929 kB/s | 15 kB 00:00 +(9/40): copy-jdk-configs-4.0-2.el8.noarch.rpm 2.2 MB/s | 30 kB 00:00 +(10/40): giflib-5.1.4-3.el8.x86_64.rpm 3.3 MB/s | 51 kB 00:00 +(11/40): graphite2-1.3.10-10.el8.x86_64.rpm 7.7 MB/s | 122 kB 00:00 +(12/40): alsa-lib-1.2.6.1-3.el8.x86_64.rpm 12 MB/s | 491 kB 00:00 +(13/40): java-11-openjdk-11.0.16.1.1-1.el8_6.x8 14 MB/s | 272 kB 00:00 +(14/40): harfbuzz-1.7.5-3.el8.x86_64.rpm 8.7 MB/s | 295 kB 00:00 +(15/40): javapackages-filesystem-5.3.0-1.module 2.0 MB/s | 30 kB 00:00 +(16/40): lcms2-2.9-2.el8.x86_64.rpm 6.7 MB/s | 164 kB 00:00 +(17/40): java-11-openjdk-devel-11.0.16.1.1-1.el 46 MB/s | 3.4 MB 00:00 +(18/40): libX11-common-1.6.8-5.el8.noarch.rpm 8.4 MB/s | 158 kB 00:00 +(19/40): libX11-1.6.8-5.el8.x86_64.rpm 17 MB/s | 611 kB 00:00 +(20/40): libXau-1.0.9-3.el8.x86_64.rpm 2.6 MB/s | 37 kB 00:00 +(21/40): libXcomposite-0.4.4-14.el8.x86_64.rpm 2.0 MB/s | 28 kB 00:00 +(22/40): libXi-1.7.10-1.el8.x86_64.rpm 2.2 MB/s | 49 kB 00:00 +(23/40): libXext-1.3.4-1.el8.x86_64.rpm 1.6 MB/s | 45 kB 00:00 +(24/40): libXtst-1.2.3-7.el8.x86_64.rpm 1.1 MB/s | 22 kB 00:00 +(25/40): libXrender-0.9.10-7.el8.x86_64.rpm 1.3 MB/s | 33 kB 00:00 +(26/40): libfontenc-1.1.3-8.el8.x86_64.rpm 2.2 MB/s | 37 kB 00:00 +(27/40): libjpeg-turbo-1.5.3-12.el8.x86_64.rpm 8.6 MB/s | 157 kB 00:00 +(28/40): libxcb-1.13.1-1.el8.x86_64.rpm 13 MB/s | 231 kB 00:00 +(29/40): lua-5.3.4-12.el8.x86_64.rpm 11 MB/s | 192 kB 00:00 +(30/40): nspr-4.34.0-3.el8_6.x86_64.rpm 7.8 MB/s | 143 kB 00:00 +(31/40): nss-3.79.0-10.el8_6.x86_64.rpm 23 MB/s | 747 kB 00:00 +(32/40): nss-softokn-3.79.0-10.el8_6.x86_64.rpm 42 MB/s | 1.2 MB 00:00 +(33/40): nss-softokn-freebl-3.79.0-10.el8_6.x86 19 MB/s | 398 kB 00:00 +(34/40): nss-sysinit-3.79.0-10.el8_6.x86_64.rpm 5.3 MB/s | 74 kB 00:00 +(35/40): nss-util-3.79.0-10.el8_6.x86_64.rpm 8.7 MB/s | 138 kB 00:00 +(36/40): ttmkfdir-3.0.9-54.el8.x86_64.rpm 4.2 MB/s | 62 kB 00:00 +(37/40): tzdata-java-2022d-1.el8.noarch.rpm 11 MB/s | 186 kB 00:00 +(38/40): xorg-x11-font-utils-7.5-41.el8.x86_64. 6.7 MB/s | 104 kB 00:00 +(39/40): xorg-x11-fonts-Type1-7.5-19.el8.noarch 24 MB/s | 522 kB 00:00 +(40/40): java-11-openjdk-headless-11.0.16.1.1-1 77 MB/s | 40 MB 00:00 +-------------------------------------------------------------------------------- +Total 74 MB/s | 50 MB 00:00 +Running transaction check +Transaction check succeeded. +Running transaction test +Transaction test succeeded. +Running transaction + Running scriptlet: copy-jdk-configs-4.0-2.el8.noarch 1/1 + Running scriptlet: java-11-openjdk-headless-1:11.0.16.1.1-1.el8_6.x86_6 1/1 + Preparing : 1/1 + Installing : nspr-4.34.0-3.el8_6.x86_64 1/40 + Running scriptlet: nspr-4.34.0-3.el8_6.x86_64 1/40 + Installing : nss-util-3.79.0-10.el8_6.x86_64 2/40 + Installing : libjpeg-turbo-1.5.3-12.el8.x86_64 3/40 + Installing : nss-softokn-freebl-3.79.0-10.el8_6.x86_64 4/40 + Installing : nss-softokn-3.79.0-10.el8_6.x86_64 5/40 + Installing : tzdata-java-2022d-1.el8.noarch 6/40 + Installing : ttmkfdir-3.0.9-54.el8.x86_64 7/40 + Installing : lua-5.3.4-12.el8.x86_64 8/40 + Installing : copy-jdk-configs-4.0-2.el8.noarch 9/40 + Installing : libfontenc-1.1.3-8.el8.x86_64 10/40 + Installing : libXau-1.0.9-3.el8.x86_64 11/40 + Installing : libxcb-1.13.1-1.el8.x86_64 12/40 + Installing : libX11-common-1.6.8-5.el8.noarch 13/40 + Installing : libX11-1.6.8-5.el8.x86_64 14/40 + Installing : libXext-1.3.4-1.el8.x86_64 15/40 + Installing : libXi-1.7.10-1.el8.x86_64 16/40 + Installing : libXtst-1.2.3-7.el8.x86_64 17/40 + Installing : libXcomposite-0.4.4-14.el8.x86_64 18/40 + Installing : libXrender-0.9.10-7.el8.x86_64 19/40 + Installing : lcms2-2.9-2.el8.x86_64 20/40 + Running scriptlet: lcms2-2.9-2.el8.x86_64 20/40 + Installing : javapackages-filesystem-5.3.0-1.module+el8+5136+7f 21/40 + Installing : graphite2-1.3.10-10.el8.x86_64 22/40 + Installing : harfbuzz-1.7.5-3.el8.x86_64 23/40 + Running scriptlet: harfbuzz-1.7.5-3.el8.x86_64 23/40 + Installing : giflib-5.1.4-3.el8.x86_64 24/40 + Installing : alsa-lib-1.2.6.1-3.el8.x86_64 25/40 + Running scriptlet: alsa-lib-1.2.6.1-3.el8.x86_64 25/40 + Installing : pkgconf-m4-1.4.2-1.el8.noarch 26/40 + Installing : lksctp-tools-1.0.18-3.el8.x86_64 27/40 + Running scriptlet: lksctp-tools-1.0.18-3.el8.x86_64 27/40 + Installing : libpkgconf-1.4.2-1.el8.x86_64 28/40 + Installing : pkgconf-1.4.2-1.el8.x86_64 29/40 + Installing : pkgconf-pkg-config-1.4.2-1.el8.x86_64 30/40 + Installing : xorg-x11-font-utils-1:7.5-41.el8.x86_64 31/40 + Installing : xorg-x11-fonts-Type1-7.5-19.el8.noarch 32/40 + Running scriptlet: xorg-x11-fonts-Type1-7.5-19.el8.noarch 32/40 + Installing : crypto-policies-scripts-20211116-1.gitae470d6.el8. 33/40 + Installing : nss-sysinit-3.79.0-10.el8_6.x86_64 34/40 + Installing : nss-3.79.0-10.el8_6.x86_64 35/40 + Installing : avahi-libs-0.7-20.el8.x86_64 36/40 + Installing : cups-libs-1:2.2.6-45.el8_6.2.x86_64 37/40 + Installing : java-11-openjdk-headless-1:11.0.16.1.1-1.el8_6.x86 38/40 + Running scriptlet: java-11-openjdk-headless-1:11.0.16.1.1-1.el8_6.x86 38/40 + Installing : java-11-openjdk-1:11.0.16.1.1-1.el8_6.x86_64 39/40 + Running scriptlet: java-11-openjdk-1:11.0.16.1.1-1.el8_6.x86_64 39/40 + Installing : java-11-openjdk-devel-1:11.0.16.1.1-1.el8_6.x86_64 40/40 + Running scriptlet: java-11-openjdk-devel-1:11.0.16.1.1-1.el8_6.x86_64 40/40 + Running scriptlet: copy-jdk-configs-4.0-2.el8.noarch 40/40 + Running scriptlet: crypto-policies-scripts-20211116-1.gitae470d6.el8. 40/40 + Running scriptlet: nss-3.79.0-10.el8_6.x86_64 40/40 + Running scriptlet: java-11-openjdk-headless-1:11.0.16.1.1-1.el8_6.x86 40/40 + Running scriptlet: java-11-openjdk-1:11.0.16.1.1-1.el8_6.x86_64 40/40 + Running scriptlet: java-11-openjdk-devel-1:11.0.16.1.1-1.el8_6.x86_64 40/40 + Verifying : avahi-libs-0.7-20.el8.x86_64 1/40 + Verifying : crypto-policies-scripts-20211116-1.gitae470d6.el8. 2/40 + Verifying : cups-libs-1:2.2.6-45.el8_6.2.x86_64 3/40 + Verifying : libpkgconf-1.4.2-1.el8.x86_64 4/40 + Verifying : lksctp-tools-1.0.18-3.el8.x86_64 5/40 + Verifying : pkgconf-1.4.2-1.el8.x86_64 6/40 + Verifying : pkgconf-m4-1.4.2-1.el8.noarch 7/40 + Verifying : pkgconf-pkg-config-1.4.2-1.el8.x86_64 8/40 + Verifying : alsa-lib-1.2.6.1-3.el8.x86_64 9/40 + Verifying : copy-jdk-configs-4.0-2.el8.noarch 10/40 + Verifying : giflib-5.1.4-3.el8.x86_64 11/40 + Verifying : graphite2-1.3.10-10.el8.x86_64 12/40 + Verifying : harfbuzz-1.7.5-3.el8.x86_64 13/40 + Verifying : java-11-openjdk-1:11.0.16.1.1-1.el8_6.x86_64 14/40 + Verifying : java-11-openjdk-devel-1:11.0.16.1.1-1.el8_6.x86_64 15/40 + Verifying : java-11-openjdk-headless-1:11.0.16.1.1-1.el8_6.x86 16/40 + Verifying : javapackages-filesystem-5.3.0-1.module+el8+5136+7f 17/40 + Verifying : lcms2-2.9-2.el8.x86_64 18/40 + Verifying : libX11-1.6.8-5.el8.x86_64 19/40 + Verifying : libX11-common-1.6.8-5.el8.noarch 20/40 + Verifying : libXau-1.0.9-3.el8.x86_64 21/40 + Verifying : libXcomposite-0.4.4-14.el8.x86_64 22/40 + Verifying : libXext-1.3.4-1.el8.x86_64 23/40 + Verifying : libXi-1.7.10-1.el8.x86_64 24/40 + Verifying : libXrender-0.9.10-7.el8.x86_64 25/40 + Verifying : libXtst-1.2.3-7.el8.x86_64 26/40 + Verifying : libfontenc-1.1.3-8.el8.x86_64 27/40 + Verifying : libjpeg-turbo-1.5.3-12.el8.x86_64 28/40 + Verifying : libxcb-1.13.1-1.el8.x86_64 29/40 + Verifying : lua-5.3.4-12.el8.x86_64 30/40 + Verifying : nspr-4.34.0-3.el8_6.x86_64 31/40 + Verifying : nss-3.79.0-10.el8_6.x86_64 32/40 + Verifying : nss-softokn-3.79.0-10.el8_6.x86_64 33/40 + Verifying : nss-softokn-freebl-3.79.0-10.el8_6.x86_64 34/40 + Verifying : nss-sysinit-3.79.0-10.el8_6.x86_64 35/40 + Verifying : nss-util-3.79.0-10.el8_6.x86_64 36/40 + Verifying : ttmkfdir-3.0.9-54.el8.x86_64 37/40 + Verifying : tzdata-java-2022d-1.el8.noarch 38/40 + Verifying : xorg-x11-font-utils-1:7.5-41.el8.x86_64 39/40 + Verifying : xorg-x11-fonts-Type1-7.5-19.el8.noarch 40/40 + +Installed: + alsa-lib-1.2.6.1-3.el8.x86_64 + avahi-libs-0.7-20.el8.x86_64 + copy-jdk-configs-4.0-2.el8.noarch + crypto-policies-scripts-20211116-1.gitae470d6.el8.noarch + cups-libs-1:2.2.6-45.el8_6.2.x86_64 + giflib-5.1.4-3.el8.x86_64 + graphite2-1.3.10-10.el8.x86_64 + harfbuzz-1.7.5-3.el8.x86_64 + java-11-openjdk-1:11.0.16.1.1-1.el8_6.x86_64 + java-11-openjdk-devel-1:11.0.16.1.1-1.el8_6.x86_64 + java-11-openjdk-headless-1:11.0.16.1.1-1.el8_6.x86_64 + javapackages-filesystem-5.3.0-1.module+el8+5136+7ff78f74.noarch + lcms2-2.9-2.el8.x86_64 + libX11-1.6.8-5.el8.x86_64 + libX11-common-1.6.8-5.el8.noarch + libXau-1.0.9-3.el8.x86_64 + libXcomposite-0.4.4-14.el8.x86_64 + libXext-1.3.4-1.el8.x86_64 + libXi-1.7.10-1.el8.x86_64 + libXrender-0.9.10-7.el8.x86_64 + libXtst-1.2.3-7.el8.x86_64 + libfontenc-1.1.3-8.el8.x86_64 + libjpeg-turbo-1.5.3-12.el8.x86_64 + libpkgconf-1.4.2-1.el8.x86_64 + libxcb-1.13.1-1.el8.x86_64 + lksctp-tools-1.0.18-3.el8.x86_64 + lua-5.3.4-12.el8.x86_64 + nspr-4.34.0-3.el8_6.x86_64 + nss-3.79.0-10.el8_6.x86_64 + nss-softokn-3.79.0-10.el8_6.x86_64 + nss-softokn-freebl-3.79.0-10.el8_6.x86_64 + nss-sysinit-3.79.0-10.el8_6.x86_64 + nss-util-3.79.0-10.el8_6.x86_64 + pkgconf-1.4.2-1.el8.x86_64 + pkgconf-m4-1.4.2-1.el8.noarch + pkgconf-pkg-config-1.4.2-1.el8.x86_64 + ttmkfdir-3.0.9-54.el8.x86_64 + tzdata-java-2022d-1.el8.noarch + xorg-x11-font-utils-1:7.5-41.el8.x86_64 + xorg-x11-fonts-Type1-7.5-19.el8.noarch + +Complete! +Last metadata expiration check: 0:00:10 ago on Mon 10 Oct 2022 04:06:28 PM UTC. +Dependencies resolved. +============================================================================================== + Package + Arch Version Repository Size +============================================================================================== +Installing: + ords noarch 22.3.0-7.el8 yum.oracle.com_repo_OracleLinux_OL8_oracle_software_x86_64 87 M +Installing dependencies: + lsof x86_64 4.93.2-1.el8 ol8_baseos_latest 253 k + +Transaction Summary +============================================================================================== +Install 2 Packages + +Total download size: 87 M +Installed size: 92 M +Downloading Packages: +(1/2): lsof-4.93.2-1.el8.x86_64.rpm 3.0 MB/s | 253 kB 00:00 +(2/2): ords-22.3.0-7.el8.noarch.rpm 66 MB/s | 87 MB 00:01 +-------------------------------------------------------------------------------- +Total 66 MB/s | 87 MB 00:01 +Running transaction check +Transaction check succeeded. +Running transaction test +Transaction test succeeded. +Running transaction + Preparing : 1/1 + Installing : lsof-4.93.2-1.el8.x86_64 1/2 + Running scriptlet: ords-22.3.0-7.el8.noarch 2/2 + Installing : ords-22.3.0-7.el8.noarch 2/2 + Running scriptlet: ords-22.3.0-7.el8.noarch 2/2 +INFO: Before starting ORDS service, run the below command as user oracle: + ords --config /etc/ords/config install + + Verifying : lsof-4.93.2-1.el8.x86_64 1/2 + Verifying : ords-22.3.0-7.el8.noarch 2/2 + +Installed: + lsof-4.93.2-1.el8.x86_64 ords-22.3.0-7.el8.noarch + +Complete! +Last metadata expiration check: 0:00:15 ago on Mon 10 Oct 2022 04:06:28 PM UTC. +Package iproute-5.15.0-4.el8_6.1.x86_64 is already installed. +Dependencies resolved. +Nothing to do. +Complete! +24 files removed +Removing intermediate container d38a69d2cc70 + ---> 3a7b8edb327e +Step 5/10 : RUN mkdir -p $ORDS_HOME/doc_root && mkdir -p $ORDS_HOME/error && mkdir -p $ORDS_HOME/secrets && chmod ug+x $ORDS_HOME/*.sh && groupadd -g 54322 dba && usermod -u 54321 -d /home/oracle -g dba -m -s /bin/bash oracle && chown -R oracle:dba $ORDS_HOME && echo "oracle ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers + ---> Running in 1d05951f8252 +Removing intermediate container 1d05951f8252 + ---> 265cb7ab4f2c +Step 6/10 : USER oracle + ---> Running in 180d432ae42d +Removing intermediate container 180d432ae42d + ---> a9caee3d9426 +Step 7/10 : WORKDIR /home/oracle + ---> Running in bf8ac95c724a +Removing intermediate container bf8ac95c724a + ---> 4623d696e603 +Step 8/10 : VOLUME ["$ORDS_HOME/config/ords"] + ---> Running in 3afce627e4c0 +Removing intermediate container 3afce627e4c0 + ---> 914d4ee42ede +Step 9/10 : EXPOSE 8888 + ---> Running in 13460b132c52 +Removing intermediate container 13460b132c52 + ---> 4c9edba5aade +Step 10/10 : CMD $ORDS_HOME/$RUN_FILE + ---> Running in f97b17d8cea4 +Removing intermediate container f97b17d8cea4 + ---> c8e95aadf5e3 +Successfully built c8e95aadf5e3 +Successfully tagged oracle/ords-dboper:latest + diff --git a/docs/multitenant/usecase01/ImagePush.log b/docs/multitenant/usecase01/ImagePush.log new file mode 100644 index 00000000..9b8df426 --- /dev/null +++ b/docs/multitenant/usecase01/ImagePush.log @@ -0,0 +1,11 @@ +/usr/bin/docker tag oracle/ords-dboper:latest /ords-dboper:latest +/usr/bin/docker push /ords-dboper:latest +The push refers to repository [/ords-dboper] +aef18205865c: Pushing [=============================> ] 56.55MB/95.45MB +2564d855e579: Pushing [=======> ] 57.08MB/357.6MB +a70a4f9a73c3: Pushed +f283c83ba6ac: Pushed +8c6709989678: Pushing [=======> ] 52.58MB/332.7MB +5bfd57d8f58a: Pushing [========> ] 37.47MB/229.2MB + + diff --git a/docs/multitenant/usecase01/README.md b/docs/multitenant/usecase01/README.md new file mode 100644 index 00000000..428a99f5 --- /dev/null +++ b/docs/multitenant/usecase01/README.md @@ -0,0 +1,555 @@ + + + +# STEP BY STEP USE CASE + +- [INTRODUCTION](#introduction) +- [OPERATION STEPS ](#operation-steps) +- [Download latest version from orahub ](#download-latest-version-from-orahub-a-namedownloada) +- [Upload webhook certificates](#upload-webhook-certificates-a-namewebhooka) +- [Create the dboperator](#create-the-dboperator-a-namedboperatora) +- [Create Secret for container registry](#create-secret-for-container-registry) +- [Build ords immage ](#build-ords-immage-a-nameordsimagea) +- [Database Configuration](#database-configuration) +- [Create CDB secret ](#create-cdb-secret) +- [Create Certificates](#create-certificates) +- [Apply cdb.yaml](#apply-cdbyaml) +- [Logs and throuble shutting](#cdb---logs-and-throuble-shutting) +- [Create PDB secret](#create-pdb-secret) +- [Other action ](#other-actions) + + + +###### INTRODUCTION + +This readme is a step by step guide used to implement database multi tenant operator. It assumes that a kubernets cluster and a database server are already available (no matter if single instance or RAC). kubectl must be configured in order to reach k8s cluster. + +The following table reports the parameters required to configure and use oracle multi tenant controller for pluggable database lifecycle management. + +| yaml file parameters | value | description /ords parameter | +|-------------- |--------------------------- |-------------------------------------------------| +| dbserver | or | [--db-hostname][1] | +| dbTnsurl | | [--db-custom-url/db.customURL][dbtnsurl] | +| port | | [--db-port][2] | +| cdbName | | Container Name | +| name | | Ords podname prefix in cdb.yaml | +| name | | pdb resource in pdb.yaml | +| ordsImage | /ords-dboper:latest|My public container registry | +| pdbName | | Pluggable database name | +| servicename | | [--db-servicename][3] | +| sysadmin_user | | [--admin-user][adminuser] | +| sysadmin_pwd | | [--password-stdin][pwdstdin] | +| cdbadmin_user | | [db.cdb.adminUser][1] | +| cdbadmin_pwd | | [db.cdb.adminUser.password][cdbadminpwd] | +| webserver_user| | [https user][http] NOT A DB USER | +| webserver_pwd | | [http user password][http] | +| ords_pwd | | [ORDS_PUBLIC_USER password][public_user] | +| pdbTlsKey | | [standalone.https.cert.key][key] | +| pdbTlsCrt | | [standalone.https.cert][cr] | +| pdbTlsCat | | certificate authority | + + + +### OPERATIONAL STEPS +---- + +##### Download latest version from github + + +```bash +git clone https://github.com/oracle/oracle-database-operator.git +``` + +If golang compiler is installed on your environment and you've got a public container registry then you can compile the operator, upload to the registry and use it + +```bash + +cd oracle-database-operator +make generate +make manifests +make install +make docker-build IMG=/operator:latest + +make operator-yaml IMG=operator:latest +``` + +> **NOTE:** The last make executions recreates the **oracle-database-operator.yaml** with the **image:** parameter pointing to your public container registry. If you don't have a golang compilation environment you can use the **oracle-database-operator.yaml** provided in the github distribution. Check [operator installation documentation](../installation/OPERATOR_INSTALLATION_README.md ) for more details. + +> **NOTE:** If you are using oracle-container-registry make sure to accept the license agreement otherwise the operator image pull fails. +---- +##### Upload webhook certificates + +```bash +kubectl apply -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml +``` + +##### Create the dboperator + +```bash +cd oracle-database-operator +/usr/bin/kubectl apply -f oracle-database-operator.yaml +``` ++ Check the status of the operator + +```bash +/usr/bin/kubectl get pods -n oracle-database-operator-system +NAME READY STATUS RESTARTS AGE +oracle-database-operator-controller-manager-557ff6c659-g7t66 1/1 Running 0 10s +oracle-database-operator-controller-manager-557ff6c659-rssmj 1/1 Running 0 10s +oracle-database-operator-controller-manager-557ff6c659-xpswv 1/1 Running 0 10s + +``` +---- +##### Create secret for container registry + ++ Make sure to login to your container registry and then create the secret for you container registry. + +```bash +docker login **** +/usr/bin/kubectl create secret generic container-registry-secret --from-file=.dockerconfigjson=/home/oracle/.docker/config.json --type=kubernetes.io/dockerconfigjson -n oracle-database-operator-system +``` + ++ Check secret + +```bash +kubectl get secret -n oracle-database-operator-system +NAME TYPE DATA AGE +container-registry-secret kubernetes.io/dockerconfigjson 1 19s +webhook-server-cert kubernetes.io/tls +``` +---- +##### Build ords immage + ++ Build the ords image, downloading ords software is no longer needed; just build the image and push it to your repository + +```bash +cd oracle-database-operator/ords +docker build -t oracle/ords-dboper:latest . +``` + +[example of execution](./BuildImage.log) ++ Login to your container registry and push the ords image. + +```bash +docker tag /ords-dboper:latest +docker push /ords-dboper:latest +``` +[example of execution](./ImagePush.log) + +---- +##### Database Configuration + ++ Configure Database + +Connect as sysdba and execute the following script in order to create the required ords accounts. + +```sql +ALTER SESSION SET "_oracle_script"=true; +DROP USER cascade; +CREATE USER IDENTIFIED BY CONTAINER=ALL ACCOUNT UNLOCK; +GRANT SYSOPER TO CONTAINER = ALL; +GRANT SYSDBA TO CONTAINER = ALL; +GRANT CREATE SESSION TO CONTAINER = ALL; +``` +---- +##### Create CDB secret + ++ Create secret for CDB connection + +```bash +kubectl apply -f cdb_secret.yaml -n oracle-database-operator-system + +``` +Exmaple: **cdb_secret.yaml** + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: cdb1-secret + namespace: oracle-database-operator-system +type: Opaque +data: + ords_pwd: "encoded value" + sysadmin_pwd: "encoded value" + cdbadmin_user: "encoded value" + cdbadmin_pwd: "encoded value" + webserver_user: "encoded value" + webserver_pwd: "encoded value" + +``` +Use **base64** command to encode/decode username and password in the secret file as shown in the following example + +- encode +```bash +echo "ThisIsMyPassword" |base64 -i +VGhpc0lzTXlQYXNzd29yZAo= +``` +- decode +```bash + echo "VGhpc0lzTXlQYXNzd29yZAo=" | base64 --decode +ThisIsMyPassword + +``` + + +>Note that we do not have to create webuser on the database. + ++ Check secret: + +```bash +kubectl get secret -n oracle-database-operator-system +NAME TYPE DATA AGE +cdb1-secret Opaque 6 7s <--- +container-registry-secret kubernetes.io/dockerconfigjson 1 2m17s +webhook-server-cert kubernetes.io/tls 3 4m55s +``` + +>**TIPS:** Use the following commands to analyze contents of an existing secret ```bash kubectl get secret -o yaml -n ``` +---- +##### Create Certificates + ++ Create certificates: At this stage we need to create certificates on our local machine and upload into kubernetes cluster by creating new secrets. + + + +```text + + +-----------+ + | openssl | + +-----------+ + | + | + +-----------+ + | tls.key | + | tls.crt +------------+ + | ca.crt | | + +-----------+ | + | + +------------------------|---------------------------+ + |KUBERNETES +------+--------+ | + |CLUSTER +---|kubernet secret|---+ | + | | +---------------+ | | + | | | | + | +----------+---+ https +--+----------+ | + | |ORDS CONTAINER|<-------------->| PDB/POD | | + | +----------+---+ +-------------+ | + | cdb.yaml | pdb.yaml | + +-------------|--------------------------------------+ + | + | + +-----------+ + | DB SERVER | + +-----------+ + +``` + +```bash + +genrsa -out 2048 +openssl req -new -x509 -days 365 -key -subj "/C=CN/ST=GD/L=SZ/O=oracle, Inc./CN=oracle Root CA" -out +openssl req -newkey rsa:2048 -nodes -keyout -subj "/C=CN/ST=GD/L=SZ/O=oracle, Inc./CN=-ords" -out server.csr +/usr/bin/echo "subjectAltName=DNS:-ords,DNS:www.example.com" > extfile.txt +openssl x509 -req -extfile extfile.txt -days 365 -in server.csr -CA -CAkey -CAcreateserial -out + +kubectl create secret tls db-tls --key="" --cert="" -n oracle-database-operator-system +kubectl create secret generic db-ca --from-file= -n oracle-database-operator-system + +``` + +[example of execution:](./openssl_execution.log) + + +---- +###### Apply cdb.yaml + ++ Create ords container + +```bash +/usr/bin/kubectl apply -f cdb.yaml -n oracle-database-operator-system +``` +Example: **cdb.yaml** + +```yaml +apiVersion: database.oracle.com/v1alpha1 +kind: CDB +metadata: + name: + namespace: oracle-database-operator-system +spec: + cdbName: "" + dbServer: "" or + dbPort: + ordsImage: "/ords-dboper:.latest" + ordsImagePullPolicy: "Always" + serviceName: + replicas: 1 + sysAdminPwd: + secret: + secretName: "cdb1-secret" + key: "sysadmin_pwd" + ordsPwd: + secret: + secretName: "cdb1-secret" + key: "ords_pwd" + cdbAdminUser: + secret: + secretName: "cdb1-secret" + key: "cdbadmin_user" + cdbAdminPwd: + secret: + secretName: "cdb1-secret" + key: "cdbadmin_pwd" + webServerUser: + secret: + secretName: "cdb1-secret" + key: "webserver_user" + webServerPwd: + secret: + secretName: "cdb1-secret" + key: "webserver_pwd" + cdbTlsKey: + secret: + secretName: "db-tls" + key: "" + cdbTlsCrt: + secret: + secretName: "db-tls" + key: ":" + +``` +> **Note** if you are working in dataguard environment with multiple sites (AC/DR) specifying the host name (dbServer/dbPort/serviceName) may not be the suitable solution for this kind of configuration, use **dbTnsurl** instead. Specify the whole tns string which includes the hosts/scan list. + +``` + +----------+ + ____| standbyB | + | | scanB | (DESCRIPTION= + +----------+ | +----------+ (CONNECT_TIMEOUT=90) + | primary |_______| (RETRY_COUNT=30)(RETRY_DELAY=10)(TRANSPORT_CONNECT_TIMEOUT=70) + | scanA | | +----------+ (TRANSPORT_CONNECT_TIMEOUT=10)(LOAD_BALLANCE=ON) + +----------+ |___| stanbyC | (ADDRESS=(PROTOCOL=TCP)(HOST=scanA.testrac.com)(PORT=1521)(IP=V4_ONLY)) + | scanC | (ADDRESS=(PROTOCOL=TCP)(HOST=scanB.testrac.com)(PORT=1521)(IP=V4_ONLY)) + +----------+ (ADDRESS=(PROTOCOL=TCP)(HOST=scanC.testrac.com)(PORT=1521)(IP=V4_ONLY)) + (CONNECT_DATA=(SERVER=DEDICATED)(SERVICE_NAME=TESTORDS))) + + + dbtnsurl:((DESCRIPTION=(CONNECT_TIMEOUT=90)(RETRY_COUNT=30)(RETRY_DELAY=10)(TRANSPORT_CONNECT_TIMEOUT=70)(TRANS...... +``` + +[example of cdb.yaml](./cdb.yaml) + +---- +###### CDB - Logs and throuble shutting + ++ Check the status of ords container + +```bash +/usr/bin/kubectl get pods -n oracle-database-operator-system +NAME READY STATUS RESTARTS AGE +cdb-dev-ords-rs-m9ggp 0/1 ContainerCreating 0 67s <----- +oracle-database-operator-controller-manager-557ff6c659-g7t66 1/1 Running 0 11m +oracle-database-operator-controller-manager-557ff6c659-rssmj 1/1 Running 0 11m +oracle-database-operator-controller-manager-557ff6c659-xpswv 1/1 Running 0 11m +``` ++ Make sure that the cdb container is running + +```bash +/usr/bin/kubectl get pods -n oracle-database-operator-system +NAME READY STATUS RESTARTS AGE +cdb-dev-ords-rs-dnshz 1/1 Running 0 31s +oracle-database-operator-controller-manager-557ff6c659-9bjfl 1/1 Running 0 2m42s +oracle-database-operator-controller-manager-557ff6c659-cx8hd 1/1 Running 0 2m42s +oracle-database-operator-controller-manager-557ff6c659-rq9xs 1/1 Running 0 2m42s +``` ++ Check the status of the services + +```bash +kubectl get cdb -n oracle-database-operator-system +NAME CDB NAME DB SERVER DB PORT REPLICAS STATUS MESSAGE +[.....................................................] Ready +``` ++ Use log file to trouble shutting + +```bash +/usr/bin/kubectl logs `/usr/bin/kubectl get pods -n oracle-database-operator-system|grep ords|cut -d ' ' -f 1` -n oracle-database-operator-system +``` +[example of execution](./cdb.log) + ++ Test REST API from the pod. By querying the metadata catalog you can verify the status of https setting + +```bash + /usr/bin/kubectl exec -it `/usr/bin/kubectl get pods -n oracle-database-operator-system|grep ords|cut -d ' ' -f 1` -n oracle-database-operator-system -i -t -- /usr/bin/curl -sSkv -k -X GET https://localhost:8888/ords/_/db-api/stable/metadata-catalog/ +``` +[example of execution](./testapi.log) + ++ Verify the pod environment varaibles + ```bash + kubectl set env pods --all --list -n oracle-database-operator-system + ``` + ++ Connect to cdb pod + +```bash + kubectl exec -it `kubectl get pods -n oracle-database-operator-system|grep ords|cut -d ' ' -f 1` -n oracle-database-operator-system bash +``` ++ Dump ords server configuration + +```bash +/usr/bin/kubectl exec -it `/usr/bin/kubectl get pods -n oracle-database-operator-system|grep ords|cut -d ' ' -f 1` -n oracle-database-operator-system -i -t -- /usr/local/bin/ords --config /etc/ords/config config list +``` +[Example of executions](./ordsconfig.log) + +----- +###### Create PDB secret + + +```bash +/usr/bin/kubectl apply -f pdb.yaml -n oracle-database-operator-system +``` +Exmaple: **pdb_secret.yaml** + +```yaml +apiVersion: v1 +kind: Secret +metadata: + name: pdb1-secret + namespace: oracle-database-operator-system +type: Opaque +data: + sysadmin_user: "encoded value" + sysadmin_pwd: "encoded value" +``` + ++ Check secret creation + +```bash +kubectl get secret -n oracle-database-operator-system +NAME TYPE DATA AGE +cdb1-secret Opaque 6 79m +container-registry-secret kubernetes.io/dockerconfigjson 1 79m +db-ca Opaque 1 78m +db-tls kubernetes.io/tls 2 78m +pdb1-secret Opaque 2 79m <--- +webhook-server-cert kubernetes.io/tls 3 79m +``` +--- +###### Apply pdb yaml file to create pdb + +```bash +/usr/bin/kubectl apply -f pdb.yaml -n oracle-database-operator-system +``` + +Example: **pdb.yaml** + +```yaml +apiVersion: database.oracle.com/v1alpha1 +kind: PDB +metadata: + name: + namespace: oracle-database-operator-system + labels: + cdb: +spec: + cdbResName: "" + cdbName: "" + pdbName: + adminName: + secret: + secretName: "pdb1-secret" + key: "sysadmin_user" + adminPwd: + secret: + secretName: "pdb1-secret" + key: "sysadmin_pwd" + pdbTlsKey: + secret: + secretName: "db-tls" + key: "" + pdbTlsCrt: + secret: + secretName: "db-tls" + key: "" + pdbTlsCat: + secret: + secretName: "db-ca" + key: "" + fileNameConversions: "NONE" + totalSize: "1G" + tempSize: "100M" + action: "Create" +``` + ++ Monitor the pdb creation status until message is success + +```bash +kubectl get pdbs --all-namespaces=true + + +-----------------------------------------+ +-----------------------------------------+ + | STATUS MESSAGE |______\ | STATUS MESSAGE | + | Creating Waiting for PDB to be created | / | Ready Success | + +-----------------------------------------+ +-----------------------------------------+ + +NAMESPACE NAME DBSERVER CDB NAME PDB NAME PDB STATE PDB SIZE STATUS MESSAGE +oracle-database-operator-system 1G Creating Waiting for PDB to be created + +[wait sometimes] + +kubectl get pdbs --all-namespaces=true +NAMESPACE NAME DBSERVER CDB NAME PDB NAME PDB STATE PDB SIZE STATUS MESSAGE +oracle-database-operator-system pdb1 READ WRITE 1G Ready Success +``` + +Connect to the hosts and verify the PDB creation. + +```text +[oracle@racnode1 ~]$ sqlplus '/as sysdba' +[...] +Oracle Database 19c Enterprise Edition Release 19.0.0.0.0 - Production +Version 19.15.0.0.0 + + +SQL> show pdbs + + CON_ID CON_NAME OPEN MODE RESTRICTED +---------- ------------------------------ ---------- ---------- + 2 PDB$SEED READ ONLY NO + 3 PDBDEV READ WRITE NO + +``` +Check controller log to debug pluggable database life cycle actions in case of problem + +```bash +kubectl logs -f $(kubectl get pods -n oracle-database-operator-system|grep oracle-database-operator-controller|head -1|cut -d ' ' -f 1) -n oracle-database-operator-system +``` + +--- +###### Other actions + +Configure and use other yaml files to perform pluggable database life cycle managment action **modify_pdb_open.yaml** **modify_pdb_close.yaml** + +> **Note** sql command *"alter pluggable database open instances=all;"* acts only on closed databases, so you want get any oracle error in case of execution against an pluggable database already opened + + + + +[1]:https://docs.oracle.com/en/database/oracle/oracle-rest-data-services/22.2/ordig/installing-and-configuring-oracle-rest-data-services.html#GUID-E9625FAB-9BC8-468B-9FF9-443C88D76FA1:~:text=Table%202%2D2%20Command%20Options%20for%20Command%2DLine%20Interface%20Installation + +[2]:https://docs.oracle.com/en/database/oracle/oracle-rest-data-services/22.2/ordig/installing-and-configuring-oracle-rest-data-services.html#GUID-E9625FAB-9BC8-468B-9FF9-443C88D76FA1:~:text=Table%202%2D2%20Command%20Options%20for%20Command%2DLine%20Interface%20Installation + +[3]:https://docs.oracle.com/en/database/oracle/oracle-rest-data-services/22.2/ordig/installing-and-configuring-oracle-rest-data-services.html#GUID-DAA027FA-A4A6-43E1-B8DD-C92B330C2341:~:text=%2D%2Ddb%2Dservicename%20%3Cstring%3E + +[adminuser]:https://docs.oracle.com/en/database/oracle/oracle-rest-data-services/22.2/ordig/installing-and-configuring-oracle-rest-data-services.html#GUID-A9AED253-4EEC-4E13-A0C4-B7CE82EC1C22:~:text=Table%202%2D6%20Command%20Options%20for%20Uninstall%20CLI + +[public_user]:https://docs.oracle.com/en/database/oracle/oracle-rest-data-services/22.2/ordig/using-multitenant-architecture-oracle-rest-data-services.html#GUID-E64A141A-A71F-4979-8D33-C5F8496D3C19:~:text=Preinstallation%20Tasks%20for%20Oracle%20REST%20Data%20Services%20CDB%20Installation + +[key]:https://docs.oracle.com/en/database/oracle/oracle-rest-data-services/22.2/ordig/about-REST-configuration-files.html#GUID-006F916B-8594-4A78-B500-BB85F35C12A0:~:text=standalone.https.cert.key + +[cr]:https://docs.oracle.com/en/database/oracle/oracle-rest-data-services/22.2/ordig/about-REST-configuration-files.html#GUID-006F916B-8594-4A78-B500-BB85F35C12A0 + +[cdbadminpwd]:https://docs.oracle.com/en/database/oracle/oracle-rest-data-services/22.2/ordig/about-REST-configuration-files.html#GUID-006F916B-8594-4A78-B500-BB85F35C12A0:~:text=Table%20C%2D1%20Oracle%20REST%20Data%20Services%20Configuration%20Settings + +[pwdstdin]:https://docs.oracle.com/en/database/oracle/oracle-rest-data-services/22.2/ordig/installing-and-configuring-oracle-rest-data-services.html#GUID-88479C84-CAC1-4133-A33E-7995A645EC05:~:text=default%20database%20pool.-,2.1.4.1%20Understanding%20Command%20Options%20for%20Command%2DLine%20Interface%20Installation,-Table%202%2D2 + +[http]:https://docs.oracle.com/en/database/oracle/oracle-rest-data-services/22.2/ordig/installing-and-configuring-oracle-rest-data-services.html#GUID-BEECC057-A8F5-4EAB-B88E-9828C2809CD8:~:text=Example%3A%20delete%20%5B%2D%2Dglobal%5D-,user%20add,-Add%20a%20user + +[dbtnsurl]:https://docs.oracle.com/en/database/oracle/oracle-rest-data-services/22.2/ordig/installing-and-configuring-oracle-rest-data-services.html#GUID-A9AED253-4EEC-4E13-A0C4-B7CE82EC1C22 \ No newline at end of file diff --git a/docs/multitenant/usecase01/cdb.log b/docs/multitenant/usecase01/cdb.log new file mode 100644 index 00000000..c75e9bf8 --- /dev/null +++ b/docs/multitenant/usecase01/cdb.log @@ -0,0 +1,372 @@ +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M +NOT_INSTALLED=2 + SETUP +==================================================== +CONFIG=/etc/ords/config ++ export ORDS_LOGS=/tmp ++ ORDS_LOGS=/tmp ++ '[' -f /opt/oracle/ords//secrets/webserver_user ']' +++ cat /opt/oracle/ords//secrets/webserver_user ++ WEBSERVER_USER=.... ++ '[' -f /opt/oracle/ords//secrets/webserver_pwd ']' +++ cat /opt/oracle/ords//secrets/webserver_pwd ++ WEBSERVER_PASSWORD=.... ++ '[' -f /opt/oracle/ords//secrets/cdbadmin_user ']' +++ cat /opt/oracle/ords//secrets/cdbadmin_user ++ CDBADMIN_USER=.... ++ '[' -f /opt/oracle/ords//secrets/cdbadmin_pwd ']' +++ cat /opt/oracle/ords//secrets/cdbadmin_pwd ++ CDBADMIN_PWD=.... ++ '[' -f /opt/oracle/ords//secrets/sysadmin_pwd ']' +++ cat /opt/oracle/ords//secrets/sysadmin_pwd ++ SYSDBA_PASSWORD=..... ++ '[' -f /opt/oracle/ords//secrets/sysadmin_pwd ']' +++ cat /opt/oracle/ords//secrets/ords_pwd ++ ORDS_PASSWORD=.... ++ setupHTTPS ++ rm -rf /home/oracle/keystore ++ '[' '!' -d /home/oracle/keystore ']' ++ mkdir /home/oracle/keystore ++ cd /home/oracle/keystore ++ cat ++ rm /home/oracle/keystore/PASSWORD ++ ls -ltr /home/oracle/keystore +total 0 ++ SetParameter ++ /usr/local/bin/ords --config /etc/ords/config config set security.requestValidationFunction false +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:22 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The setting named: security.requestValidationFunction was set to: false in configuration: default ++ /usr/local/bin/ords --config /etc/ords/config config set jdbc.MaxLimit 100 +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:23 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The setting named: jdbc.MaxLimit was set to: 100 in configuration: default ++ /usr/local/bin/ords --config /etc/ords/config config set jdbc.InitialLimit 50 +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:24 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The setting named: jdbc.InitialLimit was set to: 50 in configuration: default ++ /usr/local/bin/ords --config /etc/ords/config config set error.externalPath /opt/oracle/ords/error +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:26 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The global setting named: error.externalPath was set to: /opt/oracle/ords/error ++ /usr/local/bin/ords --config /etc/ords/config config set standalone.access.log /home/oracle +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:27 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The global setting named: standalone.access.log was set to: /home/oracle ++ /usr/local/bin/ords --config /etc/ords/config config set standalone.https.port 8888 +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:28 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The global setting named: standalone.https.port was set to: 8888 ++ /usr/local/bin/ords --config /etc/ords/config config set standalone.https.cert /opt/oracle/ords//secrets/tls.crt +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:29 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The global setting named: standalone.https.cert was set to: /opt/oracle/ords//secrets/tls.crt ++ /usr/local/bin/ords --config /etc/ords/config config set standalone.https.cert.key /opt/oracle/ords//secrets/tls.key +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:31 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The global setting named: standalone.https.cert.key was set to: /opt/oracle/ords//secrets/tls.key ++ /usr/local/bin/ords --config /etc/ords/config config set restEnabledSql.active true +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:32 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The setting named: restEnabledSql.active was set to: true in configuration: default ++ /usr/local/bin/ords --config /etc/ords/config config set security.verifySSL true +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:33 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The global setting named: security.verifySSL was set to: true ++ /usr/local/bin/ords --config /etc/ords/config config set database.api.enabled true +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:34 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The global setting named: database.api.enabled was set to: true ++ /usr/local/bin/ords --config /etc/ords/config config set plsql.gateway.mode false +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:35 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +Invalid VALUE argument false for KEY plsql.gateway.mode. ++ /usr/local/bin/ords --config /etc/ords/config config set database.api.management.services.disabled false +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:37 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The global setting named: database.api.management.services.disabled was set to: false ++ /usr/local/bin/ords --config /etc/ords/config config set misc.pagination.maxRows 1000 +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:38 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The setting named: misc.pagination.maxRows was set to: 1000 in configuration: default ++ /usr/local/bin/ords --config /etc/ords/config config set db.cdb.adminUser 'C##DBAPI_CDB_ADMIN AS SYSDBA' +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:39 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The setting named: db.cdb.adminUser was set to: C##DBAPI_CDB_ADMIN AS SYSDBA in configuration: default ++ /usr/local/bin/ords --config /etc/ords/config config secret --password-stdin db.cdb.adminUser.password +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:40 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +The setting named: db.cdb.adminUser.password was set to: ****** in configuration: default ++ /usr/local/bin/ords --config /etc/ords/config config user add --password-stdin sql_admin 'SQL Administrator, System Administrator' +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:42 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +Created user sql_admin in file /etc/ords/config/global/credentials ++ /usr/local/bin/ords --config /etc/ords/config install --admin-user 'SYS AS SYSDBA' --db-hostname racnode1.testrac.com --db-port 1521 --db-servicename TESTORDS --feature-db-api true --feature-rest-enabled-sql true --log-folder /tmp --proxy-user --password-stdin +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:45:43 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +Oracle REST Data Services - Non-Interactive Install +Connecting to database user: SYS AS SYSDBA url: jdbc:oracle:thin:@//racnode1.testrac.com:1521/TESTORDS + +Retrieving information.. +Your database connection is to a CDB. ORDS common user ORDS_PUBLIC_USER will be created in the CDB. ORDS schema will be installed in the PDBs. +Root CDB$ROOT - create ORDS common user +PDB PDB$SEED - install ORDS 22.3.0.r2781755 +PDB PDB$SEED - configure PL/SQL gateway user APEX_PUBLIC_USER in ORDS version 22.3.0.r2781755 + +The setting named: db.connectionType was set to: basic in configuration: default +The setting named: db.hostname was set to: racnode1.testrac.com in configuration: default +The setting named: db.port was set to: 1521 in configuration: default +The setting named: db.servicename was set to: TESTORDS in configuration: default +The setting named: db.serviceNameSuffix was set to: in configuration: default +The setting named: plsql.gateway.mode was set to: proxied in configuration: default +The setting named: db.username was set to: ORDS_PUBLIC_USER in configuration: default +The setting named: db.password was set to: ****** in configuration: default +The setting named: security.requestValidationFunction was set to: wwv_flow_epg_include_modules.authorize in configuration: default +2022-10-11T07:45:45.885Z INFO Installing Oracle REST Data Services version 22.3.0.r2781755 in CDB$ROOT +2022-10-11T07:45:46.703Z INFO ... Verified database prerequisites +2022-10-11T07:45:46.946Z INFO ... Created Oracle REST Data Services proxy user +2022-10-11T07:45:46.979Z INFO Completed installation for Oracle REST Data Services version 22.3.0.r2781755. Elapsed time: 00:00:01.71 + +2022-10-11T07:45:46.986Z INFO Installing Oracle REST Data Services version 22.3.0.r2781755 in PDB$SEED +2022-10-11T07:45:47.078Z INFO ... Verified database prerequisites +2022-10-11T07:45:47.290Z INFO ... Created Oracle REST Data Services proxy user +2022-10-11T07:45:47.741Z INFO ... Created Oracle REST Data Services schema +2022-10-11T07:45:48.097Z INFO ... Granted privileges to Oracle REST Data Services +2022-10-11T07:45:51.848Z INFO ... Created Oracle REST Data Services database objects +2022-10-11T07:46:00.829Z INFO Completed installation for Oracle REST Data Services version 22.3.0.r2781755. Elapsed time: 00:00:13.841 + +2022-10-11T07:46:00.898Z INFO Completed configuring PL/SQL gateway user for Oracle REST Data Services version 22.3.0.r2781755. Elapsed time: 00:00:00.68 + +2022-10-11T07:46:00.898Z INFO Completed CDB installation for Oracle REST Data Services version 22.3.0.r2781755. Total elapsed time: 00:00:15.17 + +2022-10-11T07:46:00.898Z INFO Log file written to /tmp/ords_cdb_install_2022-10-11_074545_78000.log +2022-10-11T07:46:00.901Z INFO To run in standalone mode, use the ords serve command: +2022-10-11T07:46:00.901Z INFO ords --config /etc/ords/config serve +2022-10-11T07:46:00.901Z INFO Visit the ORDS Documentation to access tutorials, developer guides and more to help you get started with the new ORDS Command Line Interface (http://oracle.com/rest). ++ '[' 0 -ne 0 ']' ++ StartUp ++ /usr/local/bin/ords --config /etc/ords/config serve --port 8888 --secure +Picked up _JAVA_OPTIONS: -Xms1126M -Xmx1126M + +ORDS: Release 22.3 Production on Tue Oct 11 07:46:02 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +2022-10-11T07:46:02.286Z INFO HTTPS and HTTPS/2 listening on host: 0.0.0.0 port: 8888 +2022-10-11T07:46:02.302Z INFO Disabling document root because the specified folder does not exist: /etc/ords/config/global/doc_root +2022-10-11T07:46:04.636Z INFO Configuration properties for: |default|lo| +db.servicename=TESTORDS +db.serviceNameSuffix= +java.specification.version=19 +conf.use.wallet=true +database.api.management.services.disabled=false +sun.jnu.encoding=UTF-8 +user.region=US +java.class.path=/opt/oracle/ords/ords.war +java.vm.vendor=Oracle Corporation +standalone.https.cert.key=/opt/oracle/ords//secrets/tls.key +sun.arch.data.model=64 +nashorn.args=--no-deprecation-warning +java.vendor.url=https://java.oracle.com/ +resource.templates.enabled=false +user.timezone=UTC +db.port=1521 +java.vm.specification.version=19 +os.name=Linux +sun.java.launcher=SUN_STANDARD +user.country=US +sun.boot.library.path=/usr/java/jdk-19/lib +sun.java.command=/opt/oracle/ords/ords.war --config /etc/ords/config serve --port 8888 --secure +jdk.debug=release +sun.cpu.endian=little +user.home=/home/oracle +oracle.dbtools.launcher.executable.jar.path=/opt/oracle/ords/ords.war +user.language=en +db.cdb.adminUser.password=****** +java.specification.vendor=Oracle Corporation +java.version.date=2022-09-20 +database.api.enabled=true +java.home=/usr/java/jdk-19 +db.username=ORDS_PUBLIC_USER +file.separator=/ +java.vm.compressedOopsMode=32-bit +line.separator= + +restEnabledSql.active=true +java.specification.name=Java Platform API Specification +java.vm.specification.vendor=Oracle Corporation +java.awt.headless=true +standalone.https.cert=/opt/oracle/ords//secrets/tls.crt +db.hostname=racnode1.testrac.com +db.password=****** +sun.management.compiler=HotSpot 64-Bit Tiered Compilers +security.requestValidationFunction=wwv_flow_epg_include_modules.authorize +misc.pagination.maxRows=1000 +java.runtime.version=19+36-2238 +user.name=oracle +error.externalPath=/opt/oracle/ords/error +stdout.encoding=UTF-8 +path.separator=: +db.cdb.adminUser=C##DBAPI_CDB_ADMIN AS SYSDBA +os.version=5.4.17-2136.308.9.el7uek.x86_64 +java.runtime.name=Java(TM) SE Runtime Environment +file.encoding=UTF-8 +plsql.gateway.mode=proxied +security.verifySSL=true +standalone.https.port=8888 +java.vm.name=Java HotSpot(TM) 64-Bit Server VM +java.vendor.url.bug=https://bugreport.java.com/bugreport/ +java.io.tmpdir=/tmp +oracle.dbtools.cmdline.ShellCommand=ords +java.version=19 +user.dir=/home/oracle/keystore +os.arch=amd64 +java.vm.specification.name=Java Virtual Machine Specification +jdbc.MaxLimit=100 +oracle.dbtools.cmdline.home=/opt/oracle/ords +native.encoding=UTF-8 +java.library.path=/usr/java/packages/lib:/usr/lib64:/lib64:/lib:/usr/lib +java.vendor=Oracle Corporation +java.vm.info=mixed mode, sharing +stderr.encoding=UTF-8 +java.vm.version=19+36-2238 +sun.io.unicode.encoding=UnicodeLittle +jdbc.InitialLimit=50 +db.connectionType=basic +java.class.version=63.0 +standalone.access.log=/home/oracle + +2022-10-11T07:46:06.669Z INFO Oracle REST Data Services initialized +Oracle REST Data Services version : 22.3.0.r2781755 +Oracle REST Data Services server info: jetty/10.0.11 +Oracle REST Data Services java info: Java HotSpot(TM) 64-Bit Server VM 19+36-2238 + diff --git a/docs/multitenant/usecase01/cdb.yaml b/docs/multitenant/usecase01/cdb.yaml new file mode 100644 index 00000000..25488168 --- /dev/null +++ b/docs/multitenant/usecase01/cdb.yaml @@ -0,0 +1,46 @@ +apiVersion: database.oracle.com/v1alpha1 +kind: CDB +metadata: + name: cdb-dev + namespace: oracle-database-operator-system +spec: + cdbName: "DB12" + dbServer: "racnode1.testrac.com" + dbPort: 1521 + ordsImage: "/ords-dboper:latest" + ordsImagePullPolicy: "Always" + serviceName: "TESTORDS" + replicas: 1 + sysAdminPwd: + secret: + secretName: "cdb1-secret" + key: "sysadmin_pwd" + ordsPwd: + secret: + secretName: "cdb1-secret" + key: "ords_pwd" + cdbAdminUser: + secret: + secretName: "cdb1-secret" + key: "cdbadmin_user" + cdbAdminPwd: + secret: + secretName: "cdb1-secret" + key: "cdbadmin_pwd" + webServerUser: + secret: + secretName: "cdb1-secret" + key: "webserver_user" + webServerPwd: + secret: + secretName: "cdb1-secret" + key: "webserver_pwd" + cdbTlsKey: + secret: + secretName: "db-tls" + key: "tls.key" + cdbTlsCrt: + secret: + secretName: "db-tls" + key: "tls.crt" + diff --git a/docs/multitenant/usecase01/openssl_execution.log b/docs/multitenant/usecase01/openssl_execution.log new file mode 100644 index 00000000..30a1c5d4 --- /dev/null +++ b/docs/multitenant/usecase01/openssl_execution.log @@ -0,0 +1,22 @@ +/usr/bin/openssl genrsa -out ca.key 2048 +Generating RSA private key, 2048 bit long modulus +......................................................................................................................................................................................+++ +...................................+++ +e is 65537 (0x10001) +/usr/bin/openssl req -new -x509 -days 365 -key ca.key -subj "/C=CN/ST=GD/L=SZ/O=oracle, Inc./CN=oracle Root CA" -out ca.crt +/usr/bin/openssl req -newkey rsa:2048 -nodes -keyout tls.key -subj "/C=CN/ST=GD/L=SZ/O=oracle, Inc./CN=cdb-dev-ords" -out server.csr +Generating a 2048 bit RSA private key +...................................+++ +........................................+++ +writing new private key to 'tls.key' +----- +/usr/bin/echo "subjectAltName=DNS:cdb-dev-ords,DNS:www.example.com" > extfile.txt +/usr/bin/openssl x509 -req -extfile extfile.txt -days 365 -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial -out tls.crt +Signature ok +subject=/C=CN/ST=GD/L=SZ/O=oracle, Inc./CN=cdb-dev-ords +Getting CA Private Key +/usr/bin/kubectl create secret tls db-tls --key="tls.key" --cert="tls.crt" -n oracle-database-operator-system +secret/db-tls created +/usr/bin/kubectl create secret generic db-ca --from-file="ca.crt" -n oracle-database-operator-system +secret/db-ca created + diff --git a/docs/multitenant/usecase01/ordsconfig.log b/docs/multitenant/usecase01/ordsconfig.log new file mode 100644 index 00000000..ad5e7bab --- /dev/null +++ b/docs/multitenant/usecase01/ordsconfig.log @@ -0,0 +1,35 @@ +: Release 22.3 Production on Tue Oct 11 12:51:50 2022 + +Copyright (c) 2010, 2022, Oracle. + +Configuration: + /etc/ords/config/ + +Database pool: default + +Setting Value Source +----------------------------------------- -------------------------------------- ----------- +database.api.enabled true Global +database.api.management.services.disabled false Global +db.cdb.adminUser C##DBAPI_CDB_ADMIN AS SYSDBA Pool +db.cdb.adminUser.password ****** Pool Wallet +db.connectionType basic Pool +db.hostname racnode1.testrac.com Pool +db.password ****** Pool Wallet +db.port 1521 Pool +db.serviceNameSuffix Pool +db.servicename TESTORDS Pool +db.username ORDS_PUBLIC_USER Pool +error.externalPath /opt/oracle/ords/error Global +jdbc.InitialLimit 50 Pool +jdbc.MaxLimit 100 Pool +misc.pagination.maxRows 1000 Pool +plsql.gateway.mode proxied Pool +restEnabledSql.active true Pool +security.requestValidationFunction wwv_flow_epg_include_modules.authorize Pool +security.verifySSL true Global +standalone.access.log /home/oracle Global +standalone.https.cert /opt/oracle/ords//secrets/tls.crt Global +standalone.https.cert.key /opt/oracle/ords//secrets/tls.key Global +standalone.https.port 8888 Global + diff --git a/docs/multitenant/usecase01/sync.sh b/docs/multitenant/usecase01/sync.sh new file mode 100644 index 00000000..d982e796 --- /dev/null +++ b/docs/multitenant/usecase01/sync.sh @@ -0,0 +1,5 @@ +#!/bin/bash + + +sudo cp README.md /home/oracle/WORKDIR/Ords22_operator/oracle-database-operator/docs/ords +sudo chown oracle:dba /home/oracle/WORKDIR/Ords22_operator/oracle-database-operator/docs/ords/README.md diff --git a/docs/multitenant/usecase01/testapi.log b/docs/multitenant/usecase01/testapi.log new file mode 100644 index 00000000..4c95b457 --- /dev/null +++ b/docs/multitenant/usecase01/testapi.log @@ -0,0 +1,49 @@ +* Trying 127.0.0.1... +* TCP_NODELAY set +* Connected to localhost (127.0.0.1) port 8888 (#0) +* ALPN, offering h2 +* ALPN, offering http/1.1 +* successfully set certificate verify locations: +* CAfile: /etc/pki/tls/certs/ca-bundle.crt + CApath: none +* TLSv1.3 (OUT), TLS handshake, Client hello (1): +* TLSv1.3 (IN), TLS handshake, Server hello (2): +* TLSv1.3 (IN), TLS handshake, [no content] (0): +* TLSv1.3 (IN), TLS handshake, Encrypted Extensions (8): +* TLSv1.3 (IN), TLS handshake, Certificate (11): +* TLSv1.3 (IN), TLS handshake, CERT verify (15): +* TLSv1.3 (IN), TLS handshake, Finished (20): +* TLSv1.3 (OUT), TLS change cipher, Change cipher spec (1): +* TLSv1.3 (OUT), TLS handshake, [no content] (0): +* TLSv1.3 (OUT), TLS handshake, Finished (20): +* SSL connection using TLSv1.3 / TLS_AES_256_GCM_SHA384 +* ALPN, server accepted to use h2 +* Server certificate: +* subject: C=CN; ST=GD; L=SZ; O=oracle, Inc.; CN=cdb-dev-ords +* start date: Oct 11 07:44:38 2022 GMT +* expire date: Oct 11 07:44:38 2023 GMT +* issuer: C=CN; ST=GD; L=SZ; O=oracle, Inc.; CN=oracle Root CA +* SSL certificate verify result: unable to get local issuer certificate (20), continuing anyway. +* Using HTTP2, server supports multi-use +* Connection state changed (HTTP/2 confirmed) +* Copying HTTP/2 data in stream buffer to connection buffer after upgrade: len=0 +* TLSv1.3 (OUT), TLS app data, [no content] (0): +* TLSv1.3 (OUT), TLS app data, [no content] (0): +* TLSv1.3 (OUT), TLS app data, [no content] (0): +* Using Stream ID: 1 (easy handle 0x564be7b0b970) +* TLSv1.3 (OUT), TLS app data, [no content] (0): +> GET /ords/_/db-api/stable/metadata-catalog/ HTTP/2 +> Host: localhost:8888 +> User-Agent: curl/7.61.1 +> Accept: */* +> +* TLSv1.3 (IN), TLS handshake, [no content] (0): +* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): +* TLSv1.3 (IN), TLS handshake, [no content] (0): +* TLSv1.3 (IN), TLS handshake, Newsession Ticket (4): +* TLSv1.3 (IN), TLS app data, [no content] (0): +* Connection state changed (MAX_CONCURRENT_STREAMS == 128)! +* TLSv1.3 (OUT), TLS app data, [no content] (0): +* TLSv1.3 (IN), TLS app data, [no content] (0): +* TLSv1.3 (IN), TLS app data, [no content] (0): + diff --git a/oracle-database-operator.yaml b/oracle-database-operator.yaml index 95786f0b..309cbcb0 100644 --- a/oracle-database-operator.yaml +++ b/oracle-database-operator.yaml @@ -619,6 +619,10 @@ spec: jsonPath: .spec.dbPort name: DB Port type: integer + - description: ' string of the tnsalias' + jsonPath: .spec.dbTnsurl + name: TNS STRING + type: string - description: Replicas jsonPath: .spec.replicas name: Replicas @@ -722,6 +726,8 @@ spec: dbServer: description: Name of the DB server type: string + dbTnsurl: + type: string nodeSelector: additionalProperties: type: string diff --git a/ords/runOrdsSSL.sh b/ords/runOrdsSSL.sh index ac357637..23b99f1e 100644 --- a/ords/runOrdsSSL.sh +++ b/ords/runOrdsSSL.sh @@ -10,6 +10,7 @@ # # MODIFIED (DD-Mon-YY) # mmalvezz 25-Jun-22 - Initial version +# mmalvezz 17-Oct-22 - db.customURL utilization export ORDS=/usr/local/bin/ords export ETCFILE=/etc/ords.conf @@ -24,13 +25,32 @@ export HN=`hostname` #export CERTIFICATE=${KEYSTORE}/${HN}.der export KEY=$ORDS_HOME/secrets/$TLSKEY export CERTIFICATE=$ORDS_HOME/secrets/$TLSCRT +export TNS_ADMIN=/opt/oracle/ords/ +export TNSNAME=${TNS_ADMIN}/tnsnames.ora +export TNSALIAS=ordstns +echo "${TNSALIAS}=${DBTNSURL}" >$TNSNAME + + -export CUSTOMURL="jdbc:oracle:thin:@(DESCRIPTION = (ADDRESS = (PROTOCOL = TCP)(HOST = racnode1)(PORT = 1521)) (CONNECT_DATA = (SERVER = DEDICATED) (SERVICE_NAME = TESTORDS)))" -echo $CUSTOMURL function SetParameter() { ##ords config info <--- Use this command to get the list +[[ ! -z "${ORACLE_HOST}" && -z "${DBTNSURL}" ]] && { + $ORDS --config ${CONFIG} config set db.hostname ${ORACLE_HOST:-racnode1} + $ORDS --config ${CONFIG} config set db.port ${ORACLE_PORT:-1521} + $ORDS --config ${CONFIG} config set db.servicename ${ORACLE_SERVICE:-TESTORDS} +} + +[[ -z "${ORACLE_HOST}" && ! -z "${DBTNSURL}" ]] && { + #$ORDS --config ${CONFIG} config set db.tnsAliasName ${TNSALIAS} + #$ORDS --config ${CONFIG} config set db.tnsDirectory ${TNS_ADMIN} + #$ORDS --config ${CONFIG} config set db.connectionType tns + + $ORDS --config ${CONFIG} config set db.connectionType customurl + $ORDS --config ${CONFIG} config set db.customURL jdbc:oracle:thin:@${DBTNSURL} +} + $ORDS --config ${CONFIG} config set security.requestValidationFunction false $ORDS --config ${CONFIG} config set jdbc.MaxLimit 100 $ORDS --config ${CONFIG} config set jdbc.InitialLimit 50 @@ -42,7 +62,7 @@ function SetParameter() { $ORDS --config ${CONFIG} config set restEnabledSql.active true $ORDS --config ${CONFIG} config set security.verifySSL true $ORDS --config ${CONFIG} config set database.api.enabled true - $ORDS --config ${CONFIG} config set plsql.gateway.mode false + $ORDS --config ${CONFIG} config set plsql.gateway.mode disabled $ORDS --config ${CONFIG} config set database.api.management.services.disabled false $ORDS --config ${CONFIG} config set misc.pagination.maxRows 1000 $ORDS --config ${CONFIG} config set db.cdb.adminUser "${CDBADMIN_USER:-C##DBAPI_CDB_ADMIN} AS SYSDBA" @@ -146,12 +166,8 @@ export ORDS_LOGS=/tmp setupHTTPS; SetParameter; - $ORDS --config ${CONFIG} install \ --admin-user ${SYSDBA_USER:-"SYS AS SYSDBA"} \ - --db-hostname ${ORACLE_HOST:-racnode1} \ - --db-port ${ORACLE_PORT:-1521} \ - --db-servicename ${ORACLE_SERVICE:-TESTORDS} \ --feature-db-api true \ --feature-rest-enabled-sql true \ --log-folder ${ORDS_LOGS} \ @@ -161,6 +177,8 @@ ${SYSDBA_PASSWORD:-WElcome_12##} ${ORDS_PASSWORD:-WElcome_12##} EOF + + if [ $? -ne 0 ] then echo "Installation error" From 6668f6be5a3473c21da8fcdcf3536cf7bb87de75 Mon Sep 17 00:00:00 2001 From: Matteo Malvezzi Date: Tue, 22 Nov 2022 15:54:28 +0000 Subject: [PATCH 34/37] fix for bug 34817258 UNPLUGPDB FUNCTION : VARIABLE TDESECRET AND TDEPASSWORD HAVE WRONG SCOPE DEFINITION --- controllers/database/pdb_controller.go | 35 +++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/controllers/database/pdb_controller.go b/controllers/database/pdb_controller.go index f9231a66..05e550c3 100644 --- a/controllers/database/pdb_controller.go +++ b/controllers/database/pdb_controller.go @@ -119,6 +119,10 @@ var ( const PDBFinalizer = "database.oracle.com/PDBfinalizer" +var tdePassword string +var tdeSecret string + + //+kubebuilder:rbac:groups=database.oracle.com,resources=pdbs,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=database.oracle.com,resources=pdbs/status,verbs=get;update;patch //+kubebuilder:rbac:groups=database.oracle.com,resources=pdbs/finalizers,verbs=get;create;update;patch;delete @@ -585,6 +589,9 @@ func (r *PDBReconciler) createPDB(ctx context.Context, req ctrl.Request, pdb *db log := r.Log.WithValues("createPDB", req.NamespacedName) var err error + var tdePassword string + var tdeSecret string + cdb, err := r.getCDBResource(ctx, req, pdb) if err != nil { @@ -613,14 +620,17 @@ func (r *PDBReconciler) createPDB(ctx context.Context, req ctrl.Request, pdb *db "getScript": strconv.FormatBool(*(pdb.Spec.GetScript))} if *(pdb.Spec.TDEImport) { - tdePassword, err := r.getSecret(ctx, req, pdb, pdb.Spec.TDEPassword.Secret.SecretName, pdb.Spec.TDEPassword.Secret.Key) + tdePassword, err = r.getSecret(ctx, req, pdb, pdb.Spec.TDEPassword.Secret.SecretName, pdb.Spec.TDEPassword.Secret.Key) if err != nil { return err } - tdeSecret, err := r.getSecret(ctx, req, pdb, pdb.Spec.TDESecret.Secret.SecretName, pdb.Spec.TDESecret.Secret.Key) + tdeSecret, err = r.getSecret(ctx, req, pdb, pdb.Spec.TDESecret.Secret.SecretName, pdb.Spec.TDESecret.Secret.Key) if err != nil { return err } + + tdeSecret = tdeSecret[:len(tdeSecret)-1] + tdePassword = tdeSecret[:len(tdePassword)-1] values["tdePassword"] = tdePassword values["tdeKeystorePath"] = pdb.Spec.TDEKeystorePath values["tdeSecret"] = tdeSecret @@ -636,6 +646,7 @@ func (r *PDBReconciler) createPDB(ctx context.Context, req ctrl.Request, pdb *db } _, err = r.callAPI(ctx, req, pdb, url, values, "POST") if err != nil { + log.Error(err, "callAPI error", err.Error()) return err } @@ -720,6 +731,8 @@ func (r *PDBReconciler) plugPDB(ctx context.Context, req ctrl.Request, pdb *dbap log := r.Log.WithValues("plugPDB", req.NamespacedName) var err error + var tdePassword string + var tdeSecret string cdb, err := r.getCDBResource(ctx, req, pdb) if err != nil { @@ -742,14 +755,17 @@ func (r *PDBReconciler) plugPDB(ctx context.Context, req ctrl.Request, pdb *dbap "getScript": strconv.FormatBool(*(pdb.Spec.GetScript))} if *(pdb.Spec.TDEImport) { - tdePassword, err := r.getSecret(ctx, req, pdb, pdb.Spec.TDEPassword.Secret.SecretName, pdb.Spec.TDEPassword.Secret.Key) + tdePassword, err = r.getSecret(ctx, req, pdb, pdb.Spec.TDEPassword.Secret.SecretName, pdb.Spec.TDEPassword.Secret.Key) if err != nil { return err } - tdeSecret, err := r.getSecret(ctx, req, pdb, pdb.Spec.TDESecret.Secret.SecretName, pdb.Spec.TDESecret.Secret.Key) + tdeSecret, err = r.getSecret(ctx, req, pdb, pdb.Spec.TDESecret.Secret.SecretName, pdb.Spec.TDESecret.Secret.Key) if err != nil { return err } + + tdeSecret = tdeSecret[:len(tdeSecret)-1] + tdePassword = tdeSecret[:len(tdePassword)-1] values["tdePassword"] = tdePassword values["tdeKeystorePath"] = pdb.Spec.TDEKeystorePath values["tdeSecret"] = tdeSecret @@ -793,6 +809,9 @@ func (r *PDBReconciler) unplugPDB(ctx context.Context, req ctrl.Request, pdb *db log := r.Log.WithValues("unplugPDB", req.NamespacedName) var err error + var tdePassword string + var tdeSecret string + cdb, err := r.getCDBResource(ctx, req, pdb) if err != nil { @@ -806,14 +825,17 @@ func (r *PDBReconciler) unplugPDB(ctx context.Context, req ctrl.Request, pdb *db if *(pdb.Spec.TDEExport) { // Get the TDE Password - tdePassword, err := r.getSecret(ctx, req, pdb, pdb.Spec.TDEPassword.Secret.SecretName, pdb.Spec.TDEPassword.Secret.Key) + tdePassword, err = r.getSecret(ctx, req, pdb, pdb.Spec.TDEPassword.Secret.SecretName, pdb.Spec.TDEPassword.Secret.Key) if err != nil { return err } - tdeSecret, err := r.getSecret(ctx, req, pdb, pdb.Spec.TDESecret.Secret.SecretName, pdb.Spec.TDESecret.Secret.Key) + tdeSecret, err = r.getSecret(ctx, req, pdb, pdb.Spec.TDESecret.Secret.SecretName, pdb.Spec.TDESecret.Secret.Key) if err != nil { return err } + + tdeSecret = tdeSecret[:len(tdeSecret)-1] + tdePassword = tdeSecret[:len(tdePassword)-1] values["tdePassword"] = tdePassword values["tdeKeystorePath"] = pdb.Spec.TDEKeystorePath values["tdeSecret"] = tdeSecret @@ -821,6 +843,7 @@ func (r *PDBReconciler) unplugPDB(ctx context.Context, req ctrl.Request, pdb *db } url := "https://" + pdb.Spec.CDBResName + "-ords:" + strconv.Itoa(cdb.Spec.ORDSPort) + "/ords/_/db-api/latest/database/pdbs/" + pdb.Spec.PDBName + "/" + log.Info("CallAPI(url)", "url", url) pdb.Status.Phase = pdbPhaseUnplug pdb.Status.Msg = "Waiting for PDB to be unplugged" From f3d13a8837dbb6a305072d9b35e50eb31ad6e606 Mon Sep 17 00:00:00 2001 From: matteo malvezzi Date: Fri, 2 Dec 2022 10:55:26 +0100 Subject: [PATCH 35/37] Bug 34855542 - WRONG DELETE PAYLOAD DEFINITION IN THE PDB CONTROLLER --- controllers/database/pdb_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/database/pdb_controller.go b/controllers/database/pdb_controller.go index 05e550c3..0b52fd88 100644 --- a/controllers/database/pdb_controller.go +++ b/controllers/database/pdb_controller.go @@ -1121,7 +1121,7 @@ func (r *PDBReconciler) deletePDBInstance(req ctrl.Request, ctx context.Context, } values := map[string]string{ - "method": "DELETE", + "action": "KEEP", "getScript": strconv.FormatBool(*(pdb.Spec.GetScript))} if pdb.Spec.DropAction != "" { From 4a72573b0ab8e30764c95b7f81088ebf425cfffc Mon Sep 17 00:00:00 2001 From: Param Saini Date: Wed, 7 Dec 2022 21:55:05 -0800 Subject: [PATCH 36/37] Added Oracle Database Operator 0.2.1 --- .gitlab-ci.yml | 30 ------------------------------ oracle-database-operator.yaml | 2 +- 2 files changed, 1 insertion(+), 31 deletions(-) delete mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index cbef6241..00000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,30 +0,0 @@ -build-operator: - stage: build - variables: - IMAGE: "$DOCKER_REPO:$CI_COMMIT_BRANCH" - OP_YAML: oracle-database-operator.yaml - script: - - echo $CI_COMMIT_TAG - - make docker-build IMG="$IMAGE" - - docker push "$IMAGE" - - newimage=$(docker inspect $IMAGE | python -c 'import json,sys; print json.load(sys.stdin)[0]["RepoDigests"][0]') - - echo $newimage - - docker rmi "$IMAGE" && docker system prune -f - - make operator-yaml IMG=$newimage - - if [ "$CI_COMMIT_BRANCH" != "master" ]; then sed -i "s/\(replicas.\) 3/\1 1/g" ./$OP_YAML; fi - - curl -s -n $ARTIFACTORY_REPO/$CI_COMMIT_BRANCH/$OP_YAML -T ./$OP_YAML - only: - variables: - - $CI_COMMIT_MESSAGE =~ /\#run-pipeline/ - - $CI_COMMIT_BRANCH =~ /master/ - - $CI_MERGE_REQUEST_ID != "" - except: - variables: - - $CI_COMMIT_MESSAGE =~ /\#skip-pipeline/ - - $CI_COMMIT_TAG != null - -cleanup: - stage: .post - script: - - echo "Clean up downloaded binaries" - - rm -rf bin/ diff --git a/oracle-database-operator.yaml b/oracle-database-operator.yaml index 309cbcb0..e92f842d 100644 --- a/oracle-database-operator.yaml +++ b/oracle-database-operator.yaml @@ -3224,7 +3224,7 @@ spec: - --enable-leader-election command: - /manager - image: lin.ocir.io/intsanjaysingh/mmalvezz/testppr/myoperator:latest + image: container-registry.oracle.com/database/operator:0.2.1 imagePullPolicy: Always name: manager ports: From 2847800b6caa13617f722f59353fe38f2829568b Mon Sep 17 00:00:00 2001 From: Param Saini Date: Wed, 7 Dec 2022 22:07:52 -0800 Subject: [PATCH 37/37] Added Oracle Database Operator 0.2.1 --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1c091927..2a9595cf 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ As part of Oracle's resolution to make Oracle Database Kubernetes-native (that is, observable and operable by Kubernetes), Oracle released _Oracle Database Operator for Kubernetes_ (`OraOperator` or the operator). OraOperator extends the Kubernetes API with custom resources and controllers for automating Oracle Database lifecycle management. -In this v0.2.0 release, `OraOperator` supports the following database configurations and infrastructure: +In this v0.2.1 release, `OraOperator` supports the following database configurations and infrastructure: * Oracle Autonomous Database on shared Oracle Cloud Infrastructure (OCI) (ADB-S) * Oracle Autonomous Database on dedicated Cloud infrastructure (ADB-D) @@ -32,7 +32,7 @@ The upcoming releases will support new configurations, operations and capabiliti ## Release Status -**CAUTION:** The current release of `OraOperator` (v0.2.0) is for development and testing only. DO NOT USE IN PRODUCTION. +**CAUTION:** The current release of `OraOperator` (v0.2.1) is for development and testing only. DO NOT USE IN PRODUCTION. This release has been installed and tested on the following Kubernetes platforms: @@ -69,7 +69,7 @@ Oracle strongly recommends that you ensure your system meets the following [Prer ``` --- **NOTE:** - The above command will also upgrade the existing v0.1.0 `OraOperator` installation to the latest version i.e. v0.2.0. + The above command will also upgrade the existing v0.2.0 `OraOperator` installation to the latest version i.e. v0.2.1. ---