From 2235b09fd3f7bfe5930570d1c39104a31595e918 Mon Sep 17 00:00:00 2001 From: mnivedithaa Date: Thu, 30 Jan 2025 15:44:28 +0530 Subject: [PATCH] [minor] Add automation for ReplicaDB in gitops (#1452) --- .secrets.baseline | 4 +- .../cli/mascli/functions/gitops_db2u_database | 35 ++++- .../gitops_deprovision_db2u_database | 1 + .../db2-databases/ibm-db2u-database.yaml.j2 | 6 + .../pipelines/gitops/gitops-mas-apps.yml.j2 | 130 ++++++++++++++++++ .../tasks/gitops/gitops-db2u-database.yml.j2 | 15 ++ 6 files changed, 186 insertions(+), 5 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index be5f53234c..0ab694e7f8 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -3,7 +3,7 @@ "files": "^.secrets.baseline$", "lines": null }, - "generated_at": "2025-01-14T17:01:44Z", + "generated_at": "2025-01-23T11:46:57Z", "plugins_used": [ { "name": "AWSKeyDetector" @@ -192,7 +192,7 @@ "hashed_secret": "1459943ba5fd876f7ef6e48f566a40b448a2bf08", "is_secret": false, "is_verified": false, - "line_number": 457, + "line_number": 469, "type": "Secret Keyword", "verified_result": null } diff --git a/image/cli/mascli/functions/gitops_db2u_database b/image/cli/mascli/functions/gitops_db2u_database index 3524c476f7..9722e48eaa 100644 --- a/image/cli/mascli/functions/gitops_db2u_database +++ b/image/cli/mascli/functions/gitops_db2u_database @@ -62,6 +62,9 @@ IBM DB2U: --jdbc-route ${COLOR_YELLOW}JDBC_ROUTE${TEXT_RESET} By default routes are not exposed to public. To expose route, set this to public. --db2-timezone ${COLOR_YELLOW}DB2_TIMEZONE${TEXT_RESET} DB2 DB Timezone (optional, DB2 default of "UTC" will be used if not specified) --db2-backup-notify-slack-url ${COLOR_YELLOW}DB2_BACKUP_NOTIFY_SLACK_URL${TEXT_RESET} Slack URL to notify DB2 backup failures + --replica-db ${COLOR_YELLOW}REPLICA_DB${TEXT_RESET} Flag to know that this is ReplicaDB instance + --is-replica-task ${COLOR_YELLOW}IS_REPLICA_TASK${TEXT_RESET} To check if this is invoked by replica task + --mas-annotations ${COLOR_YELLOW}MAS_ANNOTATIONS${TEXT_RESET} MAS Annotations Secrets Manager: --secrets-path ${COLOR_YELLOW}SECRETS_PATH${TEXT_RESET} Secrets Manager path @@ -125,6 +128,9 @@ function gitops_db2u_database_noninteractive() { --mas-app-id) export MAS_APP_ID=$1 && shift ;; + --mas-annotations) + export MAS_ANNOTATIONS=$1 && shift + ;; # Secrets Manager --secrets-path) @@ -277,6 +283,12 @@ function gitops_db2u_database_noninteractive() { --db2-backup-notify-slack-url) export DB2_BACKUP_NOTIFY_SLACK_URL=$1 && shift ;; + --replica-db) + export REPLICA_DB=$1 && shift + ;; + --is-replica-task) + export IS_REPLICA_TASK=$1 && shift + ;; # Automatic GitHub Push @@ -459,8 +471,18 @@ DB2_4K_DEVICE_SUPPORT: '${DB2_4K_DEVICE_SUPPORT}' DB2_FMP_RUN_AS_CONNECTED_USER: 'NO' DB2_WORKLOAD: '${DB2_WORKLOAD}'" fi + if [[ -z $REPLICA_DB ]]; then + export REPLICA_DB="false" + fi + if [[ -z $IS_REPLICA_TASK ]]; then + export IS_REPLICA_TASK="false" + fi if [[ -z $DB2_INSTANCE_NAME ]]; then - export DB2_INSTANCE_NAME=db2wh-${MAS_INSTANCE_ID}-${MAS_APP_ID} + if [[ $IS_REPLICA_TASK == 'true' ]]; then + export DB2_INSTANCE_NAME=db2wh-${MAS_INSTANCE_ID}-${MAS_APP_ID}-sdb + else + export DB2_INSTANCE_NAME=db2wh-${MAS_INSTANCE_ID}-${MAS_APP_ID} + fi fi if [[ -z $DB2_BACKUP_BUCKET_NAME ]]; then export DB2_BACKUP_BUCKET_NAME=${SECRET_NAME_DB2_BACKUP}#bucketName @@ -570,6 +592,9 @@ DB2_WORKLOAD: '${DB2_WORKLOAD}'" echo_reset_dim "db2-tolerate-effect ........................... ${COLOR_MAGENTA}${DB2_TOLERATE_EFFECT}" echo_reset_dim "JDBC_ROUTE .................................... ${COLOR_MAGENTA}${JDBC_ROUTE}" echo_reset_dim "db2-timezone .................................. ${COLOR_MAGENTA}${DB2_TIMEZONE}" + echo_reset_dim "replica-db .................................... ${COLOR_MAGENTA}${REPLICA_DB}" + echo_reset_dim "is-replica-task ............................... ${COLOR_MAGENTA}${IS_REPLICA_TASK}" + echo_reset_dim "mas-annotations ............................... ${COLOR_MAGENTA}${MAS_ANNOTATIONS}" reset_colors @@ -701,8 +726,12 @@ DB2_WORKLOAD: '${DB2_WORKLOAD}'" # Remove any existing config with this name yq 'del(.ibm_db2u_databases[] | select(.db2_instance_name == "'${DB2_INSTANCE_NAME}'"))' $CONFIGS_FILE > $TEMP_DIR/configs.yaml - # Render the appropriate template for the config into a new file - jinja -X .+ $CLI_DIR/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 | yq '{"ibm_db2u_databases": [] + .}' > ${TEMP_DIR}/newconfig.yaml + # Create new file + echo -n "" > ${TEMP_DIR}/newconfig.yaml + # Render the appropriate template for the config into a new file. For standby database, it will be generated only if it is a replica task and replica is enabled. + if [[ (${IS_REPLICA_TASK} == 'false') || (${IS_REPLICA_TASK} == 'true' && ${REPLICA_DB} == 'true') ]]; then + jinja -X .+ $CLI_DIR/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 | yq '{"ibm_db2u_databases": [] + .}' > ${TEMP_DIR}/newconfig.yaml + fi # Merge the two files yq eval-all '. as $item ireduce ({}; . *+ $item)' $TEMP_DIR/configs.yaml ${TEMP_DIR}/newconfig.yaml > $CONFIGS_FILE diff --git a/image/cli/mascli/functions/gitops_deprovision_db2u_database b/image/cli/mascli/functions/gitops_deprovision_db2u_database index 38dbf24681..4906b2592f 100644 --- a/image/cli/mascli/functions/gitops_deprovision_db2u_database +++ b/image/cli/mascli/functions/gitops_deprovision_db2u_database @@ -240,6 +240,7 @@ function gitops_deprovision_db2u_database() { # If the file doesn't exist, nothing to remove, so no-op if [ -f ${CONFIGS_FILE} ]; then yq 'del(.ibm_db2u_databases[] | select(.db2_instance_name == "'${DB2_INSTANCE_NAME}'"))' $CONFIGS_FILE > ${TEMP_DIR}/configs.yaml + cp ${TEMP_DIR}/configs.yaml ${CONFIGS_FILE} # If the file is there, but the configs are empty, delete the file diff --git a/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 b/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 index 55dae1a23e..90a259e036 100644 --- a/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 +++ b/image/cli/mascli/templates/gitops/appset-configs/cluster/instance/db2-databases/ibm-db2u-database.yaml.j2 @@ -64,6 +64,12 @@ db2_tolerate_value: {{DB2_TOLERATE_VALUE}} db2_tolerate_effect: {{DB2_TOLERATE_EFFECT}} cluster_domain: mas_application_id: {{MAS_APP_ID}} +{%- if MAS_ANNOTATIONS is defined and MAS_ANNOTATIONS !='' %} +mas_annotations: + {% filter indent(width=2) -%} + {{MAS_ANNOTATIONS}} + {%- endfilter %} +{%- endif %} jdbc_route: {{JDBC_ROUTE}} db2_timezone: {{DB2_TIMEZONE}} diff --git a/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 b/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 index 5cfb62850b..f37fc5c53b 100644 --- a/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 +++ b/tekton/src/pipelines/gitops/gitops-mas-apps.yml.j2 @@ -41,6 +41,9 @@ spec: default: "" - name: mas_config_dir type: string + - name: mas_annotations + type: string + default: "" - name: github_url type: string @@ -212,6 +215,9 @@ spec: - name: db2_backup_notify_slack_url type: string default: "" + - name: replica_db + type: string + default: "false" - name: custom_labels type: string @@ -879,6 +885,130 @@ spec: - input: "$(params.mas_app_channel_manage)" operator: notin values: [""] + - name: gitops-db2u-replica-database-manage + params: + {{ lookup('template', pipeline_src_dir ~ '/taskdefs/gitops/common/gitops-params.yml.j2') | indent(8) }} + + {{ lookup('template', pipeline_src_dir ~ '/taskdefs/gitops/common/secrets-params.yml.j2') | indent(8) }} + + {{ lookup('template', pipeline_src_dir ~ '/taskdefs/gitops/common/git-params.yml.j2') | indent(8) }} + + - name: mas_instance_id + value: $(params.mas_instance_id) + - name: mas_app_id + value: manage + + - name: db2_meta_storage_class + value: $(params.db2_meta_storage_class) + - name: db2_temp_storage_class + value: $(params.db2_temp_storage_class) + - name: db2_logs_storage_class + value: $(params.db2_logs_storage_class) + - name: db2_audit_logs_storage_class + value: $(params.db2_audit_logs_storage_class) + - name: db2_data_storage_class + value: $(params.db2_data_storage_class) + - name: db2_backup_storage_class + value: $(params.db2_backup_storage_class) + - name: db2_archivelogs_storage_class + value: $(params.db2_archivelogs_storage_class) + + - name: db2_meta_storage_size + value: $(params.db2_meta_storage_size) + - name: db2_temp_storage_size + value: $(params.db2_temp_storage_size) + - name: db2_logs_storage_size + value: $(params.db2_logs_storage_size) + - name: db2_audit_logs_storage_size + value: $(params.db2_audit_logs_storage_size) + - name: db2_data_storage_size + value: $(params.db2_data_storage_size) + - name: db2_backup_storage_size + value: $(params.db2_backup_storage_size) + - name: db2_archivelogs_storage_size + value: $(params.db2_archivelogs_storage_size) + + - name: db2_version + value: $(params.db2_version) + - name: db2_tls_version + value: $(params.db2_tls_version) + - name: db2_instance_registry_yaml + value: $(params.db2_instance_registry_yaml_manage) + - name: db2_instance_dbm_config_yaml + value: $(params.db2_instance_dbm_config_yaml_manage) + - name: db2_database_db_config_yaml + value: $(params.db2_database_db_config_yaml_manage) + - name: db2_addons_audit_config_yaml + value: $(params.db2_addons_audit_config_yaml_manage) + - name: db2_table_org + value: $(params.db2_table_org) + - name: db2_mln_count + value: $(params.db2_mln_count) + - name: db2_num_pods + value: $(params.db2_num_pods) + - name: db2_meta_storage_accessmode + value: $(params.db2_meta_storage_accessmode) + - name: db2_data_storage_accessmode + value: $(params.db2_data_storage_accessmode) + - name: db2_backup_storage_accessmode + value: $(params.db2_backup_storage_accessmode) + - name: db2_logs_storage_accessmode + value: $(params.db2_logs_storage_accessmode) + - name: db2_audit_logs_storage_accessmode + value: $(params.db2_audit_logs_storage_accessmode) + - name: db2_temp_storage_accessmode + value: $(params.db2_temp_storage_accessmode) + - name: db2_archivelogs_storage_accessmode + value: $(params.db2_archivelogs_storage_accessmode) + - name: db2_cpu_requests + value: $(params.db2_cpu_requests) + - name: db2_cpu_limits + value: $(params.db2_cpu_limits) + - name: db2_memory_requests + value: $(params.db2_memory_requests) + - name: db2_memory_limits + value: $(params.db2_memory_limits) + - name: db2_affinity_key + value: $(params.db2_affinity_key) + - name: db2_affinity_value + value: $(params.db2_affinity_value) + - name: db2_tolerate_key + value: $(params.db2_tolerate_key) + - name: db2_tolerate_value + value: $(params.db2_tolerate_value) + - name: db2_tolerate_effect + value: $(params.db2_tolerate_effect) + - name: jdbc_route + value: $(params.jdbc_route_manage) + - name: db2_timezone + value: $(params.db2_timezone) + - name: db2_backup_notify_slack_url + value: $(params.db2_backup_notify_slack_url) + - name: replica_db + value: $(params.replica_db) + - name: mas_annotations + value: $(params.mas_annotations) + - name: is_replica_task + value: 'true' + + workspaces: + - name: configs + workspace: configs + - name: shared-gitops-configs + workspace: shared-gitops-configs + taskRef: + kind: Task + name: gitops-db2u-database + when: + - input: "$(params.db2_action)" + operator: notin + values: [""] + - input: "$(params.jdbc_type_manage)" + operator: in + values: ["incluster-db2"] + - input: "$(params.mas_app_channel_manage)" + operator: notin + values: [""] # 4. Install & Configure IoT # ------------------------------------------------------------------------- diff --git a/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 b/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 index ecef563478..8f384a38c3 100644 --- a/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 +++ b/tekton/src/tasks/gitops/gitops-db2u-database.yml.j2 @@ -114,6 +114,15 @@ spec: - name: db2_backup_notify_slack_url type: string default: "" + - name: replica_db + type: string + default: "false" + - name: is_replica_task + type: string + default: "false" + - name: mas_annotations + type: string + default: "" stepTemplate: name: gitops-db2u-database env: @@ -223,6 +232,12 @@ spec: value: $(params.jdbc_route) - name: DB2_BACKUP_NOTIFY_SLACK_URL value: $(params.db2_backup_notify_slack_url) + - name: REPLICA_DB + value: $(params.replica_db) + - name: IS_REPLICA_TASK + value: $(params.is_replica_task) + - name: MAS_ANNOTATIONS + value: $(params.mas_annotations) envFrom: - configMapRef: name: environment-properties