diff --git a/image/cli/install/ibm-mas_devops.tar.gz b/image/cli/install/ibm-mas_devops.tar.gz new file mode 100644 index 0000000000..7eacc0542e Binary files /dev/null and b/image/cli/install/ibm-mas_devops.tar.gz differ diff --git a/image/cli/install/mas_devops.tar.gz b/image/cli/install/mas_devops.tar.gz new file mode 100644 index 0000000000..48e6260c2a Binary files /dev/null and b/image/cli/install/mas_devops.tar.gz differ diff --git a/python/src/mas/cli/cli.py b/python/src/mas/cli/cli.py index 533095ce1f..6edb55e1ff 100644 --- a/python/src/mas/cli/cli.py +++ b/python/src/mas/cli/cli.py @@ -132,9 +132,10 @@ def __init__(self): # Initialize the dictionary that will hold the parameters we pass to a PipelineRun self.params = dict() - # These dicts will hold the additional-configs, pod-templates and manual certificates secrets + # These dicts will hold the additional-configs, pod-templates, sls license file and manual certificates secrets self.additionalConfigsSecret = None self.podTemplatesSecret = None + self.slsLicenseFileSecret = None self.certsSecret = None self._isSNO = None diff --git a/python/src/mas/cli/install/app.py b/python/src/mas/cli/install/app.py index 20e909d693..800007d7c5 100644 --- a/python/src/mas/cli/install/app.py +++ b/python/src/mas/cli/install/app.py @@ -29,6 +29,7 @@ from ..gencfg import ConfigGeneratorMixin from .argBuilder import installArgBuilderMixin from .argParser import installArgParser +from .argChecker import verifyArgs from .settings import InstallSettingsMixin from .summarizer import InstallSummarizerMixin from .params import requiredParams, optionalParams @@ -40,11 +41,15 @@ WorkspaceNameFormatValidator, TimeoutFormatValidator, StorageClassValidator, - OptimizerInstallPlanValidator + OptimizerInstallPlanValidator, + SLSConfigValidator, + SLSInstanceSelectionValidator, + NewNamespaceValidator ) from mas.devops.ocp import createNamespace, getStorageClasses from mas.devops.mas import getCurrentCatalog, getDefaultStorageClasses +from mas.devops.sls import listSLSInstances, verifySLSConnection, findSLSByNamespace from mas.devops.data import getCatalog from mas.devops.tekton import ( installOpenShiftPipelines, @@ -249,16 +254,102 @@ def configCatalog(self): self.setParam("mas_channel", releaseSelection) + @logMethodCall + def validateExternalSLSConnection(self) -> None: + self.verifySLSConnection = verifySLSConnection(self.getParam("sls_url"), self.slsCertsDirLocal + 'ca.crt') + + if not self.noConfirm: + if not self.verifySLSConnection: + if not self.yesOrNo("Could not verify SLS connection, proceed anyway"): + exit(1) + @logMethodCall def configSLS(self) -> None: self.printH1("Configure Product License") - self.slsLicenseFileLocal = self.promptForFile("License file", mustExist=True, envVar="SLS_LICENSE_FILE_LOCAL") + + self.slsLicenseFileLocal = None + self.slsCertsDirLocal = None + existingSLSInstances = listSLSInstances(self.dynamicClient) + numSLSInstances = len(existingSLSInstances) + description = [ + "IBM Suite License Service (SLS) is the licensing enforcement for Maximo Application Suite.", + "" + ] + + if self.showAdvancedOptions: + self.slsConfigOptions = [] + self.slsInstanceOptions = [] + + description.insert(1, "Choose how to configure SLS:") + description.insert(2, " - New: Deploy a new instance on the cluster.") + description.insert(3, " - External: Point to an external instance outside of the cluster.") + + self.slsConfigOptions.append("New") + self.slsConfigOptions.append("External") + + if numSLSInstances > 0: + self.slsConfigOptions.insert(1, "Existing") + description.insert(3, " - Existing: Select an existing instance on the cluster. This is useful for sharing SLS with multiple MAS instances.") + + slsConfigCompleter = WordCompleter(self.slsConfigOptions) + + self.printDescription(description) + slsConfigSelection = self.promptForString("Select SLS config option", completer=slsConfigCompleter, validator=SLSConfigValidator()) + + if slsConfigSelection == "New": + self.setParam("sls_action", "install") + defaultVal = "ibm-sls" + if numSLSInstances > 0: + defaultVal = "" + self.promptForString("SLS namespace", "sls_namespace", default=defaultVal, validator=NewNamespaceValidator()) + self.slsLicenseFileLocal = self.promptForFile("License file", mustExist=True, envVar="SLS_LICENSE_FILE_LOCAL") + + if slsConfigSelection == "Existing": + self.setParam("sls_action", "gencfg") + print_formatted_text(HTML("Select an existing SLS instance from the list below:")) + for slsInstance in existingSLSInstances: + print_formatted_text(HTML(f"- {slsInstance['metadata']['namespace']} | {slsInstance['metadata']['name']} | v{slsInstance['status']['versions']['reconciled']}")) + self.slsInstanceOptions.append(slsInstance['metadata']['namespace']) + slsInstanceCompleter = WordCompleter(self.slsInstanceOptions) + print() + self.promptForString("Select SLS namespace", "sls_namespace", completer=slsInstanceCompleter, validator=SLSInstanceSelectionValidator()) + if self.yesOrNo("Upload/Replace the license file"): + self.slsLicenseFileLocal = self.promptForFile("License file", mustExist=True, envVar="SLS_LICENSE_FILE_LOCAL") + self.setParam("sls_action", "install") + + if slsConfigSelection == "External": + self.setParam("sls_action", "gencfg") + self.promptForString("SLS url", "sls_url") + self.promptForString("SLS registrationKey", "sls_registration_key") + self.slsCertsDirLocal = self.promptForDir("Enter the path containing the SLS certificate(s)", mustExist=True) + self.validateExternalSLSConnection() + + else: + self.setParam("sls_action", "install") + sls_default_namespace = "ibm-sls" + self.setParam("sls_namespace", sls_default_namespace) + + if numSLSInstances == 0: + description.insert(1, f"A new instance of SLS will be deployed on the cluster in the namespace '{sls_default_namespace}'.") + self.printDescription(description) + self.slsLicenseFileLocal = self.promptForFile("License file", mustExist=True, envVar="SLS_LICENSE_FILE_LOCAL") + + if numSLSInstances > 0: + self.printDescription(description) + if findSLSByNamespace(sls_default_namespace, instances=existingSLSInstances): + print_formatted_text(HTML(f"SLS auto-detected: {sls_default_namespace}")) + if self.yesOrNo("Upload/Replace the license file"): + self.slsLicenseFileLocal = self.promptForFile("License file", mustExist=True, envVar="SLS_LICENSE_FILE_LOCAL") + else: + self.setParam("sls_action", "gencfg") + + @logMethodCall + def configDRO(self) -> None: self.promptForString("Contact e-mail address", "uds_contact_email") self.promptForString("Contact first name", "uds_contact_firstname") self.promptForString("Contact last name", "uds_contact_lastname") if self.showAdvancedOptions: - self.promptForString("IBM Suite License Services (SLS) Namespace", "sls_namespace", default="ibm-sls") self.promptForString("IBM Data Reporter Operator (DRO) Namespace", "dro_namespace", default="redhat-marketplace") @logMethodCall @@ -730,6 +821,7 @@ def interactiveMode(self, simplified: bool, advanced: bool) -> None: # Licensing (SLS and DRO) self.configSLS() + self.configDRO() self.configICRCredentials() # MAS Core @@ -896,10 +988,6 @@ def nonInteractiveMode(self) -> None: if value is None: self.fatalError(f"{key} must be set") self.pipelineStorageClass = value - elif key == "license_file": - if value is None: - self.fatalError(f"{key} must be set") - self.slsLicenseFileLocal = value elif key.startswith("approval_"): if key not in self.approvals: @@ -929,6 +1017,26 @@ def nonInteractiveMode(self) -> None: self.setParam("mas_manual_cert_mgmt", False) self.manualCertsDir = None + # SLS arguments + elif key == "sls_namespace": + if value is not None and value != "": + self.setParam("sls_namespace", value) + if findSLSByNamespace(value, dynClient= self.dynamicClient): + self.setParam("sls_action", "gencfg") + else: + self.setParam("sls_action", "install") + else: + self.setParam("sls_namespace", "ibm-sls") + self.setParam("sls_action", "gencfg") + elif key == "license_file": + if value is not None and value != "": + self.slsLicenseFileLocal = value + self.setParam("sls_action", "install") + elif key == "sls_certificates": + if value is not None and value != "": + self.slsCertsDirLocal = value + self.setParam("sls_action", "gencfg") + # Fail if there's any arguments we don't know how to handle else: print(f"Unknown option: {key} {value}") @@ -941,6 +1049,8 @@ def nonInteractiveMode(self) -> None: if not self.devMode: self.validateCatalogSource() self.licensePrompt() + + self.validateExternalSLSConnection() @logMethodCall def install(self, argv): @@ -948,6 +1058,7 @@ def install(self, argv): Install MAS instance """ args = installArgParser.parse_args(args=argv) + verifyArgs(installArgParser, args) # We use the presence of --mas-instance-id to determine whether # the CLI is being started in interactive mode or not @@ -960,6 +1071,10 @@ def install(self, argv): self.devMode = args.dev_mode self.skipGrafanaInstall = args.skip_grafana_install + # Set image_pull_policy of the CLI in interactive mode + if args.image_pull_policy and args.image_pull_policy != "": + self.setParam("image_pull_policy", args.image_pull_policy) + self.approvals = {} # Store all args @@ -1007,13 +1122,10 @@ def install(self, argv): if self.deployCP4D: self.configCP4D() - # The entitlement file for SLS is mounted as a secret in /workspace/entitlement - entitlementFileBaseName = path.basename(self.slsLicenseFileLocal) - self.setParam("sls_entitlement_file", f"/workspace/entitlement/{entitlementFileBaseName}") - - # Set up the secrets for additional configs, podtemplates and manual certificates + # Set up the secrets for additional configs, podtemplates, sls license file and manual certificates self.additionalConfigs() self.podTemplates() + self.slsLicenseFile() self.manualCertificates() # Show a summary of the installation configuration @@ -1065,7 +1177,7 @@ def install(self, argv): prepareInstallSecrets( dynClient=self.dynamicClient, instanceId=self.getParam("mas_instance_id"), - slsLicenseFile=self.slsLicenseFileLocal, + slsLicenseFile=self.slsLicenseFileSecret, additionalConfigs=self.additionalConfigsSecret, podTemplates=self.podTemplatesSecret, certs=self.certsSecret diff --git a/python/src/mas/cli/install/argBuilder.py b/python/src/mas/cli/install/argBuilder.py index 8574547885..49ff7be0a9 100644 --- a/python/src/mas/cli/install/argBuilder.py +++ b/python/src/mas/cli/install/argBuilder.py @@ -122,9 +122,15 @@ def buildCommand(self) -> str: # IBM Suite License Service # ----------------------------------------------------------------------------- - command += f" --license-file \"{self.slsLicenseFileLocal}\"{newline}" - if self.getParam("sls_namespace") != "ibm-sls": + if self.getParam("sls_namespace") and self.getParam("sls_namespace") != "ibm-sls": command += f" --sls-namespace \"{self.getParam('sls_namespace')}\"{newline}" + if self.slsLicenseFileLocal: + command += f" --license-file \"{self.slsLicenseFileLocal}\"{newline}" + # External SLS configuration + if self.getParam("sls_url"): + command += f" --sls-url \"{self.getParam('sls_url')}\"{newline}" + command += f" --sls-registration-key \"{self.getParam('sls_registration_key')}\"{newline}" + command += f" --sls-certificates \"{self.slsCertsDirLocal}\"{newline}" # IBM Data Reporting Operator (DRO) # ----------------------------------------------------------------------------- diff --git a/python/src/mas/cli/install/argChecker.py b/python/src/mas/cli/install/argChecker.py new file mode 100644 index 0000000000..9339a1f9cf --- /dev/null +++ b/python/src/mas/cli/install/argChecker.py @@ -0,0 +1,27 @@ +# ***************************************************************************** +# Copyright (c) 2025 IBM Corporation and other Contributors. +# +# All rights reserved. This program and the accompanying materials +# are made available under the terms of the Eclipse Public License v1.0 +# which accompanies this distribution, and is available at +# http://www.eclipse.org/legal/epl-v10.html +# +# ***************************************************************************** + +import logging + +logger = logging.getLogger(__name__) + + +def verifyArgs(parser, args): + verifySLSArgs(parser, args) + +def verifySLSArgs(parser, args): + group_1 = [args.sls_namespace, args.license_file] + group_2 = [args.sls_url, args.sls_registration_key, args.sls_certificates] + + if any(v is not None for v in group_1) and any(v is not None for v in group_2): + parser.error("Cannot combine [--sls-namespace, --license-file] with [--sls-url, --sls-registration-key, --sls-certificates].") + + if not all(v is not None for v in group_2) and any(v is not None for v in group_2): + parser.error("When providing any of --sls-url, --sls-registration-key and --sls-certificates, all three are required.") diff --git a/python/src/mas/cli/install/argParser.py b/python/src/mas/cli/install/argParser.py index 31f407dccb..400ee0b085 100644 --- a/python/src/mas/cli/install/argParser.py +++ b/python/src/mas/cli/install/argParser.py @@ -21,6 +21,11 @@ def isValidFile(parser, arg) -> str: else: return arg +def isValidDir(parser, arg) -> str: + if not path.isdir(arg): + parser.error(f"Error: The directory {arg} does not exist") + else: + return arg installArgParser = argparse.ArgumentParser( prog="mas install", @@ -260,8 +265,23 @@ def isValidFile(parser, arg) -> str: slsArgGroup.add_argument( "--sls-namespace", required=False, - help="Customize the SLS install namespace", - default="ibm-sls" + help="Set namespace for new SLS install or point to existing instance", +) +slsArgGroup.add_argument( + "--sls-url", + required=False, + help="Point to the external SLS URL", +) +slsArgGroup.add_argument( + "--sls-registration-key", + required=False, + help="Set the SLS RegistrationKey", +) +slsArgGroup.add_argument( + "--sls-certificates", + required=False, + help="Path to SLS certificates", + type=lambda x: isValidDir(installArgParser, x) ) # IBM Data Reporting Operator (DRO) diff --git a/python/src/mas/cli/install/params.py b/python/src/mas/cli/install/params.py index c0beb58f93..28931d7ebc 100644 --- a/python/src/mas/cli/install/params.py +++ b/python/src/mas/cli/install/params.py @@ -59,7 +59,8 @@ "mas_appws_components", "mas_domain", # SLS - "sls_namespace", + "sls_url", + "sls_registration_key", # DNS Providers # TODO: Add CloudFlare and Route53 support "dns_provider", diff --git a/python/src/mas/cli/install/settings/additionalConfigs.py b/python/src/mas/cli/install/settings/additionalConfigs.py index 9c5856d220..5b8e74a990 100644 --- a/python/src/mas/cli/install/settings/additionalConfigs.py +++ b/python/src/mas/cli/install/settings/additionalConfigs.py @@ -122,19 +122,17 @@ def podTemplates(self) -> None: self.podTemplatesSecret = podTemplatesSecret def manualCertificates(self) -> None: - - if self.getParam("mas_manual_cert_mgmt"): - certsSecret = { - "apiVersion": "v1", - "kind": "Secret", - "type": "Opaque", - "metadata": { - "name": "pipeline-certificates" - } + certsSecret = { + "apiVersion": "v1", + "kind": "Secret", + "type": "Opaque", + "metadata": { + "name": "pipeline-certificates" } + } + extensions = ["key", "crt"] - extensions = ["key", "crt"] - + if self.getParam("mas_manual_cert_mgmt"): apps = { "mas_app_channel_assist": { "dir": self.manualCertsDir + "/assist/", @@ -182,7 +180,32 @@ def manualCertificates(self) -> None: self.certsSecret = certsSecret - def addFilesToSecret(self, secretDict: dict, configPath: str, extension: str, keyPrefix: str = '') -> dict: + if self.slsCertsDirLocal: + # Currently SLS only needs ca.crt + for file in ["ca.crt"]: + if file not in map(path.basename, glob(f'{self.slsCertsDirLocal}/*')): + self.fatalError(f'{file} is not present in {self.slsCertsDirLocal}/') + for ext in extensions: + certsSecret = self.addFilesToSecret(certsSecret, self.slsCertsDirLocal, ext, "sls.") + + # The ca cert for SLS is mounted as a secret in /workspace/certificates + self.setParam("sls_tls_crt_local_file_path", "/workspace/certificates/sls.ca.crt") + self.certsSecret = certsSecret + + def slsLicenseFile(self) -> None: + if self.slsLicenseFileLocal: + slsLicenseFileSecret = { + "apiVersion": "v1", + "kind": "Secret", + "type": "Opaque", + "metadata": { + "name": "pipeline-sls-entitlement" + } + } + self.setParam("sls_entitlement_file", f"/workspace/entitlement/{path.basename(self.slsLicenseFileLocal)}") + self.slsLicenseFileSecret = self.addFilesToSecret(slsLicenseFileSecret, self.slsLicenseFileLocal, '') + + def addFilesToSecret(self, secretDict: dict, configPath: str, extension: str, keyPrefix: str = '', encoding: str = 'ascii') -> dict: """ Add file (or files) to pipeline-additional-configs """ @@ -205,6 +228,6 @@ def addFilesToSecret(self, secretDict: dict, configPath: str, extension: str, ke # Add/update an entry to the secret data if "data" not in secretDict: secretDict["data"] = {} - secretDict["data"][keyPrefix + fileName] = b64encode(data.encode('ascii')).decode("ascii") + secretDict["data"][keyPrefix + fileName] = b64encode(data.encode(encoding)).decode(encoding) return secretDict diff --git a/python/src/mas/cli/install/summarizer.py b/python/src/mas/cli/install/summarizer.py index f2c3dd30f4..6a57335aad 100644 --- a/python/src/mas/cli/install/summarizer.py +++ b/python/src/mas/cli/install/summarizer.py @@ -265,9 +265,18 @@ def droSummary(self) -> None: def slsSummary(self) -> None: self.printH2("IBM Suite License Service") - self.printSummary("License File", self.slsLicenseFileLocal) - self.printParamSummary("IBM Open Registry", "sls_icr_cpopen") - self.printParamSummary("Namespace", "sls_namespace") + if self.getParam("sls_url"): + self.printParamSummary("URL", "sls_url") + self.printParamSummary("RegistrationKey", "sls_registration_key") + self.printSummary("Certificates", self.slsCertsDirLocal) + self.printSummary("Connection verified", self.verifySLSConnection) + else: + self.printParamSummary("Namespace", "sls_namespace") + if self.getParam("sls_action") == "install": + self.printSummary("Subscription Channel", "3.x") + self.printParamSummary("IBM Open Registry", "sls_icr_cpopen") + if self.slsLicenseFileLocal: + self.printSummary("License File", self.slsLicenseFileLocal) def cosSummary(self) -> None: self.printH2("Cloud Object Storage") diff --git a/python/src/mas/cli/validators.py b/python/src/mas/cli/validators.py index 29bada8ee3..1017388892 100644 --- a/python/src/mas/cli/validators.py +++ b/python/src/mas/cli/validators.py @@ -18,8 +18,9 @@ from prompt_toolkit.validation import Validator, ValidationError -from mas.devops.ocp import getStorageClass +from mas.devops.ocp import getStorageClass, getNamespace from mas.devops.mas import verifyMasInstance +from mas.devops.sls import listSLSInstances import logging @@ -136,3 +137,78 @@ def validate(self, document): response = document.text if response not in ["full", "limited"]: raise ValidationError(message='Enter a valid response: full, limited', cursor_position=len(response)) + + +class SLSConfigValidator(Validator): + def validate(self, document): + """ + Validate that a response is a valid config plan for SLS + """ + response = document.text + validOptions = ["New", "External"] + + dynClient = dynamic.DynamicClient( + api_client.ApiClient(configuration=config.load_kube_config()) + ) + existingSLSInstances = listSLSInstances(dynClient) + numSLSInstances = len(existingSLSInstances) + if numSLSInstances > 0: + validOptions.insert(1, "Existing") + + if response not in validOptions: + raise ValidationError( + message=f"Enter a valid response: {', '.join(validOptions)}", + cursor_position=len(response), + ) + + +class SLSInstanceSelectionValidator(Validator): + def validate(self, document): + """ + Validate that a response is a valid SLS instance on the cluster + """ + response = document.text + + if response == "": + raise ValidationError( + message=f"Enter a valid response", cursor_position=len(response) + ) + + validOptions = [] + + dynClient = dynamic.DynamicClient( + api_client.ApiClient(configuration=config.load_kube_config()) + ) + + for instance in listSLSInstances(dynClient): + validOptions.append(instance["metadata"]["namespace"]) + + if response not in validOptions: + raise ValidationError( + message=f"Enter a valid response", + cursor_position=len(response), + ) + + +class NewNamespaceValidator(Validator): + def validate(self, document): + """ + Validate that a namespace does not exist + """ + namespace = document.text + + # ToDo: Add namespace regex validation + if namespace == "": + raise ValidationError( + message=f"Namespace cannot be empty", cursor_position=len(namespace) + ) + + dynClient = dynamic.DynamicClient( + api_client.ApiClient(configuration=config.load_kube_config()) + ) + + if getNamespace(dynClient, namespace): + raise ValidationError( + message=f"Namespace '{namespace}' already exists on the cluster, must be unique", + cursor_position=len(namespace), + ) diff --git a/tekton/src/params/install.yml.j2 b/tekton/src/params/install.yml.j2 index b01b6836d5..96b033a4fa 100644 --- a/tekton/src/params/install.yml.j2 +++ b/tekton/src/params/install.yml.j2 @@ -27,7 +27,7 @@ default: "" - name: sls_entitlement_file type: string - default: "/workspace/entitlement/entitlement.lic" + default: "" - name: sls_mongodb_cfg_file type: string # The default value works for the default in-cluster install, it will need @@ -42,6 +42,18 @@ - name: sls_icr_cpopen type: string default: "" +- name: sls_action + type: string + default: "" +- name: sls_url + type: string + default: "" +- name: sls_registration_key + type: string + default: "" +- name: sls_tls_crt_local_file_path + type: string + default: "" # Dependencies - MongoDb # ----------------------------------------------------------------------------- diff --git a/tekton/src/pipelines/taskdefs/dependencies/sls.yml.j2 b/tekton/src/pipelines/taskdefs/dependencies/sls.yml.j2 index 14b89de145..71ee72bffb 100644 --- a/tekton/src/pipelines/taskdefs/dependencies/sls.yml.j2 +++ b/tekton/src/pipelines/taskdefs/dependencies/sls.yml.j2 @@ -23,6 +23,15 @@ value: $(params.sls_channel) - name: sls_icr_cpopen value: $(params.sls_icr_cpopen) + - name: sls_action + value: $(params.sls_action) + + - name: sls_url + value: $(params.sls_url) + - name: sls_registration_key + value: $(params.sls_registration_key) + - name: sls_tls_crt_local_file_path + value: $(params.sls_tls_crt_local_file_path) # New way of bootstrapping license file - name: sls_entitlement_file @@ -44,3 +53,5 @@ workspace: shared-entitlement - name: pod-templates workspace: shared-pod-templates + - name: certificates + workspace: shared-certificates diff --git a/tekton/src/tasks/dependencies/sls.yml.j2 b/tekton/src/tasks/dependencies/sls.yml.j2 index a6f25ba7af..ccaa643af2 100644 --- a/tekton/src/tasks/dependencies/sls.yml.j2 +++ b/tekton/src/tasks/dependencies/sls.yml.j2 @@ -32,6 +32,16 @@ spec: type: string default: "" + - name: sls_url + type: string + default: "" + - name: sls_registration_key + type: string + default: "" + - name: sls_tls_crt_local_file_path + type: string + default: "" + # New way of bootstrapping license file since SLS 3.7.0 - name: sls_entitlement_file type: string @@ -80,6 +90,13 @@ spec: - name: SLS_ICR_CPOPEN value: $(params.sls_icr_cpopen) + - name: SLS_URL + value: $(params.sls_url) + - name: SLS_REGISTRATION_KEY + value: $(params.sls_registration_key) + - name: SLS_TLS_CERT_LOCAL_FILE_BASE64_PATH + value: $(params.sls_tls_crt_local_file_path) + # New way of bootstrapping license file since SLS 3.7.0 - name: SLS_ENTITLEMENT_FILE value: $(params.sls_entitlement_file) @@ -118,3 +135,5 @@ spec: optional: true - name: pod-templates optional: true + - name: certificates + optional: true