From 0252bfb477b3157c3aeaf534357b6906672f829d Mon Sep 17 00:00:00 2001 From: Kat Morgan Date: Sat, 21 Sep 2024 12:04:04 -0700 Subject: [PATCH] refine core and module codebase --- pulumi/__main__.py | 3 - pulumi/core/config.py | 14 +- pulumi/core/deploy_module.py | 14 +- pulumi/core/helm_chart_versions.py | 20 --- pulumi/core/init.py | 100 +++++++------- pulumi/core/introspection.py | 2 +- pulumi/core/metadata.py | 187 ++++----------------------- pulumi/core/versions.py | 15 ++- pulumi/modules/cert_manager/types.py | 18 +-- pulumi/modules/kubevirt/types.py | 9 -- 10 files changed, 108 insertions(+), 274 deletions(-) diff --git a/pulumi/__main__.py b/pulumi/__main__.py index d749681..235a764 100644 --- a/pulumi/__main__.py +++ b/pulumi/__main__.py @@ -10,7 +10,6 @@ from core.config import export_results from core.deploy_module import deploy_module - def main(): try: init = initialize_pulumi() @@ -33,7 +32,6 @@ def main(): pulumi.log.error(f"Deployment failed: {str(e)}") raise - def deploy_modules( modules: List[str], config: pulumi.Config, @@ -56,6 +54,5 @@ def deploy_modules( configurations=configurations, ) - if __name__ == "__main__": main() diff --git a/pulumi/core/config.py b/pulumi/core/config.py index aeb9692..681c9c2 100644 --- a/pulumi/core/config.py +++ b/pulumi/core/config.py @@ -13,10 +13,10 @@ import pulumi def get_module_config( - module_name: str, - config: pulumi.Config, - default_versions: Dict[str, Any], -) -> Tuple[Dict[str, Any], bool]: + module_name: str, + config: pulumi.Config, + default_versions: Dict[str, Any], + ) -> Tuple[Dict[str, Any], bool]: """ Retrieves and prepares the configuration for a module. @@ -33,7 +33,11 @@ def get_module_config( module_config['version'] = module_config.get('version', default_versions.get(module_name)) return module_config, module_enabled -def export_results(versions: Dict[str, str], configurations: Dict[str, Dict[str, Any]], compliance: Dict[str, Any]): +def export_results( + versions: Dict[str, str], + configurations: Dict[str, Dict[str, Any]], + compliance: Dict[str, Any] + ): """ Exports the results of the deployment processes including versions, configurations, and compliance information. diff --git a/pulumi/core/deploy_module.py b/pulumi/core/deploy_module.py index b220c92..f95a556 100644 --- a/pulumi/core/deploy_module.py +++ b/pulumi/core/deploy_module.py @@ -4,7 +4,7 @@ import inspect import pulumi import pulumi_kubernetes as k8s -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Type from .introspection import discover_config_class, discover_deploy_function from .config import get_module_config @@ -33,7 +33,6 @@ def deploy_module( TypeError: If any arguments have incorrect types. ValueError: If any module-specific errors occur. """ - # Validate parameter types if not isinstance(module_name, str): raise TypeError("module_name must be a string") if not isinstance(config, pulumi.Config): @@ -49,21 +48,17 @@ def deploy_module( if not isinstance(configurations, dict): raise TypeError("configurations must be a dictionary") - # Get the module's configuration from Pulumi config module_config_dict, module_enabled = get_module_config(module_name, config, default_versions) if module_enabled: - # Discover the configuration class and deploy function dynamically ModuleConfigClass = discover_config_class(module_name) deploy_func = discover_deploy_function(module_name) config_obj = ModuleConfigClass.merge(module_config_dict) - # Infer the required argument name for the deploy function deploy_func_args = inspect.signature(deploy_func).parameters.keys() - config_arg_name = list(deploy_func_args)[0] # Assuming the first argument is the config object + config_arg_name = list(deploy_func_args)[0] - # Deploy the module using its deploy function with the correct arguments try: result = deploy_func( **{config_arg_name: config_obj}, @@ -71,7 +66,6 @@ def deploy_module( k8s_provider=k8s_provider, ) - # Handle the result based on its structure if isinstance(result, tuple) and len(result) == 3: version, release, exported_value = result elif isinstance(result, tuple) and len(result) == 2: @@ -80,20 +74,16 @@ def deploy_module( else: raise ValueError(f"Unexpected return value structure from {module_name} deploy function") - # Record the deployed version and configuration versions[module_name] = version configurations[module_name] = {"enabled": module_enabled} - # Export any additional values if needed if exported_value: pulumi.export(f"{module_name}_exported_value", exported_value) - # Add the release to global dependencies to maintain resource ordering global_depends_on.append(release) except Exception as e: pulumi.log.error(f"Deployment failed for module {module_name}: {str(e)}") raise - else: pulumi.log.info(f"Module {module_name} is not enabled.") diff --git a/pulumi/core/helm_chart_versions.py b/pulumi/core/helm_chart_versions.py index 8d052df..019c342 100644 --- a/pulumi/core/helm_chart_versions.py +++ b/pulumi/core/helm_chart_versions.py @@ -14,15 +14,6 @@ CHART_NOT_FOUND = "Chart not found" def is_stable_version(version_str: str) -> bool: - """ - Check if the version string is a valid and stable semantic version. - - Args: - version_str (str): The version string to check. - - Returns: - bool: True if the version is stable, False otherwise. - """ try: parsed_version = parse_version(version_str) return isinstance(parsed_version, Version) and not parsed_version.is_prerelease and not parsed_version.is_devrelease @@ -30,22 +21,11 @@ def is_stable_version(version_str: str) -> bool: return False def get_latest_helm_chart_version(url: str, chart_name: str) -> str: - """ - Fetches the latest stable version of a Helm chart from a given URL. - - Args: - url (str): The URL of the Helm chart repository. - chart_name (str): The name of the Helm chart. - - Returns: - str: The latest stable version of the Helm chart, or an error message if the chart is not found or an error occurs during fetching. - """ try: logging.info(f"Fetching URL: {url}") response = requests.get(url) response.raise_for_status() - # Parse the YAML content index = yaml.safe_load(response.content) if chart_name in index['entries']: chart_versions = index['entries'][chart_name] diff --git a/pulumi/core/init.py b/pulumi/core/init.py index f7f0059..404d180 100644 --- a/pulumi/core/init.py +++ b/pulumi/core/init.py @@ -20,66 +20,64 @@ from .utils import generate_global_transformations def initialize_pulumi() -> Dict[str, Any]: - """ - Initializes the Pulumi configuration, Kubernetes provider, and global resources. - - Returns: - Dict[str, Any]: A dictionary containing initialized resources. - """ config = pulumi.Config() stack_name = pulumi.get_stack() project_name = pulumi.get_project() - default_versions = load_default_versions(config) - versions: Dict[str, str] = {} - configurations: Dict[str, Dict[str, Any]] = {} - global_depends_on: List[pulumi.Resource] = [] + try: + default_versions = load_default_versions(config) + versions: Dict[str, str] = {} + configurations: Dict[str, Dict[str, Any]] = {} + global_depends_on: List[pulumi.Resource] = [] - kubernetes_config = config.get_object("kubernetes") or {} - kubeconfig = kubernetes_config.get("kubeconfig") or os.getenv('KUBECONFIG') - kubernetes_context = kubernetes_config.get("context") + kubernetes_config = config.get_object("kubernetes") or {} + kubeconfig = kubernetes_config.get("kubeconfig") or os.getenv('KUBECONFIG') + kubernetes_context = kubernetes_config.get("context") - pulumi.log.info(f"Kubeconfig: {kubeconfig}") - pulumi.log.info(f"Kubernetes context: {kubernetes_context}") + pulumi.log.info(f"Kubeconfig: {kubeconfig}") + pulumi.log.info(f"Kubernetes context: {kubernetes_context}") - k8s_provider = Provider( - "k8sProvider", - kubeconfig=kubeconfig, - context=kubernetes_context, - ) + k8s_provider = Provider( + "k8sProvider", + kubeconfig=kubeconfig, + context=kubernetes_context, + ) - git_info = collect_git_info() - configurations["source_repository"] = { - "remote": git_info["remote"], - "branch": git_info["branch"], - "commit": git_info["commit"] - } + git_info = collect_git_info() + configurations["source_repository"] = { + "remote": git_info["remote"], + "branch": git_info["branch"], + "commit": git_info["commit"] + } - compliance_config_dict = config.get_object('compliance') or {} - compliance_config = ComplianceConfig.merge(compliance_config_dict) - compliance_labels = generate_compliance_labels(compliance_config) - compliance_annotations = generate_compliance_annotations(compliance_config) + compliance_config_dict = config.get_object('compliance') or {} + compliance_config = ComplianceConfig.merge(compliance_config_dict) + compliance_labels = generate_compliance_labels(compliance_config) + compliance_annotations = generate_compliance_annotations(compliance_config) - git_labels = generate_git_labels(git_info) - git_annotations = generate_git_annotations(git_info) - global_labels = {**compliance_labels, **git_labels} - global_annotations = {**compliance_annotations, **git_annotations} + git_labels = generate_git_labels(git_info) + git_annotations = generate_git_annotations(git_info) + global_labels = {**compliance_labels, **git_labels} + global_annotations = {**compliance_annotations, **git_annotations} - set_global_labels(global_labels) - set_global_annotations(global_annotations) - generate_global_transformations(global_labels, global_annotations) + set_global_labels(global_labels) + set_global_annotations(global_annotations) + generate_global_transformations(global_labels, global_annotations) - return { - "config": config, - "stack_name": stack_name, - "project_name": project_name, - "default_versions": default_versions, - "versions": versions, - "configurations": configurations, - "global_depends_on": global_depends_on, - "k8s_provider": k8s_provider, - "git_info": git_info, - "compliance_config": compliance_config, - "global_labels": global_labels, - "global_annotations": global_annotations, - } + return { + "config": config, + "stack_name": stack_name, + "project_name": project_name, + "default_versions": default_versions, + "versions": versions, + "configurations": configurations, + "global_depends_on": global_depends_on, + "k8s_provider": k8s_provider, + "git_info": git_info, + "compliance_config": compliance_config, + "global_labels": global_labels, + "global_annotations": global_annotations, + } + except Exception as e: + pulumi.log.error(f"Initialization error: {str(e)}") + raise diff --git a/pulumi/core/introspection.py b/pulumi/core/introspection.py index 670eab0..14a7dc3 100644 --- a/pulumi/core/introspection.py +++ b/pulumi/core/introspection.py @@ -3,7 +3,7 @@ import inspect import importlib -from typing import Type, Callable, Optional +from typing import Type, Callable def discover_config_class(module_name: str) -> Type: """ diff --git a/pulumi/core/metadata.py b/pulumi/core/metadata.py index fd21729..3f2e288 100644 --- a/pulumi/core/metadata.py +++ b/pulumi/core/metadata.py @@ -5,10 +5,28 @@ # - git release tag import subprocess -import threading import pulumi from typing import Dict -from .utils import sanitize_label_value, extract_repo_name + +class MetadataSingleton: + _instance: Dict[str, Dict[str, str]] = {} + + def __new__(cls, *args, **kwargs): + if not cls._instance: + cls._instance = {"_global_labels": {}, "_global_annotations": {}} + return cls._instance + +def set_global_labels(labels: Dict[str, str]): + MetadataSingleton()["_global_labels"] = labels + +def set_global_annotations(annotations: Dict[str, str]): + MetadataSingleton()["_global_annotations"] = annotations + +def get_global_labels() -> Dict[str, str]: + return MetadataSingleton()["_global_labels"] + +def get_global_annotations() -> Dict[str, str]: + return MetadataSingleton()["_global_annotations"] def collect_git_info() -> Dict[str, str]: """ @@ -22,176 +40,23 @@ def collect_git_info() -> Dict[str, str]: Dict[str, str]: A dictionary containing the remote URL, branch name, and commit hash. """ try: - # Fetch the remote URL from git configuration - remote = subprocess.check_output( - ['git', 'config', '--get', 'remote.origin.url'], - stderr=subprocess.STDOUT - ).strip().decode('utf-8') - - # Fetch the current branch name - branch = subprocess.check_output( - ['git', 'rev-parse', '--abbrev-ref', 'HEAD'], - stderr=subprocess.STDOUT - ).strip().decode('utf-8') - - # Fetch the latest commit hash - commit = subprocess.check_output( - ['git', 'rev-parse', 'HEAD'], - stderr=subprocess.STDOUT - ).strip().decode('utf-8') - + remote = subprocess.check_output(['git', 'config', '--get', 'remote.origin.url'], stderr=subprocess.STDOUT).strip().decode('utf-8') + branch = subprocess.check_output(['git', 'rev-parse', '--abbrev-ref', 'HEAD'], stderr=subprocess.STDOUT).strip().decode('utf-8') + commit = subprocess.check_output(['git', 'rev-parse', 'HEAD'], stderr=subprocess.STDOUT).strip().decode('utf-8') return {'remote': remote, 'branch': branch, 'commit': commit} - except subprocess.CalledProcessError as e: - # In case of an error, log it and return 'N/A' for all values - pulumi.log.error(f"Error fetching git information: {e.output.decode('utf-8')}") + pulumi.log.error(f"Error fetching git information: {e}") return {'remote': 'N/A', 'branch': 'N/A', 'commit': 'N/A'} - def generate_git_labels(git_info: Dict[str, str]) -> Dict[str, str]: - """ - Generates labels for Kubernetes resources from git information. - - Labels are key/value pairs that are attached to objects, such as pods. Labels are intended to be - used to specify identifying attributes of objects that are meaningful and relevant to users. - - Args: - git_info (Dict[str, str]): A dictionary containing git information. - - Returns: - Dict[str, str]: A dictionary containing sanitized git branch name and shortened commit hash. - """ return { - "git.branch": sanitize_label_value(git_info.get("branch", "")), - "git.commit": git_info.get("commit", "")[:7], # Shorten commit hash + "git.branch": git_info.get("branch", ""), + "git.commit": git_info.get("commit", "")[:7], } - def generate_git_annotations(git_info: Dict[str, str]) -> Dict[str, str]: - """ - Generates annotations for Kubernetes resources from git information. - - Annotations are key/value pairs that can hold arbitrary non-identifying metadata. They are - intended to store information that can be useful for debugging, auditing, or other purposes. - - Args: - git_info (Dict[str, str]): A dictionary containing git information. - - Returns: - Dict[str, str]: A dictionary containing git remote URL, full commit hash, and branch name. - """ return { "git.remote": git_info.get("remote", ""), "git.commit.full": git_info.get("commit", ""), "git.branch": git_info.get("branch", "") } - -########################################## -# Global labels and annotations -########################################## - -class MetadataSingleton: - """ - Singleton class to manage global labels and annotations for Kubernetes resources. - - This class uses the singleton pattern to ensure only one instance is created. - This is important for global state management. - """ - - __lock = threading.Lock() - __instance = None - - def __new__(cls): - """ - Ensure only one instance of this class exists (Singleton pattern). - - This method checks if an instance already exists. If not, it creates one. - """ - with cls.__lock: - if cls.__instance is None: - cls.__instance = super(MetadataSingleton, cls).__new__(cls) - cls.__instance._global_labels = {} - cls.__instance._global_annotations = {} - return cls.__instance - - @property - def global_labels(self) -> Dict[str, str]: - """ - Property method to get global labels. - - Labels are useful for identifying and grouping Kubernetes resources. - """ - return self._global_labels - - @global_labels.setter - def global_labels(self, labels: Dict[str, str]): - """ - Property method to set global labels. - - Args: - labels (Dict[str, str]): A dictionary of labels to set globally. - """ - self._global_labels = labels - - @property - def global_annotations(self) -> Dict[str, str]: - """ - Property method to get global annotations. - - Annotations provide additional context and useful metadata for Kubernetes resources. - """ - return self._global_annotations - - @global_annotations.setter - def global_annotations(self, annotations: Dict[str, str]): - """ - Property method to set global annotations. - - Args: - annotations (Dict[str, str]): A dictionary of annotations to set globally. - """ - self._global_annotations = annotations - - -# Singleton instance for managing global labels and annotations -_metadata_singleton = MetadataSingleton() - - -def set_global_labels(labels: Dict[str, str]): - """ - Function to set global labels using the singleton instance. - - Args: - labels (Dict[str, str]): A dictionary of labels to set globally. - """ - _metadata_singleton.global_labels = labels - - -def set_global_annotations(annotations: Dict[str, str]): - """ - Function to set global annotations using the singleton instance. - - Args: - annotations (Dict[str, str]): A dictionary of annotations to set globally. - """ - _metadata_singleton.global_annotations = annotations - - -def get_global_labels() -> Dict[str, str]: - """ - Function to get global labels from the singleton instance. - - Returns: - Dict[str, str]: A dictionary of global labels. - """ - return _metadata_singleton.global_labels - - -def get_global_annotations() -> Dict[str, str]: - """ - Function to get global annotations from the singleton instance. - - Returns: - Dict[str, str]: A dictionary of global annotations. - """ - return _metadata_singleton.global_annotations diff --git a/pulumi/core/versions.py b/pulumi/core/versions.py index e02c055..6c25b57 100644 --- a/pulumi/core/versions.py +++ b/pulumi/core/versions.py @@ -8,7 +8,7 @@ # TODO: replace with official github releases artifact URLs when released DEFAULT_VERSIONS_URL_TEMPLATE = 'https://raw.githubusercontent.com/ContainerCraft/Kargo/rerefactor/pulumi/' -def load_default_versions(config: pulumi.Config) -> dict: +def load_default_versions(config: pulumi.Config, force_refresh=False) -> dict: """ Loads the default versions for modules based on the specified configuration settings. @@ -27,6 +27,14 @@ def load_default_versions(config: pulumi.Config) -> dict: Raises: Exception: If default versions cannot be loaded from any source. """ + cache_file = '/tmp/default_versions.json' + if not force_refresh and os.path.exists(cache_file): + try: + with open(cache_file) as f: + return json.load(f) + except Exception as e: + pulumi.log.warn(f"Error reading cache file: {e}") + stack_name = pulumi.get_stack() default_versions_source = config.get('default_versions.source') versions_channel = config.get('versions.channel') or 'stable' @@ -34,7 +42,6 @@ def load_default_versions(config: pulumi.Config) -> dict: default_versions = {} def load_versions_from_file(file_path: str) -> dict: - """Loads versions from a local JSON file.""" try: with open(file_path, 'r') as f: versions = json.load(f) @@ -45,7 +52,6 @@ def load_versions_from_file(file_path: str) -> dict: return {} def load_versions_from_url(url: str) -> dict: - """Loads versions from a remote URL.""" try: response = requests.get(url) response.raise_for_status() @@ -83,4 +89,7 @@ def load_versions_from_url(url: str) -> dict: if not default_versions: raise Exception("Cannot proceed without default versions.") + with open(cache_file, 'w') as f: + json.dump(default_versions, f) + return default_versions diff --git a/pulumi/modules/cert_manager/types.py b/pulumi/modules/cert_manager/types.py index a621295..7928626 100644 --- a/pulumi/modules/cert_manager/types.py +++ b/pulumi/modules/cert_manager/types.py @@ -1,4 +1,13 @@ # ./pulumi/modules/cert_manager/types.py +""" +Merges user-provided configuration with default configuration. + +Args: + user_config (Dict[str, Any]): The user-provided configuration. + +Returns: + CertManagerConfig: The merged configuration object. +""" from dataclasses import dataclass from typing import Optional, Dict, Any @@ -13,15 +22,6 @@ class CertManagerConfig: @staticmethod def merge(user_config: Dict[str, Any]) -> 'CertManagerConfig': - """ - Merges user-provided configuration with default configuration. - - Args: - user_config (Dict[str, Any]): The user-provided configuration. - - Returns: - CertManagerConfig: The merged configuration object. - """ default_config = CertManagerConfig() for key, value in user_config.items(): if hasattr(default_config, key): diff --git a/pulumi/modules/kubevirt/types.py b/pulumi/modules/kubevirt/types.py index 3054004..872c9ee 100644 --- a/pulumi/modules/kubevirt/types.py +++ b/pulumi/modules/kubevirt/types.py @@ -17,15 +17,6 @@ class KubeVirtConfig: @classmethod def merge(cls, user_config: Dict[str, Any]) -> 'KubeVirtConfig': - """ - Merge user-provided configuration with default configuration. - - Args: - user_config (Dict[str, Any]): The user-provided configuration. - - Returns: - KubeVirtConfig: The merged configuration object. - """ default_config = cls() merged_config = default_config.__dict__.copy()