[K8s] Generate migration_services.yaml from ConfigMaps

.github/workflows/CI.yml

@@ -63,17 +63,18 @@ jobs:
        run: |
         flake8 $(git ls-files '*.py')
 
-  console-python-tests:
+  python-tests:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         py-project:
-          - lib/console_link
-          - cluster_tools
-          - console_api
+          - migrationConsole/lib/console_link
+          - migrationConsole/cluster_tools
+          - migrationConsole/console_api
+          - k8sConfigMapUtilScripts
     env:
-      WORKING_DIR: ./TrafficCapture/dockerSolution/src/main/docker/migrationConsole/${{ matrix.py-project }}
+      WORKING_DIR: ./TrafficCapture/dockerSolution/src/main/docker/${{ matrix.py-project }}
     defaults:
       run:
         working-directory: ${{ env.WORKING_DIR }}
@@ -90,7 +91,7 @@ jobs:
       - name: Get Sanitized Name
         env:
           PY_PROJECT: ${{ matrix.py-project }}
-        run: echo "SANITIZED_PY_PROJECT=${PY_PROJECT/\//-}" >> $GITHUB_ENV
+        run: echo "SANITIZED_PY_PROJECT=${PY_PROJECT//\//-}" >> $GITHUB_ENV
       - name: Upload Coverage Reports
         uses: actions/upload-artifact@v4
         with:
@@ -248,6 +249,7 @@ jobs:
             --exclude "https://localhost*"
             --exclude "http://capture-proxy*"
             --exclude "https://capture-proxy*"
+            --exclude-path "TrafficCapture/dockerSolution/src/main/docker/k8sConfigMapUtilScripts/tests/data"
             --exclude-mail
           fail: true
         env:
@@ -256,7 +258,7 @@ jobs:
 
   all-ci-checks-pass:
     needs:
-      - console-python-tests
+      - python-tests
       - gradle-tests
       - link-checker
       - node-tests

TrafficCapture/dockerSolution/src/main/docker/docker-compose.yml

@@ -97,7 +97,7 @@ services:
       - migrations
     volumes:
       - sharedLogsVolume:/shared-logs-output
-      - ./migrationConsole/lib/console_link/services.yaml:/etc/migration_services.yaml
+      - ./migrationConsole/lib/console_link/services.yaml:/config/migration_services.yaml
       # this is a convenience thing for testing -- it should be removed before this makes it to prod.
       # - ./migrationConsole/lib/console_link:/root/lib/console_link
       - ~/.aws:/root/.aws

TrafficCapture/dockerSolution/src/main/docker/k8sConfigMapUtilScripts/.coveragerc

@@ -0,0 +1,6 @@
+[run]
+source = configmap2yaml
+
+[report]
+omit =
+   */tests/*

TrafficCapture/dockerSolution/src/main/docker/k8sConfigMapUtilScripts/Dockerfile

@@ -33,7 +33,7 @@ COPY Pipfile .
 COPY Pipfile.lock .
 RUN pipenv install --deploy
 
-COPY configmap2yaml/* /root/
-RUN chmod ug+x /root/*.py
+COPY configmap2yaml /root/configmap2yaml/
+RUN chmod ug+x /root/configmap2yaml/*.py
 
 ENTRYPOINT ["tail", "-f", "/dev/null"]

TrafficCapture/dockerSolution/src/main/docker/k8sConfigMapUtilScripts/Pipfile

@@ -9,6 +9,15 @@ pyyaml = ">=6.0.2"
 Jinja2 = ">=3.1.4"
 
 [dev-packages]
+pytest = "*"
+pytest-mock = "*"
+coverage = "*"
 
 [requires]
 python_version = "3.11"
+
+[scripts]
+config_watcher = "python3 -m configmap2yaml.config_watcher"
+print_env_vars_as_exports = "python3 -m configmap2yaml.print_env_vars_as_exports"
+test = "coverage run -m pytest"
+coverage = "coverage"

TrafficCapture/dockerSolution/src/main/docker/k8sConfigMapUtilScripts/README.md

@@ -0,0 +1,33 @@
+# Kubernetes Config Map Utility Library
+
+This library provides utilities for working with Kubernetes ConfigMaps and specifically generating a `migration_services.yaml` usable by the Migration Console from ConfigMap values 
+
+## Development
+
+To install the library for development purposes, create a virtual env and install the library. It will automatically install its dependencies as well.
+
+```shell
+pipenv install
+```
+
+To run `config_watcher.py` locally, a command like below can be used:
+```shell
+pipenv run config_watcher --outfile ./test.yaml
+```
+
+### Unit Tests
+
+Unit tests can be run from this current `k8sConfigMapUtilScripts/` directory by first installing dependencies then running pytest:
+
+```shell
+pipenv install --dev
+pipenv run test
+```
+
+### Coverage
+
+_Code coverage_ metrics can be generated after a unit-test run. A report can either be printed on the command line:
+
+```shell
+pipenv run coverage report
+```

TrafficCapture/dockerSolution/src/main/docker/k8sConfigMapUtilScripts/configmap2yaml/config_watcher.py

@@ -1,17 +1,20 @@
 import argparse
-from format_services_yaml import YAMLTemplateConverter
-from io import StringIO
-from kubernetes import client, config, watch
 import logging
 import os
 import signal
 import tempfile
-from typing import Dict, Any
+import time
 import sys
 import yaml
 
+from io import StringIO
+from kubernetes import client, config, watch
+from typing import Dict, Any
+
+from configmap2yaml.format_services_yaml import YAMLTemplateConverter
+
 
-logging.basicConfig(level=logging.INFO)
+logging.basicConfig(format='%(asctime)s [%(levelname)s] %(message)s', level=logging.INFO)
 logger = logging.getLogger(__name__)
 
 
@@ -21,7 +24,7 @@ def __init__(self, label_selector: str, namespace: str, output_file: str):
         self.namespace = namespace
         self.output_file = output_file
         self.current_data: Dict[str, Any] = {}
-        self.formatter = YAMLTemplateConverter()
+        self.formatter = YAMLTemplateConverter(namespace=namespace)
 
         # Validate output file path
         output_dir = os.path.dirname(output_file)
@@ -36,15 +39,16 @@ def __init__(self, label_selector: str, namespace: str, output_file: str):
             logger.warning("Unable to load in-cluster config, falling back to local kubeconfig")
             config.load_kube_config()
 
-        self.v1 = client.CoreV1Api()
+        self.k8s_client = client.CoreV1Api()
 
     def update_yaml_file(self) -> None:
         """Update the output YAML file with new ConfigMap data"""
         try:
             # Create a temporary file in the same directory as the target file
             output_dir = os.path.dirname(self.output_file)
             with tempfile.NamedTemporaryFile(mode='w', dir=output_dir, delete=False) as temp_file:
-                YAMLTemplateConverter().convert(StringIO(yaml.safe_dump(self.current_data)), temp_file)
+                YAMLTemplateConverter(namespace=self.namespace).convert(StringIO(yaml.safe_dump(self.current_data)),
+                                                                        temp_file)
                 temp_file.flush()
                 os.fsync(temp_file.fileno())  # Ensure all data is written to disk
 
@@ -61,46 +65,57 @@ def update_yaml_file(self) -> None:
                     pass
             raise
 
-    def watch_configmaps(self) -> None:
-        """Watch ConfigMaps for changes and write the contents at startup and upon an configMap changes"""
-        w = watch.Watch()
-
-        # First, get existing ConfigMaps
-        logger.info(f"Loading existing ConfigMaps for {self.namespace} and {self.label_selector}")
-        existing_configmaps = self.v1.list_namespaced_config_map(
+    def init_config_map_data(self) -> None:
+        """Write initial contents of existing ConfigMaps"""
+        logger.info(f"Loading existing ConfigMaps for '{self.namespace}' namespace and "
+                    f"{self.label_selector if self.label_selector else 'no'} label selector")
+        existing_configmaps = self.k8s_client.list_namespaced_config_map(
             namespace=self.namespace,
             label_selector=self.label_selector
         )
-        logger.info(f"Got configmaps: {existing_configmaps}")
+        logger.debug(f"Got configmaps: {existing_configmaps}")
         for configmap in existing_configmaps.items:
-            logger.info(f"configmap={configmap}")
+            logger.debug(f"configmap={configmap}")
             self.current_data[configmap.metadata.name] = configmap.data if configmap.data else {}
 
         self.update_yaml_file()
 
-        # Then watch for changes
-        try:
-            for event in w.stream(
-                    self.v1.list_namespaced_config_map,
-                    namespace=self.namespace,
-                    label_selector=self.label_selector
-            ):
-                configmap = event['object']
-                event_type = event['type']
-
-                if event_type in ['ADDED', 'MODIFIED']:
-                    self.current_data[configmap.metadata.name] = configmap.data if configmap.data else {}
-                elif event_type == 'DELETED':
-                    name = configmap.metadata.name
-                    if name in self.current_data:
-                        logger.info(f"Removing ConfigMap: {name}")
-                        del self.current_data[name]
-
-                self.update_yaml_file()
+    def watch_configmaps(self) -> None:
+        """Watch ConfigMaps for changes and write the contents upon any configMap changes"""
+        w = watch.Watch()
 
-        except Exception as e:
-            logger.error(f"Error watching ConfigMaps: {e}")
-            raise
+        while True:
+            try:
+                self.process_events(k8s_watch=w)
+            except Exception as error:
+                if "Expired" in str(error):
+                    logger.info("Resource version expired, restarting watch...")
+                    time.sleep(5)
+                    continue
+                else:
+                    logger.error(f"Error watching ConfigMaps: {error}")
+                    raise
+
+    def process_events(self, k8s_watch):
+        for event in k8s_watch.stream(
+                self.k8s_client.list_namespaced_config_map,
+                namespace=self.namespace,
+                label_selector=self.label_selector,
+        ):
+            configmap = event['object']
+            event_type = event['type']
+            logger.info(f"Received {event_type} event for {configmap.metadata.name}")
+            logger.debug(f"Incoming event: {event}")
+
+            if event_type in ['ADDED', 'MODIFIED']:
+                self.current_data[configmap.metadata.name] = configmap.data if configmap.data else {}
+            elif event_type == 'DELETED':
+                name = configmap.metadata.name
+                if name in self.current_data:
+                    logger.info(f"Removing ConfigMap: {name}")
+                    del self.current_data[name]
+
+            self.update_yaml_file()
 
 
 def parse_args():
@@ -119,8 +134,8 @@ def parse_args():
     )
     parser.add_argument(
         '--namespace',
-        default=os.getenv('NAMESPACE', 'default'),
-        help='Kubernetes namespace (default: default)'
+        default=os.getenv('NAMESPACE', 'ma'),
+        help='Kubernetes namespace (default: ma)'
     )
     return parser.parse_args()
 
@@ -141,6 +156,7 @@ def sigterm_handler(signum, frame):
             namespace=args.namespace,
             output_file=args.outfile
         )
+        watcher.init_config_map_data()
         watcher.watch_configmaps()
     except KeyboardInterrupt:
         # Handle Ctrl+C cleanly too