Grizz 0118

.github/workflows/tests.yml

@@ -7,36 +7,35 @@ jobs:
   linting:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Install virtualenv from poetry
         uses: 20c/workflows/poetry@v1
       - name: Run linters
         run: |
-            poetry run pre-commit run --all-files
+          poetry run pre-commit run --all-files
 
   test:
     needs: linting
     strategy:
       fail-fast: false
       matrix:
         os: [ "ubuntu-latest", "macos-latest" ]
-        python-version: [ "3.7", "3.8", "3.9", "3.10" ]
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Check out repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Install virtualenv from poetry
         uses: 20c/workflows/poetry@v1
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Run tests
+      - name: Install dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install tox-gh-actions
-          poetry run tox -e py
-      # upload coverage stats
+          pip install tox tox-gh-actions
+      - name: Run tests
+        run: tox -e py
       - name: Upload coverage
         uses: codecov/codecov-action@v3
         with:
-#          file: ./coverage.xml
           fail_ci_if_error: true

.pre-commit-config.yaml

@@ -1,40 +1,28 @@
 fail_fast: false
 exclude: |
-    (?x)^(
-        tests/data/.*
-    )$
+  (?x)^(
+      tests/data/.*
+  )$
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.3.0
+    rev: v4.5.0
     hooks:
       - id: check-yaml
       - id: trailing-whitespace
-  - repo: local
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.1.14
     hooks:
-      - id: system
-        name: isort
-        entry: poetry run isort .
-        language: system
-        pass_filenames: false
+      # Run the linter.
+      - id: ruff
+        args: [--fix]
+      # Run the formatter.
+      - id: ruff-format
   - repo: local
     hooks:
-    -   id: pyupgrade
+      - id: pyupgrade
         name: pyupgrade
-        entry: poetry run pyupgrade --py37-plus
+        entry: poetry run pyupgrade --py39-plus
         language: python
         types: [python]
         pass_filenames: true
-  - repo: local
-    hooks:
-      - id: system
-        name: Black
-        entry: poetry run black .
-        language: system
-        pass_filenames: false
-  - repo: local
-    hooks:
-      - id: system
-        name: flake8
-        entry: poetry run flake8 .
-        language: system
-        pass_filenames: false

CHANGELOG.yaml

@@ -1,3 +1,16 @@
+Unreleased:
+  added:
+  - python to 3.9
+  - ansible Templar templating system
+  - loopcontrol
+  - allow adding filters
+  changed: []
+  deprecated: []
+  fixed: []
+  removed:
+  - python 3.7, 3.8 support
+  security: []
+
 0.2.0:
   added:
   - python to 3.9
@@ -7,10 +20,3 @@
   removed:
   - python 3.4 and 3.5 support
   - Pipfile
-Unreleased:
-  added: []
-  changed: []
-  deprecated: []
-  fixed: []
-  removed: []
-  security: []

pyproject.toml

@@ -12,10 +12,10 @@ classifiers = [
     "Intended Audience :: System Administrators",
     "Intended Audience :: Telecommunications Industry",
     "License :: OSI Approved :: Apache Software License",
-    "Programming Language :: Python :: 3.7",
-    "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Topic :: Internet",
 ]
@@ -30,11 +30,16 @@ netom = "netom.cli:main"
 
 
 [tool.poetry.dependencies]
-python = "^3.7"
-Jinja2 = "^3"
-tmpl = "^1"
-confu = "^1.7.1"
-munge = "^1.1.0"
+python = "^3.9"
+jinja2 = ">=3"
+confu = ">=1.7.1"
+munge = ">=1.1.0"
+ansible-core = "^2"
+tmpl = ">=1"
+
+# [tool.poetry.dependencies.tmpl]
+# path = "../tmpl"
+# develop = true
 
 [tool.poetry.dev-dependencies]
 # testing
@@ -47,24 +52,24 @@ tox = ">=3"
 
 # linting
 bandit = ">=1.6.2"
-black = ">=20"
-isort = ">=5.7"
-flake8 = ">=3.8"
 mypy = ">=0.950"
 pre-commit = ">=2.13"
 pyupgrade = ">=2.19"
+ruff = ">=0.1"
 
 # docs
 markdown = "*"
 markdown-include = ">=0.5,<1"
-mkdocs = "^1.2.3"
+mkdocs = ">=1.2.3"
 
 # ctl
 ctl = ">=1"
-jinja2 = ">=2"
-tmpl = ">=1"
 
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
+
+[tool.ruff.lint]
+# Enable the isort rules.
+extend-select = ["I"]

src/netom/__init__.py

@@ -1,12 +1,12 @@
 import io
-import ipaddress
 import os
 
-import confu
 import tmpl
 from confu import schema
 from pkg_resources import get_distribution
 
+import netom.filters
+
 from .exception import NetomValidationError
 
 # TODO move out of this namespace
@@ -15,36 +15,73 @@
 __version__ = get_distribution("netom").version
 
 
-def make_variable_name(value):
-    """
-    Makes passed value into a variable name.
-    """
-    value = str(value).translate(str.maketrans(" .:", "___"))
-    return value
+from ansible.parsing.dataloader import DataLoader
+from ansible.template import Templar
 
 
-def ip_version(value):
-    """
-    Returns version of passed IP address.
-    """
-    return ipaddress.ip_address(value).version
+class NetomTemplar(Templar):
+    """A custom Templar class which includes additional jinja2 filters."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        for name in netom.filters.__all__:
+            self.environment.filters[name] = getattr(netom.filters, name)
+
+
+class PathLoader(DataLoader):
+    def __init__(self, search_path):
+        self._search_path = search_path
+        super().__init__()
+
+    def path_dwim_relative(self, path):
+        raise Exception("die")
+        return super().path_dwim_relative(self._search_path, path)
+
+
+class TemplarEngine:
+    """Engine for Ansible's Templar system for lazy vars."""
+
+    def __init__(self, *args, search_path="", **kwargs):
+        print(f"INIT searchPath {search_path}")
+        self.dataloader = PathLoader(search_path)
+        print(f"searchPath2 {self.dataloader._search_path}")
+
+    def _render_str_to_str(self, instr, data):
+        # Instantiate Templar with the data loader and variable data
+        templar = NetomTemplar(loader=self.dataloader, variables=data)
+
+        print(f"searchPath {self.dataloader._search_path}")
+        print(instr)
+        # Use the templar object to render the template string
+        return templar.template(instr)
 
 
 class Render:
     """
     Renders data to defined type.
     """
 
-    def __init__(self, model_version, model_type, search_path=None):
+    def __init__(
+        self, model_version, model_type, search_path=None, engine="jinja2", filters=None
+    ):
         """
         Create a render object.
 
+        model_version is the data model version (currently at "netom0")
+
         subtype is typically vendor name
-        version is the model version to use
         """
 
         self.version = model_version
         self.type = model_type
+        self.filters = {}
+        self._engine_type = engine
+
+        for name in netom.filters.__all__:
+            self.filters[name] = getattr(netom.filters, name)
+
+        if filters:
+            self.filters.update(filters)
 
         if not search_path:
             # FIXME use pkg resources
@@ -54,10 +91,14 @@ def __init__(self, model_version, model_type, search_path=None):
         self.set_search_path(search_path)
 
     def set_search_path(self, search_path):
-        self.engine = tmpl.get_engine("jinja2")(search_path=search_path)
-        self.engine.engine.filters["make_variable_name"] = make_variable_name
-        self.engine.engine.filters["ip_version"] = ip_version
-        # self.engine.search_path = os.path.dirname(search_path)
+        self._search_path = search_path
+
+        if self._engine_type == "templar":
+            self.engine = TemplarEngine(search_path=search_path)
+        else:
+            self.engine = tmpl.get_engine(self._engine_type)(search_path=search_path)
+            self.engine.engine.filters.update(self.filters)
+            self.engine.engine.add_extension("jinja2.ext.loopcontrols")
 
     def _render(self, filename, data, fobj):
         # engine.engine.undefined = IgnoreUndefined
@@ -70,6 +111,9 @@ def render_string(self, filename, data):
             self._render(filename, data, fobj)
             return fobj.getvalue()
 
+    def render_from_string(self, instr: str, data: dict):
+        return self.engine._render_str_to_str(instr, data)
+
     def bgp_neighbors(self, data, fobj, validate=True):
         """
         Renders BGP neighbors using `template/{{ typ }}/bgp/neighbors`.
@@ -88,6 +132,20 @@ def bgp_neighbors(self, data, fobj, validate=True):
         return self._render(filename, groups, fobj)
 
 
+class TemplarRender(Render):
+    def __init__(self, *args, **kwargs):
+        kwargs["engine"] = "templar"
+        super().__init__(*args, **kwargs)
+
+    def render_string(self, filename, data):
+        with open(os.path.join(self._search_path, filename)) as file:
+            template_string = file.read()
+            return self.render_from_string(template_string, data)
+
+    def render_from_string(self, instr, data):
+        return self.engine._render_str_to_str(instr, data)
+
+
 def validate(model, data, strict=True):
     schema.apply_defaults(model, data)
     success, errors, warnings = schema.validate(model, data, log=print)