Update example05

.gitignore

@@ -289,3 +289,6 @@ fabric.properties
 
 ## Sphinx Documentation ##
 docs/build
+
+## Secrets
+creds.env

docs/source/examples/index.rst

@@ -8,3 +8,4 @@ For each example, the complete source code is `available in Github <https://gith
 .. mdinclude:: ../../../examples/02-callback-function/README.md
 .. mdinclude:: ../../../examples/03-remote-system/README.md
 .. mdinclude:: ../../../examples/04-get-update-instantiate/README.md
+.. mdinclude:: ../../../examples/05-nautobot-peeringdb/README.md

examples/05-nautobot-peeringdb/README.md

@@ -4,63 +4,72 @@
 
 The goal of this example is to synchronize some data from [PeeringDB](https://www.peeringdb.com/), that as the name suggests is a DB where peering entities define their facilities and presence to facilitate peering, towards [Nautobot Demo](https://demo.nautobot.com/) that is a always on demo service for [Nautobot](https://nautobot.readthedocs.io/), an open source Source of Truth.
 
-In Peering DB there is a model that defines a `Facility` and you can get information about the actual data center and the city where it is placed. In Nautobot, this information could be mapped to the `Region` and `Site` models, where `Region` can define hierarchy. For instance, Barcelona is in Spain and Spain is in Europe, and all of them are `Regions`. And, finally, the actual datacenter will refer to the `Region` where it is placed.
+In Peering DB there is a model that defines a `Facility` and you can get information about the actual data center and the city where it is placed. In Nautobot, this information could be mapped to the `Region` and `Site` models, where `Region` can depend from other `Region` and also contain `Site` as children. For instance, Barcelona is in Spain and Spain is in Europe, and all of them are `Regions`. And, finally, the actual datacenter will refer to the `Region` where it is placed.
 
-Because of the nature of the demo, we will focus on syncing from PeeringDB to Nautobot (we can assume that PeeringDB is the authoritative System of Record) and we will skip the `delete` part of the `diffsync` library.
+Because of the nature of the demo, we will focus on syncing from PeeringDB to Nautobot (we assume that PeeringDB is the authoritative System of Record) and we will skip the `delete` part of the `diffsync` library, using diffsync flags.
 
 We have 3 files:
 
 - `models.py`: defines the reference models that we will use: `RegionMode` and `SiteModel`
 - `adapter_peeringdb.py`: defines the PeeringDB adapter to translate via `load()` the data from PeeringDB into the reference models commented above. Notice that we don't define CRUD methods because we will sync from it (no to it)
-- `adapter_nautobot.py`: deifnes the Nautobot adapter with the `load()` and the CRUD methods
+- `adapter_nautobot.py`: defines the Nautobot adapter with the `load()` and the CRUD methods
 
 > The source code for this example is in Github in the [examples/05-nautobot-peeringdb/](https://github.com/networktocode/diffsync/tree/main/examples/05-nautobot-peeringdb) directory.
 
-## Install dependencies
+## Get PeeringDB API Key (optional)
+
+To ensure a good performance from PeeringDB API, you should provide an API Key: https://docs.peeringdb.com/howto/api_keys/
+
+Then, copy the example `creds.example.env` into `creds.env`, and place your new API Key.
 
 ```bash
-python3 -m venv .venv
-source .venv/bin/activate
-pip3 install -r requirements.txt
+$ cp examples/05-nautobot-peeringdb/creds.example.env examples/05-nautobot-peeringdb/creds.env
+
 ```
 
-## Run it interactively
+> Without API Key it could also work, but it could fail due API rate limiting.
 
-```python
-from IPython import embed
-embed(colors="neutral")
+## Set up local docker environment
 
-# Import Adapters
-from diffsync.enum import DiffSyncFlags
+```bash
+$ git clone https://github.com/networktocode/diffsync.git
 
+$ docker-compose -f examples/05-nautobot-peeringdb/docker-compose.yml up -d --build
+
+$ docker exec -it 05-nautobot-peeringdb_example_1 bash
+```
+
+## Interactive execution
+
+```python
 from adapter_nautobot import NautobotRemote
 from adapter_peeringdb import PeeringDB
+from diffsync.enum import DiffSyncFlags
+from diffsync.store.redis import RedisStore
 
-# Initialize PeeringDB adapter, using CATNIX id for demonstration
-peeringdb = PeeringDB(ix_id=62)
+store_one = RedisStore(host="redis")
+store_two = RedisStore(host="redis")
+
+peeringdb = PeeringDB(
+    ix_id=62,
+    internal_storage_engine=store_one
+)
 
-# Initialize Nautobot adapter, pointing to the demo instance (it's also the default settings)
 nautobot = NautobotRemote(
     url="https://demo.nautobot.com",
-    token="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+    token="a" * 40,
+    internal_storage_engine=store_two
 )
 
-# Load PeeringDB info into the adapter
 peeringdb.load()
 
-# We can check the data that has been imported, some as `site` and some as `region` (with the parent relationships)
 peeringdb.dict()
 
-# Load Nautobot info into the adapter
 nautobot.load()
 
-# Let's diffsync do it's magic
-diff = nautobot.diff_from(peeringdb)
+diff = nautobot.diff_from(peeringdb, flags=DiffSyncFlags.SKIP_UNMATCHED_DST)
 
-# Quick summary of the expected changes (remember that delete ones are dry-run)
 diff.summary()
 
-# Execute the synchronization
 nautobot.sync_from(peeringdb, flags=DiffSyncFlags.SKIP_UNMATCHED_DST)
-
 ```

examples/05-nautobot-peeringdb/adapter_nautobot.py

@@ -1,13 +1,13 @@
 """Diffsync adapter class for Nautobot."""
 # pylint: disable=import-error,no-name-in-module
 import os
-import requests
+import pynautobot
 from models import RegionModel, SiteModel
 from diffsync import DiffSync
 
 
 NAUTOBOT_URL = os.getenv("NAUTOBOT_URL", "https://demo.nautobot.com")
-NAUTOBOT_TOKEN = os.getenv("NAUTOBOT_TOKEN", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+NAUTOBOT_TOKEN = os.getenv("NAUTOBOT_TOKEN", 40 * "a")
 
 
 class RegionNautobotModel(RegionModel):
@@ -30,7 +30,9 @@ def create(cls, diffsync, ids, attrs):
             data["description"] = attrs["description"]
         if attrs["parent_name"]:
             data["parent"] = str(diffsync.get(diffsync.region, attrs["parent_name"]).pk)
-        diffsync.post("/api/dcim/regions/", data)
+
+        diffsync.nautobot_api.dcim.regions.create(**data)
+
         return super().create(diffsync, ids=ids, attrs=attrs)
 
     def update(self, attrs):
@@ -39,22 +41,25 @@ def update(self, attrs):
         Args:
             attrs (dict): Updated values for this record's _attributes
         """
+        region = self.diffsync.nautobot_api.dcim.regions.get(name=self.name)
         data = {}
         if "slug" in attrs:
             data["slug"] = attrs["slug"]
         if "description" in attrs:
             data["description"] = attrs["description"]
         if "parent_name" in attrs:
             if attrs["parent_name"]:
-                data["parent"] = str(self.diffsync.get(self.diffsync.region, attrs["parent_name"]).pk)
+                data["parent"] = str(self.diffsync.get(self.diffsync.region, attrs["parent_name"]).name)
             else:
                 data["parent"] = None
-        self.diffsync.patch(f"/api/dcim/regions/{self.pk}/", data)
+
+        region.update(data=data)
+
         return super().update(attrs)
 
     def delete(self):  # pylint: disable= useless-super-delegation
         """Delete an existing Region record from remote Nautobot."""
-        # self.diffsync.delete(f"/api/dcim/regions/{self.pk}/")
+        # Not implemented
         return super().delete()
 
 
@@ -70,17 +75,14 @@ def create(cls, diffsync, ids, attrs):
             ids (dict): Initial values for this model's _identifiers
             attrs (dict): Initial values for this model's _attributes
         """
-        diffsync.post(
-            "/api/dcim/sites/",
-            {
-                "name": ids["name"],
-                "slug": attrs["slug"],
-                "description": attrs["description"],
-                "status": attrs["status_slug"],
-                "region": {"name": attrs["region_name"]} if attrs["region_name"] else None,
-                "latitude": attrs["latitude"],
-                "longitude": attrs["longitude"],
-            },
+        diffsync.nautobot_api.dcim.sites.create(
+            name=ids["name"],
+            slug=attrs["slug"],
+            description=attrs["description"],
+            status=attrs["status_slug"],
+            region={"name": attrs["region_name"]} if attrs["region_name"] else None,
+            latitude=attrs["latitude"],
+            longitude=attrs["longitude"],
         )
         return super().create(diffsync, ids=ids, attrs=attrs)
 
@@ -90,6 +92,8 @@ def update(self, attrs):
         Args:
             attrs (dict): Updated values for this record's _attributes
         """
+        site = self.diffsync.nautobot_api.dcim.sites.get(name=self.name)
+
         data = {}
         if "slug" in attrs:
             data["slug"] = attrs["slug"]
@@ -106,12 +110,14 @@ def update(self, attrs):
             data["latitude"] = attrs["latitude"]
         if "longitude" in attrs:
             data["longitude"] = attrs["longitude"]
-        self.diffsync.patch(f"/api/dcim/sites/{self.pk}/", data)
+
+        site.update(data=data)
+
         return super().update(attrs)
 
     def delete(self):  # pylint: disable= useless-super-delegation
         """Delete an existing Site record from remote Nautobot."""
-        # self.diffsync.delete(f"/api/dcim/sites/{self.pk}/")
+        # Not implemented
         return super().delete()
 
 
@@ -123,7 +129,7 @@ class NautobotRemote(DiffSync):
     site = SiteNautobotModel
 
     # Top-level class labels, i.e. those classes that are handled directly rather than as children of other models
-    top_level = ("region", "site")
+    top_level = ["region"]
 
     def __init__(self, *args, url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN, **kwargs):
         """Instantiate this class, but do not load data immediately from the remote system.
@@ -136,21 +142,11 @@ def __init__(self, *args, url=NAUTOBOT_URL, token=NAUTOBOT_TOKEN, **kwargs):
         super().__init__(*args, **kwargs)
         if not url or not token:
             raise ValueError("Both url and token must be specified!")
-        self.url = url
-        self.token = token
-        self.headers = {
-            "Accept": "application/json",
-            "Authorization": f"Token {self.token}",
-        }
+        self.nautobot_api = pynautobot.api(url=url, token=token)
 
     def load(self):
         """Load Region and Site data from the remote Nautobot instance."""
-        region_data = requests.get(f"{self.url}/api/dcim/regions/", headers=self.headers, params={"limit": 0}).json()
-        regions = region_data["results"]
-        while region_data["next"]:
-            region_data = requests.get(region_data["next"], headers=self.headers, params={"limit": 0}).json()
-            regions.extend(region_data["results"])
-
+        regions = self.nautobot_api.dcim.regions.all()
         for region_entry in regions:
             region = self.region(
                 name=region_entry["name"],
@@ -161,12 +157,7 @@ def load(self):
             )
             self.add(region)
 
-        site_data = requests.get(f"{self.url}/api/dcim/sites/", headers=self.headers, params={"limit": 0}).json()
-        sites = site_data["results"]
-        while site_data["next"]:
-            site_data = requests.get(site_data["next"], headers=self.headers, params={"limit": 0}).json()
-            sites.extend(site_data["results"])
-
+        sites = self.nautobot_api.dcim.sites.all()
         for site_entry in sites:
             site = self.site(
                 name=site_entry["name"],
@@ -179,21 +170,7 @@ def load(self):
                 pk=site_entry["id"],
             )
             self.add(site)
-
-    def post(self, path, data):
-        """Send an appropriately constructed HTTP POST request."""
-        response = requests.post(f"{self.url}{path}", headers=self.headers, json=data)
-        response.raise_for_status()
-        return response
-
-    def patch(self, path, data):
-        """Send an appropriately constructed HTTP PATCH request."""
-        response = requests.patch(f"{self.url}{path}", headers=self.headers, json=data)
-        response.raise_for_status()
-        return response
-
-    def delete(self, path):
-        """Send an appropriately constructed HTTP DELETE request."""
-        response = requests.delete(f"{self.url}{path}", headers=self.headers)
-        response.raise_for_status()
-        return response
+            if site_entry["region"]:
+                region = self.get(self.region, site_entry["region"]["name"])
+                region.add_child(site)
+                self.update(region)  # pylint: disable=no-member

examples/05-nautobot-peeringdb/adapter_peeringdb.py

@@ -1,5 +1,6 @@
 """Diffsync adapter class for PeeringDB."""
 # pylint: disable=import-error,no-name-in-module
+import os
 import requests
 from slugify import slugify
 import pycountry
@@ -9,6 +10,7 @@
 
 
 PEERINGDB_URL = "https://peeringdb.com/"
+peeringdb_api_key = os.environ.get("PEERINGDB_API_KEY", "").strip()
 
 
 class PeeringDB(DiffSync):
@@ -19,7 +21,7 @@ class PeeringDB(DiffSync):
     site = SiteModel
 
     # Top-level class labels, i.e. those classes that are handled directly rather than as children of other models
-    top_level = ("region", "site")
+    top_level = ["region"]
 
     def __init__(self, *args, ix_id, **kwargs):
         """Initialize the PeeringDB adapter."""
@@ -28,12 +30,16 @@ def __init__(self, *args, ix_id, **kwargs):
 
     def load(self):
         """Load data via from PeeringDB."""
-        ix_data = requests.get(f"{PEERINGDB_URL}/api/ix/{self.ix_id}").json()
+        headers = {}
+        if peeringdb_api_key:
+            headers["Authorization"] = f"Api-Key {peeringdb_api_key}"
+
+        ix_data = requests.get(f"{PEERINGDB_URL}/api/ix/{self.ix_id}", headers=headers).json()
 
         for fac in ix_data["data"][0]["fac_set"]:
             # PeeringDB has no Region entity, so we must avoid duplicates
             try:
-                self.get(self.region, fac["city"])
+                region = self.get(self.region, fac["city"])
             except ObjectNotFound:
                 # Use pycountry to translate the country code (like "DE") to a country name (like "Germany")
                 parent_name = pycountry.countries.get(alpha_2=fac["country"]).name
@@ -65,3 +71,5 @@ def load(self):
                 pk=fac["id"],
             )
             self.add(site)
+            region.add_child(site)
+            self.update(region)  # pylint: disable=no-member

examples/05-nautobot-peeringdb/creds.example.env

@@ -0,0 +1 @@
+PEERINGDB_API_KEY=""

examples/05-nautobot-peeringdb/docker-compose.yml

@@ -0,0 +1,17 @@
+---
+version: "3.8"
+services:
+  example:
+    build:
+      context: "./"
+      dockerfile: "Dockerfile"
+    tty: true
+    depends_on:
+      redis:
+        condition: "service_started"
+    volumes:
+      - "./:/local"
+    env_file:
+      - "creds.env"
+  redis:
+    image: "redis:6-alpine"

examples/05-nautobot-peeringdb/dockerfile

@@ -0,0 +1,14 @@
+ARG PYTHON_VER=3.8.10
+
+FROM python:${PYTHON_VER}-slim
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends git \
+    && apt-get purge -y --auto-remove \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /local
+COPY . /local
+
+RUN pip install --upgrade pip \
+    && pip install -r requirements.txt