From a13c62f4f7f7a91a4b7de81ce0f0d5a5daf93140 Mon Sep 17 00:00:00 2001 From: Thomas Scott Date: Fri, 10 Nov 2023 14:39:21 -0500 Subject: [PATCH] allow multiple sites --- README.md | 12 +++++----- docs/CONFIGURATION.md | 4 ++-- jupyter/ntopex.ipynb | 14 ++++++------ nrx.conf | 4 ++-- nrx/nrx.py | 51 ++++++++++++++++++++++++------------------- tests/colo/nrx.conf | 4 ++-- tests/dc1/nrx.conf | 4 ++-- tests/dc2/nrx.conf | 4 ++-- tests/h88/nrx.conf | 4 ++-- tests/lrg/nrx.conf | 4 ++-- tests/site1/nrx.conf | 4 ++-- 11 files changed, 58 insertions(+), 51 deletions(-) diff --git a/README.md b/README.md index c1ff11f..e6de12e 100644 --- a/README.md +++ b/README.md @@ -138,7 +138,7 @@ optional arguments: -i, --input INPUT input source: netbox (default) | cyjs -o, --output OUTPUT output format: cyjs | gml | clab | cml | graphite | d2 -a, --api API netbox API URL - -s, --site SITE netbox site to export + -s, --sites SITES netbox site(s) to export, for multiple sites use a comma-separated list: site1,site2,site3 (uses OR logic) -t, --tags TAGS netbox tags to export, for multiple tags use a comma-separated list: tag1,tag2,tag3 (uses AND logic) -n, --noconfigs disable device configuration export (enabled by default) -k, --insecure allow insecure server connections when using TLS @@ -194,7 +194,7 @@ source nrx39/bin/activate ```Shell export NB_API_TOKEN='replace_with_valid_API_token' - ./nrx.py --api https://demo.netbox.dev --templates templates --output clab --dir demo --site DM-Albany + ./nrx.py --api https://demo.netbox.dev --templates templates --output clab --dir demo --sites DM-Albany ``` 2. Now you're ready to start the Containerlab topology. Here is the example for "DM-Albany" site @@ -207,7 +207,7 @@ source nrx39/bin/activate ```Shell export NB_API_TOKEN='replace_with_valid_API_token' - ./nrx.py --api https://demo.netbox.dev --site DM-Albany --dir demo + ./nrx.py --api https://demo.netbox.dev --sites DM-Albany --dir demo ``` 5. If you have a CYJS file, run `./nrx.py --input cyjs --file .cyjs --output clab` to create a Containerlab topology file from the CYJS graph you exported in the previous step. For example, run: @@ -222,7 +222,7 @@ source nrx39/bin/activate ```Shell export NB_API_TOKEN='replace_with_valid_API_token' - ./nrx.py --api https://demo.netbox.dev --templates templates --output cml --dir demo --site DM-Akron + ./nrx.py --api https://demo.netbox.dev --templates templates --output cml --dir demo --sites DM-Akron ``` 2. Now you're ready to start the "DM-Akron" topology in CML. @@ -237,7 +237,7 @@ source nrx39/bin/activate ```Shell export NB_API_TOKEN='replace_with_valid_API_token' - ./nrx.py --api https://demo.netbox.dev --dir demo --site DM-Akron + ./nrx.py --api https://demo.netbox.dev --dir demo --sites DM-Akron ``` 4. If you have a CYJS file, run `./nrx.py --input cyjs --file .cyjs --output cml` to create a topology file from the CYJS graph you exported in the previous step. For example, run: @@ -256,7 +256,7 @@ Follow a two-step process: ```Shell export NB_API_TOKEN='replace_with_valid_API_token' - ./nrx.py --api https://demo.netbox.dev --site DM-Akron --templates templates --output graphite + ./nrx.py --api https://demo.netbox.dev --sites DM-Akron --templates templates --output graphite ``` 2. Start Graphite to visualize "DM-Akron" site: diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md index a32c689..032cc11 100644 --- a/docs/CONFIGURATION.md +++ b/docs/CONFIGURATION.md @@ -29,8 +29,8 @@ TEMPLATES_PATH = ['templates'] # List of NetBox Device Roles to export EXPORT_DEVICE_ROLES = ['router', 'core-switch', 'distribution-switch', 'access-switch', 'tor-switch', 'server'] -# NetBox Site to export. Alternatively, use --site argument -EXPORT_SITE = 'DM-Akron' +# NetBox Site to export. Alternatively, use --sites argument +EXPORT_SITES = ['DM-Akron'] # NetBox tags to export. Alternatively, use --tags argument EXPORT_TAGS = [] # Export device configurations, when available diff --git a/jupyter/ntopex.ipynb b/jupyter/ntopex.ipynb index 0d578fa..f7480d8 100644 --- a/jupyter/ntopex.ipynb +++ b/jupyter/ntopex.ipynb @@ -24,7 +24,7 @@ "metadata": {}, "outputs": [], "source": [ - "export_site = \"nr-1\"\n", + "export_sites = [\"nr-1\"]\n", "nb_api_url = 'https://demo.netbox.dev'\n", "nb_api_token = 'd2fa978199167d24f75aa477fc24a1a59eba9db7'\n", "export_device_roles = [\"router\", \"core-switch\", \"access-switch\", \"distribution-switch\", \"tor-switch\"]" @@ -107,7 +107,7 @@ } ], "source": [ - "s = nb.dcim.sites.get(name=export_site)\n", + "s = nb.dcim.sites.get(name=export_sites)\n", "nodes, devices, interfaces = [], [], []\n", "device_ids, interface_ids, cable_ids = [], [], []\n", "for device in list(nb.dcim.devices.filter(site_id=s.id,role=export_device_roles)):\n", @@ -167,7 +167,7 @@ ], "source": [ "import networkx as nx\n", - "G = nx.Graph(name=export_site)\n", + "G = nx.Graph(name=export_sites)\n", "\n", "for cable in list(nb.dcim.cables.filter(id=cable_ids)):\n", " if len(cable.a_terminations) == 1 and len(cable.b_terminations) == 1:\n", @@ -235,7 +235,7 @@ "id": "2a78d1be", "metadata": {}, "source": [ - "Serialize the graph into GML format and save it as `.gml`" + "Serialize the graph into GML format and save it as `.gml`" ] }, { @@ -400,7 +400,7 @@ ], "source": [ "print(\"\\n\".join(nx.generate_gml(G)))\n", - "nx.write_gml(G, export_site+\".gml\")\n" + "nx.write_gml(G, export_sites+\".gml\")\n" ] }, { @@ -409,7 +409,7 @@ "id": "9cd785d6", "metadata": {}, "source": [ - "Also export in Cytoscape JSON format `.cyjs`" + "Also export in Cytoscape JSON format `.cyjs`" ] }, { @@ -634,7 +634,7 @@ "import json\n", "cyjs = nx.cytoscape_data(G)\n", "print(json.dumps(cyjs, indent=4))\n", - "with open(export_site + \".cyjs\", 'w', encoding='utf-8') as f:\n", + "with open(export_sites + \".cyjs\", 'w', encoding='utf-8') as f:\n", " json.dump(cyjs, f, indent=4)" ] } diff --git a/nrx.conf b/nrx.conf index fd712ff..401b608 100644 --- a/nrx.conf +++ b/nrx.conf @@ -12,8 +12,8 @@ OUTPUT_FORMAT = 'clab' OUTPUT_DIR = 'demo' # List of NetBox Device Roles to export EXPORT_DEVICE_ROLES = ['router', 'core-switch', 'distribution-switch', 'access-switch', 'tor-switch', 'server'] -# NetBox Site to export. Alternatively, use --site argument -EXPORT_SITE = 'DM-Akron' +# NetBox Site to export. Alternatively, use --sites argument +EXPORT_SITES = ['DM-Akron'] # NetBox tags to export. Alternatively, use --tags argument EXPORT_TAGS = [] # Export device configurations, when available diff --git a/nrx/nrx.py b/nrx/nrx.py index 276fad8..13505d7 100755 --- a/nrx/nrx.py +++ b/nrx/nrx.py @@ -123,15 +123,17 @@ class NBFactory: def __init__(self, config): self.config = config self.nb_net = NBNetwork() - if len(config['export_site']) > 0: - self.topology_name = config['export_site'] + if len(config['export_sites']) > 1: + self.topology_name = "__".join(config['export_sites']) + elif len(config['export_sites']) > 0: + self.topology_name = config['export_sites'][0] elif len(config['export_tags']) > 0: self.topology_name = "-".join(config['export_tags']) self.G = nx.Graph(name=self.topology_name) self.nb_session = pynetbox.api(self.config['nb_api_url'], token=self.config['nb_api_token'], threading=True) - self.nb_site = None + self.nb_sites = None if not config['tls_validate']: self.nb_session.http_session.verify = False urllib3.disable_warnings() @@ -140,16 +142,16 @@ def __init__(self, config): self.nb_session.http_session.mount("http://", adapter) self.nb_session.http_session.mount("https://", adapter) print(f"Connecting to NetBox at: {config['nb_api_url']}") - if len(config['export_site']) > 0: - debug(f"Fetching site: {config['export_site']}") + if len(config['export_sites']) > 0: + debug(f"Fetching sites: {config['export_sites']}") try: - self.nb_site = self.nb_session.dcim.sites.get(name=config['export_site']) + self.nb_sites = self.nb_session.dcim.sites.filter(name=config['export_sites']) except (pynetbox.core.query.RequestError, pynetbox.core.query.ContentError) as e: error("NetBox API failure at get site:", e) - if self.nb_site is None: - error(f"Site not found: {config['export_site']}") + if self.nb_sites is None: + error(f"One of these site not found: {config['export_site']}") else: - print(f"Fetching devices from site: {config['export_site']}") + print(f"Fetching devices from sites: {config['export_sites']}") else: print(f"Fetching devices with tags: {','.join(config['export_tags'])}") @@ -189,15 +191,20 @@ def _get_nb_objects(self, kind, block_size): def _get_nb_devices(self): """Get device list from NetBox filtered by site, tags and device roles""" - devices = None - if self.nb_site is None: - devices = self.nb_session.dcim.devices.filter(tag=self.config['export_tags'], + devices = [] + if len(self.nb_sites) == 0: + devices.append(self.nb_session.dcim.devices.filter(tag=self.config['export_tags'], role=self.config['export_device_roles']) + ) else: - devices = self.nb_session.dcim.devices.filter(site_id=self.nb_site.id, - tag=self.config['export_tags'], - role=self.config['export_device_roles']) - for device in list(devices): + site_ids = [] + for site in self.nb_sites: + debug(f'Site ID: {site.id} - Site Name: {site.name}') + site_ids.append(str(site.id)) + devices = self.nb_session.dcim.devices.filter(site_id=site_ids, + tag=self.config['export_tags'], + role=self.config['export_device_roles']) + for device in devices: d = self._init_device(device) self.nb_net.nodes.append(d) d["node_id"] = len(self.nb_net.nodes) - 1 @@ -766,7 +773,7 @@ def parse_args(): parser.add_argument('-o', '--output', required=False, help='output format: cyjs | gml | clab | cml | graphite | d2', type=arg_output_check, ) parser.add_argument('-a', '--api', required=False, help='netbox API URL') - parser.add_argument('-s', '--site', required=False, help='netbox site to export') + parser.add_argument('-s', '--sites', required=False, help='netbox site to export') parser.add_argument('-t', '--tags', required=False, help='netbox tags to export, for multiple tags use a comma-separated list: tag1,tag2,tag3 (uses AND logic)') parser.add_argument('-n', '--noconfigs', required=False, help='disable device configuration export (enabled by default)', action=argparse.BooleanOptionalAction) @@ -810,7 +817,7 @@ def load_toml_config(filename): 'super-spine': 3, 'router': 4, }, - 'export_site': '', + 'export_sites': [], 'export_tags': [], 'export_configs': True, 'templates_path': ['.'], @@ -846,13 +853,13 @@ def config_apply_netbox_args(config, args): error("Need an API URL to connect to NetBox.\nUse --api argument, NB_API_URL environment variable or key in --config file") if len(config['nb_api_token']) == 0: error("Need an API token to connect to NetBox.\nUse NB_API_TOKEN environment variable or key in --config file") - if args.site is not None and len(args.site) > 0: - config['export_site'] = args.site + if args.sites is not None and len(args.sites) > 0: + config['export_sites'] = args.sites.split(',') if args.tags is not None and len(args.tags) > 0: config['export_tags'] = args.tags.split(',') debug(f"List of tags to filter devices for export: {config['export_tags']}") - if len(config['export_site']) == 0 and len(config['export_tags']) == 0: - error("Need a Site name or Tags to export. Use --site/--tags arguments, or EXPORT_SITE/EXPORT_TAGS key in --config file") + if len(config['export_sites']) == 0 and len(config['export_tags']) == 0: + error("Need a Site name or Tags to export. Use --sites/--tags arguments, or EXPORT_SITE/EXPORT_TAGS key in --config file") if args.noconfigs is not None: if args.noconfigs: config['export_configs'] = False diff --git a/tests/colo/nrx.conf b/tests/colo/nrx.conf index 4bb5b01..4a9be38 100644 --- a/tests/colo/nrx.conf +++ b/tests/colo/nrx.conf @@ -4,7 +4,7 @@ OUTPUT_FORMAT = 'cyjs' TEMPLATES_PATH = ['../../../templates'] # List of NetBox Device Roles to export EXPORT_DEVICE_ROLES = ['access-switch'] -# NetBox Site to export. Alternatively, use --site argument -EXPORT_SITE = 'colo' +# NetBox Site to export. Alternatively, use --sites argument +EXPORT_SITES = ['colo'] # Output directory. Alternatively, use --dir argument OUTPUT_DIR = '.' \ No newline at end of file diff --git a/tests/dc1/nrx.conf b/tests/dc1/nrx.conf index 6316b32..f094366 100644 --- a/tests/dc1/nrx.conf +++ b/tests/dc1/nrx.conf @@ -4,7 +4,7 @@ OUTPUT_FORMAT = 'clab' TEMPLATES_PATH = ['../../../templates'] # List of NetBox Device Roles to export EXPORT_DEVICE_ROLES = ['server','access-switch','leaf','spine','tor-switch'] -# NetBox Site to export. Alternatively, use --site argument -EXPORT_SITE = 'dc1' +# NetBox Site to export. Alternatively, use --sites argument +EXPORT_SITES = ['dc1'] # Output directory. Alternatively, use --dir argument OUTPUT_DIR = '.' \ No newline at end of file diff --git a/tests/dc2/nrx.conf b/tests/dc2/nrx.conf index 1cd1b04..3c38c0f 100644 --- a/tests/dc2/nrx.conf +++ b/tests/dc2/nrx.conf @@ -6,7 +6,7 @@ TEMPLATES_PATH = ['../../../templates'] TLS_VALIDATE = false # List of NetBox Device Roles to export EXPORT_DEVICE_ROLES = ['tor-switch', 'spine', 'server'] -# NetBox Site to export. Alternatively, use --site argument -EXPORT_SITE = 'dc2' +# NetBox Site to export. Alternatively, use --sites argument +EXPORT_SITES = ['dc2'] # Output directory. Alternatively, use --dir argument OUTPUT_DIR = '.' \ No newline at end of file diff --git a/tests/h88/nrx.conf b/tests/h88/nrx.conf index 0ef436a..faa8d8a 100644 --- a/tests/h88/nrx.conf +++ b/tests/h88/nrx.conf @@ -4,8 +4,8 @@ TLS_VALIDATE = true OUTPUT_FORMAT = 'clab' # List of NetBox Device Roles to export EXPORT_DEVICE_ROLES = ['router', 'core-switch', 'distribution-switch', 'access-switch', 'tor-switch'] -# NetBox Site to export. Alternatively, use --site argument -EXPORT_SITE = 'HQ' +# NetBox Site to export. Alternatively, use --sites argument +EXPORT_SITES = ['HQ'] # Templates path TEMPLATES_PATH = ['../../../templates'] # Output directory. Alternatively, use --dir argument diff --git a/tests/lrg/nrx.conf b/tests/lrg/nrx.conf index f41d56a..053ed3f 100644 --- a/tests/lrg/nrx.conf +++ b/tests/lrg/nrx.conf @@ -4,8 +4,8 @@ OUTPUT_FORMAT = 'clab' TEMPLATES_PATH = ['../../../templates'] # List of NetBox Device Roles to export EXPORT_DEVICE_ROLES = ['server','leaf','spine','core'] -# NetBox Site to export. Alternatively, use --site argument -EXPORT_SITE = 'lrg' +# NetBox Site to export. Alternatively, use --sites argument +EXPORT_SITES = ['lrg'] # Output directory. Alternatively, use --dir argument OUTPUT_DIR = '.' # Netbox API bulk queries optimization diff --git a/tests/site1/nrx.conf b/tests/site1/nrx.conf index 502d7de..c74d5b9 100644 --- a/tests/site1/nrx.conf +++ b/tests/site1/nrx.conf @@ -4,7 +4,7 @@ OUTPUT_FORMAT = 'cyjs' TEMPLATES_PATH = ['../../../templates'] # List of NetBox Device Roles to export EXPORT_DEVICE_ROLES = ['router','server'] -# NetBox Site to export. Alternatively, use --site argument -EXPORT_SITE = 'site1' +# NetBox Site to export. Alternatively, use --sites argument +EXPORT_SITES = ['site1'] # Output directory. Alternatively, use --dir argument OUTPUT_DIR = '.' \ No newline at end of file