Merge tag 'v1.4.1' into nniehoff

.github/workflows/oui_filepull.yml

@@ -1,5 +1,5 @@
 ---
-name: "Flat"
+name: "OUI-Updates"
 
 on: # yamllint disable-line rule:truthy
   schedule:
@@ -11,6 +11,8 @@ jobs:
     env:
       BRANCH_NAME: "OUI_Updates"
     steps:
+      - name: "Delete existing branch"
+        run: "git branch -D $BRANCH_NAME || true"
       # Checkout repo
       - name: "Check out code"
         uses: "actions/checkout@v2"

.github/workflows/proto_filepull.yml

@@ -0,0 +1,53 @@
+---
+name: "Protocol-Updates"
+
+on: # yamllint disable-line rule:truthy
+  schedule:
+    - cron: "0 8 * * 4"
+
+jobs:
+  data_gathering:
+    runs-on: "ubuntu-latest"
+    env:
+      BRANCH_NAME: "PROTO_Updates"
+    steps:
+      # Checkout repo
+      - name: "Check out code"
+        uses: "actions/checkout@v2"
+        with:
+          ref: "develop"
+      # Create branch for Flatbot
+      - name: "Create Flatbot branch"
+        run: "git checkout -b $BRANCH_NAME"
+      # Push new branch so Flatbot can make its commit
+      - name: "Push Flatbot branch"
+        run: "git push --set-upstream origin $BRANCH_NAME"
+      # Install Black
+      - name: "Install Python Black"
+        run: "pip install black"
+      # This step installs Deno, which is a new Javascript runtime that improves on Node. Can be used for an optional postprocessing step
+      - name: "Setup deno"
+        uses: "denoland/setup-deno@main"
+        with:
+          deno-version: "v1.10.x"
+      # The Flat Action step. We fetch the data in the http_url and save it as downloaded_filename
+      - name: "Fetch data"
+        uses: "githubocto/flat@v3"
+        with:
+          http_url: "https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv"
+          downloaded_filename: "./netutils/data_files/protocol_mappings.py"
+          postprocess: "./flat_postprocess/protocol_postprocess.ts"
+  pr_creation:
+    runs-on: "ubuntu-latest"
+    needs: "data_gathering"
+    steps:
+      # Checkout repo
+      - name: "Check out code"
+        uses: "actions/checkout@v2"
+        with:
+          ref: "PROTO_Updates"
+      # Create PR from branch created above into develop
+      - name: "Create a Pull Request"
+        run: "gh pr create -B develop -H PROTO_Updates --title 'Flatbot PROTOCOL File Updates' --body 'Created by Flatbot action'"
+        env:
+          GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"

docs/admin/release_notes/version_1.4.md

@@ -8,6 +8,20 @@
 - Fix and cleanup tests and parser information.
 - Fix tcp_ping behavior.
 
+## [v1.4.1] - 2023-02
+
+### Changed
+
+- [#171](https://github.com/networktocode/netutils/pull/171) Allow protocol mapper to be auto-generated.
+- [#197](https://github.com/networktocode/netutils/pull/197) Prepend OUI Flatbot step to delete existing branch.
+- [#200](https://github.com/networktocode/netutils/pull/200) Allow bandwith to be represented without "ps" in bandwidth.
+
+### Fixed
+
+- [#206](https://github.com/networktocode/netutils/pull/206) Fix Passwords docs that show ASN details.
+- [#209](https://github.com/networktocode/netutils/pull/209) Fixed interface_range_expansion with trailing constant.
+- [#210](https://github.com/networktocode/netutils/pull/210) Install black into flat environment, change flat action name.
+
 ## [v1.4.0] - 2023-01
 
 ### Added

docs/dev/code_reference/password.md

@@ -1,5 +1,5 @@
-# BGP ASNs
+# Passwords
 
-::: netutils.asn
+::: netutils.password
     options:
         show_submodules: True

docs/user/lib_use_cases_protocol_mappers.md

@@ -64,4 +64,26 @@ print(SCTP_PORT_1021_APPLICATION_NAME)
 UDP_PORT_20_APPLICATION_NAME = UDP_NUM_TO_NAME[20]
 print(UDP_PORT_20_APPLICATION_NAME)
 # "ftp-data
-```
+```
+
+## Handling of Duplicate Services
+
+The ports are defined by iana and generated via an autogenerated process, however, there is not a strict name to number mapping provided by iana. Often names are duplicated across multiple port numbers. 
+
+Take the below example.
+
+|Service Name| Port Number| Transport Protocol| Description |
+|------------| -----------| ------------------| ------------|
+|compressnet| 2| tcp| Management Utility |
+|compressnet| 2| udp| Management Utility |
+|compressnet| 3| tcp| Compression Process |
+|compressnet| 3| udp| Compression Process |
+
+As you can see, the service name and port comboniation is not unique. As such, on the second one, a `-secondary` is appended to the name to accomdate. The resulting structure looks something like:
+
+```python
+    "compressnet": {"port_number": 2, "protocols": ["tcp", "udp"]},
+    "compressnet-secondary": {"port_number": 3, "protocols": ["tcp", "udp"]},
+```
+
+This behaviour is to be expected.

flat_postprocess/oui_postprocess.py

@@ -24,6 +24,6 @@
         oui_mappings.write("# pylint: disable=too-many-lines\n")
         oui_mappings.write("import typing\n\n")
         oui_mappings.write("OUI_MAPPINGS: typing.Dict[str, str] = {\n")
-        for mac, company in OUI_MAPPINGS.items():
+        for mac, company in sorted(OUI_MAPPINGS.items()):
             oui_mappings.write(f'    "{mac}": "{company}",\n')
         oui_mappings.write("}\n")

flat_postprocess/protocol_postprocess.py

@@ -0,0 +1,38 @@
+"""Python code used to postprocess Flat github action data related to Protocol mappings."""
+import csv
+import os
+import sys
+from urllib.request import urlopen
+
+
+if __name__ == "__main__":
+    if len(sys.argv) == 2:
+        URL = "https://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.csv"
+        oui_textfile = urlopen(URL).read().decode("utf-8")  # nosec: pylint: disable=consider-using-with
+        with open(sys.argv[1], "w", encoding="utf-8") as proto_mappings:
+            proto_mappings.write(oui_textfile)
+
+    protocol_mapping = {}
+    with open(sys.argv[1], encoding="utf-8") as file:
+        next(file)
+        reader = csv.reader(file)
+        for row in reader:
+            name = row[0]
+            port = row[1]
+            protocol = row[2]
+            if not port.isnumeric():
+                continue
+            if name and port and protocol:
+                if protocol_mapping.get(name) and protocol_mapping[name]["port_number"] != int(port):
+                    name = name + "-secondary"
+                if not protocol_mapping.get(name):
+                    protocol_mapping[name] = {"port_number": int(port), "protocols": []}
+                protocol_mapping[name]["protocols"].append(protocol)
+
+    with open(sys.argv[1], "w", encoding="utf-8") as proto_mappings:
+        proto_mappings.write('"""Dictionary object to store Protocol information."""\n')
+        proto_mappings.write("# pylint: disable=too-many-lines\n")
+        proto_mappings.write("from typing import Any, Dict\n\n")
+        proto_mappings.write(f"PROTOCOLS: Dict[str, Any] = {protocol_mapping}")
+
+    os.system(f"black {sys.argv[1]}")  # nosec

flat_postprocess/protocol_postprocess.ts

@@ -0,0 +1,7 @@
+// Forwards the execution to the python script
+const py_run = Deno.run({
+    // Run `python ./flat_postprocess/protocol_postprocess.py netutils/data_files/protocol_mappings.py` to test locally.
+    cmd: ["python", "./flat_postprocess/protocol_postprocess.py"].concat(Deno.args),
+});
+
+await py_run.status();

netutils/bandwidth.py

@@ -35,6 +35,32 @@ def _get_bytes_mapping() -> t.Dict[str, t.Dict[str, int]]:
 BYTES_MAPPING = _get_bytes_mapping()
 
 
+def _normalize_bw(speed: str) -> str:
+    per_second_mapping = {
+        "b": "bps",
+        "Kb": "Kbps",
+        "Mb": "Mbps",
+        "Gb": "Gbps",
+        "Tb": "Tbps",
+        "Pb": "Pbps",
+        "Eb": "Ebps",
+        "Zb": "Zbps",
+        "B": "Bps",
+        "KB": "KBps",
+        "MB": "MBps",
+        "GB": "GBps",
+        "TB": "TBps",
+        "PB": "PBps",
+        "EB": "EBps",
+        "ZB": "ZBps",
+    }
+    speed = speed.replace(" ", "")
+    tail = speed.lstrip(r"0123456789")
+    head = speed[: -len(tail)].rstrip()
+    tail = per_second_mapping.get(tail, tail)
+    return f"{head}{tail}"
+
+
 def name_to_bits(speed: str) -> int:
     """Method to convert a short bandwidth name to int value in bps.
 
@@ -52,10 +78,12 @@ def name_to_bits(speed: str) -> int:
         33600
         >>> name_to_bits("2.5Gbps")
         2500000000
+        >>> name_to_bits('100 MB')
+        800000000
     """
     if not isinstance(speed, str):
         raise ValueError(f"Speed of {speed} was not a valid speed representation.")
-    speed = speed.replace(" ", "")
+    speed = _normalize_bw(speed)
     match = re.match(r"([0-9.]+)([A-Z]?[bB]ps)", speed)
     if not match:
         raise ValueError(f"Speed of {speed} was not a valid speed representation.")
@@ -84,10 +112,12 @@ def name_to_bytes(speed: str) -> float:
         12500000.0
         >>> name_to_bytes("100GBps")
         100000000000.0
+        >>> name_to_bytes('100 GB')
+        100000000000.0
     """
     if not isinstance(speed, str):
         raise ValueError(f"Speed of {speed} was not a valid speed representation.")
-    speed = speed.replace(" ", "")
+    speed = _normalize_bw(speed)
     match = re.match(r"([0-9.]+)([A-Z]?[bB]ps)", speed)
     if not match:
         raise ValueError(f"Speed of {speed} was not a valid speed representation.")

netutils/constants.py

@@ -1,11 +1,6 @@
 """Constant definitions used in project."""
-import json
-from os.path import dirname
-from netutils import __file__ as netutils_file
+from netutils.data_files.protocol_mappings import PROTOCOLS  # noqa: F401 # pylint:disable=unused-import
 
-# Load the PROTOCOLS json file.
-with open("/".join([dirname(netutils_file), "protocols.json"]), encoding="utf-8") as fh:
-    PROTOCOLS = json.loads(fh.read())
 
 # This variable provides mapping for known interface variants, to the associated long form.
 BASE_INTERFACES = {