Add a "raw" parser concept, also add sync device platform support for…

… hp_comware (#250)

* add first attempt at comware
* add raw parser and force to fake json payload
* fix new raw parser type
* fix comware yaml file
* add raw parser and add new sync devices platform hp_comware
* add raw parser docs

README.md

@@ -19,15 +19,15 @@ Regardless, the Onboarding App greatly simplifies the onboarding process by allo
 
 ### Support Matrix (Sync Devices From Network)
 
-|     Data Attribute      | Cisco IOS          | Cisco XE           | Cisco NXOS         | Cisco WLC          | Juniper Junos      | Arista EOS         | F5  |
-| ----------------------- | :----------------: |  :--------------:  |  :--------------:  |  :--------------:  |  :--------------:  |  :--------------:  | :-: |
-| Hostname                | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
-| Platform                | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
-| Manufacturer            | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
-| Serial Number           | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
-| Device Type             | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
-| Mgmt Interface          | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
-| Mgmt IP Address         | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
+|     Data Attribute      | Cisco IOS          | Cisco XE           | Cisco NXOS         | Cisco WLC          | Juniper Junos      | Arista EOS         | F5  | HP Comware |
+| ----------------------- | :----------------: |  :--------------:  |  :--------------:  |  :--------------:  |  :--------------:  |  :--------------:  | :-: | :-: |
+| Hostname                | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
+| Platform                | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
+| Manufacturer            | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
+| Serial Number           | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
+| Device Type             | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
+| Mgmt Interface          | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
+| Mgmt IP Address         | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | 🧪 |
 
 ### Support Matrix (Sync Data From Network)
 

changes/249.added

@@ -0,0 +1,2 @@
+Added raw parser choice to skip jpath extraction.
+Added initial hp_comware support for sync_devices job.

docs/user/app_yaml_overrides.md

@@ -9,8 +9,8 @@ There are only a few components to the file and they're described below:
 - `root key data name` - Is fully defined in the schema definition.
 - `commands` - List of commands to execute in order to get the required data.
 - `command` - Actual `show` command to execute.
-- `parser` - Whether to use a parser (textfsm, pyats, ttp, etc) alternatively `none` can be used if the platform supports some other method to return structured data. E.g. (`| display json`) or an equivalent.
-- `jpath` - The jmespath (specifically jdiffs implementation) to extract the data from the parsed json returned from parser.
+- `parser` - Whether to use a parser (textfsm, pyats, ttp, etc) alternatives are `none` which can be used if the platform supports some other method to return structured data. E.g. (`| display json`) or an equivalent, or `raw` which allows a command to be run and **NO** jmespath extraction to take place, this is useful when simple text extractions via the `post_processor` are good enough.
+- `jpath` - The jmespath (specifically jdiffs implementation) to extract the data from the parsed json returned from parser. If `raw` is used as the `parser` then `jpath` should also be set to `raw` which will be the dictionary key to extract the raw command data.
 - `post_processor` - Jinja2 capable code to further transform the returned data post jpath extraction.
 - `iterable_type` - A optional value to force a parsed result to a specific data type.
 

nautobot_device_onboarding/command_mappers/hp_comware.yml

@@ -0,0 +1,31 @@
+---
+sync_devices:
+  hostname:
+    commands:
+      - command: "display current | include sysname"
+        parser: "raw"
+        jpath: "raw"
+        post_processor: "{{ obj.lstrip().split(' ')[1] }}"
+  serial:
+    commands:
+      - command: "display device manuinfo"
+        parser: "textfsm"
+        jpath: "[?slot_type==`Slot`].device_serial_number"
+  device_type:
+    commands:
+      - command: "display device manuinfo"
+        parser: "textfsm"
+        jpath: "[?slot_type==`Slot`].device_name"
+  mgmt_interface:
+    commands:
+      - command: "display ip interface"
+        parser: "textfsm"
+        jpath: "[*].{interface: interface, ip_addr: ip_address[0]}"
+        post_processor: "{% for i in obj %}{% if i['ip_addr'] %}{% if original_host in i['ip_addr'] %}{{ i['interface'] }}{% endif %}{% endif %}{% endfor %}"
+  mask_length:
+    commands:
+      - command: "display ip interface"
+        parser: "textfsm"
+        jpath: "[*].ip_address[?contains(@, `{{ obj }}`)][]"
+        post_processor: "{{ obj[0].split('/')[1] }}"
+        iterable_type: "int"

nautobot_device_onboarding/nornir_plays/command_getter.py

@@ -156,6 +156,10 @@ def netmiko_send_commands(
                                 task.results[result_idx].result = []
                                 task.results[result_idx].failed = False
             else:
+                if command["parser"] == "raw":
+                    raw = {"raw": current_result.result}
+                    task.results[result_idx].result = json.loads(json.dumps(raw))
+                    task.results[result_idx].failed = False
                 if command["parser"] == "none":
                     try:
                         jsonified = json.loads(current_result.result)

nautobot_device_onboarding/tests/mock/cisco_wlc/sync_devices/expected_result_1.json

@@ -1,5 +1,7 @@
-{"device_type": "AIR-CT5520-K9",
- "hostname": "ABCWC-01",
- "mask_length": 28,
- "mgmt_interface": "management",
- "serial": "FCH2145V28L"}
+{
+    "device_type": "AIR-CT5520-K9",
+    "hostname": "ABCWC-01",
+    "mask_length": 28,
+    "mgmt_interface": "management",
+    "serial": "FCH2145V28L"
+}

nautobot_device_onboarding/tests/mock/hp_comware/command_getter_result_1.json

@@ -0,0 +1,147 @@
+{
+    "display current | include sysname": {
+        "raw": " sysname 133-SW-Core"
+      },
+    "display device manuinfo": [
+      {
+        "chassis_id": "",
+        "slot_type": "Slot",
+        "slot_id": "1",
+        "device_name": "5510 24G 4SFP+ HI JH145A",
+        "device_serial_number": "CN73H0Y0H9",
+        "manufacturing_date": "2017-03-14",
+        "vendor_name": "HPE",
+        "mac_address": "40B9-3CA0-4B2D"
+      },
+      {
+        "chassis_id": "",
+        "slot_type": "Subslot",
+        "slot_id": "1",
+        "device_name": "HP 5130/5510 10GbE SFP+ 2p Module JH157A",
+        "device_serial_number": "CN83H1B0DN",
+        "manufacturing_date": "2018-04-06",
+        "vendor_name": "HPE",
+        "mac_address": ""
+      },
+      {
+        "chassis_id": "",
+        "slot_type": "Fan",
+        "slot_id": "1",
+        "device_name": "",
+        "device_serial_number": "",
+        "manufacturing_date": "",
+        "vendor_name": "",
+        "mac_address": ""
+      },
+      {
+        "chassis_id": "",
+        "slot_type": "Fan",
+        "slot_id": "2",
+        "device_name": "",
+        "device_serial_number": "",
+        "manufacturing_date": "",
+        "vendor_name": "",
+        "mac_address": ""
+      },
+      {
+        "chassis_id": "",
+        "slot_type": "Power",
+        "slot_id": "1",
+        "device_name": "HPE X361 150W AC Power Supply JD362B",
+        "device_serial_number": "CN70JW52NY",
+        "manufacturing_date": "2017-10-23",
+        "vendor_name": "HPE",
+        "mac_address": ""
+      }
+    ],
+    "display ip interface": [
+      {
+        "interface": "GigabitEthernet1/0/4",
+        "line_status": "UP",
+        "protocol_status": "UP",
+        "route_map": "",
+        "ip_address": [
+          "10.51.100.1/29"
+        ],
+        "mtu": "1500"
+      },
+      {
+        "interface": "M-GigabitEthernet0/0/0",
+        "line_status": "DOWN",
+        "protocol_status": "DOWN",
+        "route_map": "",
+        "ip_address": [],
+        "mtu": ""
+      },
+      {
+        "interface": "Vlan-interface16",
+        "line_status": "UP",
+        "protocol_status": "UP",
+        "route_map": "",
+        "ip_address": [
+          "10.133.16.1/22"
+        ],
+        "mtu": "1500"
+      },
+      {
+        "interface": "Vlan-interface20",
+        "line_status": "UP",
+        "protocol_status": "UP",
+        "route_map": "",
+        "ip_address": [
+          "10.133.20.1/23"
+        ],
+        "mtu": "1500"
+      },
+      {
+        "interface": "Vlan-interface32",
+        "line_status": "UP",
+        "protocol_status": "UP",
+        "route_map": "",
+        "ip_address": [
+          "10.133.32.1/24"
+        ],
+        "mtu": "1500"
+      },
+      {
+        "interface": "Vlan-interface64",
+        "line_status": "UP",
+        "protocol_status": "UP",
+        "route_map": "",
+        "ip_address": [
+          "10.133.64.1/23"
+        ],
+        "mtu": "1500"
+      },
+      {
+        "interface": "Vlan-interface68",
+        "line_status": "UP",
+        "protocol_status": "UP",
+        "route_map": "",
+        "ip_address": [
+          "10.133.68.1/23"
+        ],
+        "mtu": "1500"
+      },
+      {
+        "interface": "Vlan-interface100",
+        "line_status": "UP",
+        "protocol_status": "UP",
+        "route_map": "",
+        "ip_address": [
+          "198.51.100.1/24"
+        ],
+        "mtu": "1500"
+      },
+      {
+        "interface": "Vlan-interface102",
+        "line_status": "UP",
+        "protocol_status": "UP",
+        "route_map": "",
+        "ip_address": [
+          "10.133.102.1/24"
+        ],
+        "mtu": "1500"
+      }
+    ]
+  }

nautobot_device_onboarding/tests/mock/hp_comware/sync_devices/expected_result_1.json

@@ -0,0 +1,7 @@
+{
+    "device_type": "5510 24G 4SFP+ HI JH145A",
+    "hostname": "133-SW-Core",
+    "mask_length": 24,
+    "mgmt_interface": "Vlan-interface100",
+    "serial": "CN73H0Y0H9"
+}

nautobot_device_onboarding/tests/test_formatter.py

@@ -427,7 +427,15 @@ def setUp(self, mock_repo):
 
     def test_add_platform_parsing_info_sane_defaults(self):
         # Note: This is also officially tested in test_transform, but secondary check here as well.
-        default_mappers = ["cisco_ios", "arista_eos", "cisco_wlc", "cisco_xe", "juniper_junos", "cisco_nxos"]
+        default_mappers = [
+            "cisco_ios",
+            "arista_eos",
+            "cisco_wlc",
+            "cisco_xe",
+            "juniper_junos",
+            "cisco_nxos",
+            "hp_comware",
+        ]
         self.assertEqual(sorted(default_mappers), list(sorted(self.platform_parsing_info.keys())))
 
     def test_create_inventory_host_per_platform(self):
@@ -510,6 +518,7 @@ def test_perform_data_extraction_simple_host_values(self):
     def test_perform_data_extraction_sync_network_data_no_options(self):
         supported_platforms = list(self.platform_parsing_info.keys())
         supported_platforms.remove("cisco_wlc")
+        supported_platforms.remove("hp_comware")
         for platform in supported_platforms:
             self.host.platform = platform
             current_test_dir = f"{MOCK_DIR}/{platform}/"
@@ -577,6 +586,7 @@ def test_perform_data_extraction_simple_host_values(self):
     def test_perform_data_extraction_sync_network_data_with_vrfs(self):
         supported_platforms = list(self.platform_parsing_info.keys())
         supported_platforms.remove("cisco_wlc")
+        supported_platforms.remove("hp_comware")
         for platform in supported_platforms:
             self.host.platform = platform
             current_test_dir = f"{MOCK_DIR}/{platform}/"
@@ -644,6 +654,7 @@ def test_perform_data_extraction_simple_host_values(self):
     def test_perform_data_extraction_sync_network_data_with_vlans(self):
         supported_platforms = list(self.platform_parsing_info.keys())
         supported_platforms.remove("cisco_wlc")
+        supported_platforms.remove("hp_comware")
         for platform in supported_platforms:
             self.host.platform = platform
             current_test_dir = f"{MOCK_DIR}/{platform}/"
@@ -712,6 +723,7 @@ def test_perform_data_extraction_simple_host_values(self):
     def test_perform_data_extraction_sync_network_data_all(self):
         supported_platforms = list(self.platform_parsing_info.keys())
         supported_platforms.remove("cisco_wlc")
+        supported_platforms.remove("hp_comware")
         for platform in supported_platforms:
             self.host.platform = platform
             current_test_dir = f"{MOCK_DIR}/{platform}/"

nautobot_device_onboarding/tests/test_transform.py

@@ -24,7 +24,15 @@ def setUp(self):
 
     def test_add_platform_parsing_info_sane_defaults(self):
         command_mappers = add_platform_parsing_info()
-        default_mappers = ["cisco_ios", "arista_eos", "cisco_wlc", "cisco_xe", "juniper_junos", "cisco_nxos"]
+        default_mappers = [
+            "cisco_ios",
+            "arista_eos",
+            "cisco_wlc",
+            "cisco_xe",
+            "juniper_junos",
+            "cisco_nxos",
+            "hp_comware",
+        ]
         self.assertEqual(sorted(default_mappers), list(sorted(command_mappers.keys())))
 
     def test_load_command_mappers_from_dir(self):