Merge pull request #2 from dailymotion/feature/chriss/external_drive_bay

Manage blade expansions as independent devices

netbox_agent/config.py

@@ -28,6 +28,8 @@ def get_config():
     p.add_argument('--update-inventory', action='store_true', help='Update inventory')
     p.add_argument('--update-location', action='store_true', help='Update location')
     p.add_argument('--update-psu', action='store_true', help='Update PSU')
+    p.add_argument('--expansion-as-device', action='store_true',
+                   help='Manage blade expansions as external devices')
 
     p.add_argument('--log_level', default='debug')
     p.add_argument('--netbox.url', help='Netbox URL')

netbox_agent/inventory.py

@@ -46,10 +46,11 @@ class Inventory():
     - no scan of NVMe devices
     """
 
-    def __init__(self, server):
+    def __init__(self, server, update_expansion=False):
         self.create_netbox_tags()
         self.server = server
-        netbox_server = self.server.get_netbox_server()
+        self.update_expansion = update_expansion
+        netbox_server = self.server.get_netbox_server(update_expansion)
 
         self.device_id = netbox_server.id if netbox_server else None
         self.raid = None
@@ -220,7 +221,7 @@ def do_netbox_cpus(self):
 
             self.create_netbox_cpus()
 
-    def get_raid_cards(self):
+    def get_raid_cards(self, filter_cards=False):
         raid_class = None
         if self.server.manufacturer == 'Dell':
             if is_tool('omreport'):
@@ -235,9 +236,15 @@ def get_raid_cards(self):
             return []
 
         self.raid = raid_class()
-        controllers = self.raid.get_controllers()
-        if len(self.raid.get_controllers()):
-            return controllers
+
+        if filter_cards and config.expansion_as_device \
+                and self.server.own_expansion_slot():
+            return [
+                c for c in self.raid.get_controllers()
+                if c.is_external() is self.update_expansion
+            ]
+        else:
+            return self.raid.get_controllers()
 
     def create_netbox_raid_card(self, raid_card):
         manufacturer = self.find_or_create_manufacturer(
@@ -276,7 +283,7 @@ def do_netbox_raid_cards(self):
             device_id=self.device_id,
             tag=[INVENTORY_TAG['raid_card']['slug']]
         )
-        raid_cards = self.get_raid_cards()
+        raid_cards = self.get_raid_cards(filter_cards=True)
 
         # delete cards that are in netbox but not locally
         # use the serial_number has the comparison element
@@ -336,7 +343,7 @@ def get_hw_disks(self):
                 d['Vendor'] = get_vendor(disk['product'])
             disks.append(d)
 
-        for raid_card in self.get_raid_cards():
+        for raid_card in self.get_raid_cards(filter_cards=True):
             disks += raid_card.get_physical_disks()
 
         # remove duplicate serials
@@ -463,21 +470,24 @@ def do_netbox_gpus(self):
             tag=INVENTORY_TAG['gpu']['slug'],
         )
 
-        if not len(nb_gpus) or \
+        if config.expansion_as_device and len(nb_gpus):
+            for x in nb_gpus:
+                x.delete()
+        elif not len(nb_gpus) or \
            len(nb_gpus) and len(gpus) != len(nb_gpus):
             for x in nb_gpus:
                 x.delete()
-
             self.create_netbox_gpus()
 
     def create_or_update(self):
         if config.inventory is None or config.update_inventory is None:
             return False
-        self.do_netbox_cpus()
-        self.do_netbox_memories()
-        self.do_netbox_raid_cards()
-        self.do_netbox_disks()
-        self.do_netbox_interfaces()
-        self.do_netbox_motherboard()
+        if self.update_expansion is False:
+            self.do_netbox_cpus()
+            self.do_netbox_memories()
+            self.do_netbox_interfaces()
+            self.do_netbox_motherboard()
         self.do_netbox_gpus()
+        self.do_netbox_disks()
+        self.do_netbox_raid_cards()
         return True

netbox_agent/raid/base.py

@@ -15,6 +15,9 @@ def get_firmware_version(self):
     def get_physical_disks(self):
         raise NotImplementedError
 
+    def is_external(self):
+        return False
+
 
 class Raid():
     def get_controllers(self):

netbox_agent/raid/hp.py

@@ -1,106 +1,65 @@
 import re
 import subprocess
 
+from netbox_agent.config import config
 from netbox_agent.misc import get_vendor
 from netbox_agent.raid.base import Raid, RaidController
 
 REGEXP_CONTROLLER_HP = re.compile(r'Smart Array ([a-zA-Z0-9- ]+) in Slot ([0-9]+)')
 
 
-def _get_indentation(string):
-    """Return the number of spaces before the current line."""
-    return len(string) - len(string.lstrip(' '))
-
-
-def _get_key_value(string):
-    """Return the (key, value) as a tuple from a string."""
-    # Normally all properties look like this:
-    #   Unique Identifier: 600508B1001CE4ACF473EE9C826230FF
-    #   Disk Name: /dev/sda
-    #   Mount Points: None
-    key = ''
-    value = ''
-    try:
-        key, value = string.split(':')
-    except ValueError:
-        # This handles the case when the property of a logical drive
-        # returned is as follows. Here we cannot split by ':' because
-        # the disk id has colon in it. So if this is about disk,
-        # then strip it accordingly.
-        #   Mirror Group 0: physicaldrive 6I:1:5
-        string = string.lstrip(' ')
-        if string.startswith('physicaldrive'):
-            fields = string.split(' ')
-            key = fields[0]
-            value = fields[1]
-        else:
-            # TODO(rameshg87): Check if this ever occurs.
-            return None, None
-
-    return key.lstrip(' ').rstrip(' '), value.lstrip(' ').rstrip(' ')
-
-
-def _get_dict(lines, start_index, indentation):
-    """Recursive function for parsing hpssacli/ssacli output."""
-
-    info = {}
-    current_item = None
-
-    i = start_index
-    while i < len(lines):
-        current_line = lines[i]
-        if current_line.startswith('Note:'):
-            i = i + 1
-            continue
+def _parse_ctrl_output(lines):
+    controllers = {}
+    current_ctrl = None
 
-        current_line_indentation = _get_indentation(current_line)
-        # This check ignore some useless information that make
-        # crash the parsing
-        product_name = REGEXP_CONTROLLER_HP.search(current_line)
-        if current_line_indentation == 0 and not product_name:
-            i = i + 1
+    for line in lines:
+        if not line or line.startswith('Note:'):
+            continue
+        ctrl = REGEXP_CONTROLLER_HP.search(line)
+        if ctrl is not None:
+            current_ctrl = ctrl.group(1)
+            controllers[current_ctrl] = {"Slot": ctrl.group(2)}
+            if "Embedded" not in line:
+                controllers[current_ctrl]["External"] = True
             continue
+        attr, val = line.split(": ", 1)
+        attr = attr.strip()
+        val = val.strip()
+        controllers[current_ctrl][attr] = val
+    return controllers
 
-        if current_line_indentation == indentation:
-            current_item = current_line.lstrip(' ')
 
-            info[current_item] = {}
-            i = i + 1
-            continue
+def _parse_pd_output(lines):
+    drives = {}
+    current_array = None
+    current_drv = None
 
-        if i >= len(lines) - 1:
-            key, value = _get_key_value(current_line)
-            # If this is some unparsable information, then
-            # just skip it.
-            if key:
-                info[current_item][key] = value
-            return info, i
-
-        next_line = lines[i + 1]
-        next_line_indentation = _get_indentation(next_line)
-
-        if current_line_indentation == next_line_indentation:
-            key, value = _get_key_value(current_line)
-            if key:
-                info[current_item][key] = value
-            i = i + 1
-        elif next_line_indentation > current_line_indentation:
-            ret_dict, j = _get_dict(lines, i, current_line_indentation)
-            info[current_item].update(ret_dict)
-            i = j + 1
-        elif next_line_indentation < current_line_indentation:
-            key, value = _get_key_value(current_line)
-            if key:
-                info[current_item][key] = value
-            return info, i
-
-    return info, i
+    for line in lines:
+        line = line.strip()
+        if not line or line.startswith('Note:'):
+            continue
+        # Parses the Array the drives are in
+        if line.startswith("Array"):
+            current_array = line.split(None, 1)[1]
+        # Detects new physical drive
+        if line.startswith("physicaldrive"):
+            current_drv = line.split(None, 1)[1]
+            drives[current_drv] = {}
+            if current_array is not None:
+                drives[current_drv]["Array"] = current_array
+            continue
+        if ": " not in line:
+            continue
+        attr, val = line.split(": ", 1)
+        drives.setdefault(current_drv, {})[attr] = val
+    return drives
 
 
 class HPRaidController(RaidController):
     def __init__(self, controller_name, data):
         self.controller_name = controller_name
         self.data = data
+        self.drives = self._get_physical_disks()
 
     def get_product_name(self):
         return self.controller_name
@@ -114,40 +73,42 @@ def get_serial_number(self):
     def get_firmware_version(self):
         return self.data['Firmware Version']
 
-    def get_physical_disks(self):
-        ret = []
+    def is_external(self):
+        return self.data.get('External', False)
+
+    def _get_physical_disks(self):
         output = subprocess.getoutput(
             'ssacli ctrl slot={slot} pd all show detail'.format(slot=self.data['Slot'])
         )
         lines = output.split('\n')
         lines = list(filter(None, lines))
-        j = -1
-        while j < len(lines):
-            info_dict, j = _get_dict(lines, j + 1, 0)
-
-        key = next(iter(info_dict))
-        for array, physical_disk in info_dict[key].items():
-            for _, pd_attr in physical_disk.items():
-                model = pd_attr.get('Model', '').strip()
-                vendor = None
-                if model.startswith('HP'):
-                    vendor = 'HP'
-                elif len(model.split()) > 1:
-                    vendor = get_vendor(model.split()[1])
-                else:
-                    vendor = get_vendor(model)
-
-                ret.append({
-                    'Model': model,
-                    'Vendor': vendor,
-                    'SN': pd_attr.get('Serial Number', '').strip(),
-                    'Size': pd_attr.get('Size', '').strip(),
-                    'Type': 'SSD' if pd_attr.get('Interface Type') == 'Solid State SATA'
-                    else 'HDD',
-                    '_src': self.__class__.__name__,
-                })
+        drives = _parse_pd_output(lines)
+        ret = []
+
+        for name, attrs in drives.items():
+            model = attrs.get('Model', '').strip()
+            vendor = None
+            if model.startswith('HP'):
+                vendor = 'HP'
+            elif len(model.split()) > 1:
+                vendor = get_vendor(model.split()[1])
+            else:
+                vendor = get_vendor(model)
+
+            ret.append({
+                'Model': model,
+                'Vendor': vendor,
+                'SN': attrs.get('Serial Number', '').strip(),
+                'Size': attrs.get('Size', '').strip(),
+                'Type': 'SSD' if attrs.get('Interface Type') == 'Solid State SATA'
+                else 'HDD',
+                '_src': self.__class__.__name__,
+            })
         return ret
 
+    def get_physical_disks(self):
+        return self.drives
+
 
 class HPRaid(Raid):
     def __init__(self):
@@ -158,16 +119,11 @@ def __init__(self):
     def convert_to_dict(self):
         lines = self.output.split('\n')
         lines = list(filter(None, lines))
-        j = -1
-        while j < len(lines):
-            info_dict, j = _get_dict(lines, j + 1, 0)
-            if len(info_dict.keys()):
-                _product_name = list(info_dict.keys())[0]
-                product_name = REGEXP_CONTROLLER_HP.search(_product_name)
-                if product_name:
-                    self.controllers.append(
-                        HPRaidController(product_name.group(1), info_dict[_product_name])
-                    )
+        controllers = _parse_ctrl_output(lines)
+        for controller, attrs in controllers.items():
+            self.controllers.append(
+                HPRaidController(controller, attrs)
+            )
 
     def get_controllers(self):
         return self.controllers