From a1a394fbe6e4185860d12187bdfc8c74cf0d39b6 Mon Sep 17 00:00:00 2001 From: Christophe Henry Date: Tue, 17 Sep 2024 17:23:12 +0200 Subject: [PATCH 1/2] Storage mangment API: disks infos --- debian/control | 1 + share/actionsmap.yml | 15 +++++++ src/disks.py | 101 +++++++++++++++++++++++++++++++++++++++++++ src/storage.py | 4 ++ src/utils/disks.py | 12 +++++ 5 files changed, 133 insertions(+) create mode 100644 src/disks.py create mode 100644 src/storage.py create mode 100644 src/utils/disks.py diff --git a/debian/control b/debian/control index 06a665dec6..dc95129334 100644 --- a/debian/control +++ b/debian/control @@ -15,6 +15,7 @@ Depends: ${python3:Depends}, ${misc:Depends} , python3-miniupnpc, python3-dbus, python3-jinja2 , python3-toml, python3-packaging, python3-publicsuffix2 , python3-ldap, python3-zeroconf (>= 0.36), python3-lexicon, + , python3-pyudev , python-is-python3 , nginx, nginx-extras (>=1.18) , apt, apt-transport-https, apt-utils, aptitude, dirmngr diff --git a/share/actionsmap.yml b/share/actionsmap.yml index 70f80e4633..f23e83cf3a 100755 --- a/share/actionsmap.yml +++ b/share/actionsmap.yml @@ -2082,3 +2082,18 @@ diagnosis: help: Remove a filter (it should be an existing filter as listed with "ignore --list") nargs: "*" metavar: CRITERIA + + +############################# +# Storage # +############################# +storage: + category_help: Manage hard-drives, filesystem, pools + subcategories: + disk: + subcategory_help: Manage et get infos about hard-drives + actions: + # storage_disks_list + infos: + action_help: Gets infos about hard-drives currently attached to this system + api: GET /storage/disk/infos diff --git a/src/disks.py b/src/disks.py new file mode 100644 index 0000000000..9124a23b68 --- /dev/null +++ b/src/disks.py @@ -0,0 +1,101 @@ +from collections import OrderedDict +import dataclasses +from glob import glob +from typing import Optional + +import pyudev +import psutil + +from moulinette.utils.log import getActionLogger + + +from yunohost.utils.disks import filter_device + + +logger = getActionLogger("yunohost.storage") + + +@dataclasses.dataclass +class DiskParts: + devname: str + filesystem: str + encrypted: bool + mountpoint: str + + @staticmethod + def from_parent_device(device: pyudev.Device, partitions): + result = OrderedDict() + for child_dev in sorted( + filter(filter_device, device.children), key=lambda it: it.device_node + ): + encrypted_provider = glob(f"/sys/block/dm-*/slaves/{child_dev.sys_name}") + if encrypted_provider: + # retrive the dm-x part + dm = encrypted_provider[0].split("/")[3] + enc_dev = pyudev.Devices.from_name(device.context, "block", dm) + # This work for LUKS, what about other partition mecanisms? + partname = f"/dev/mapper/{enc_dev.properties['DM_NAME']}" + encrypted = True + else: + partname = child_dev.device_node + encrypted = False + + if partname not in partitions: + logger.warning( + f"{child_dev.device_node} not found by 'psutil.disk_partitions'" + ) + continue + + result[child_dev.sys_name] = DiskParts( + devname=device.device_node, + filesystem=partitions[partname].fstype, + encrypted=encrypted, + mountpoint=partitions[partname].mountpoint, + ) + + return result + + +@dataclasses.dataclass +class DiskInfos: + devname: str + model: str + serial: str + size: int + links: list[str] + partitions: Optional[list[DiskParts]] + + @staticmethod + def from_device(device, partitions): + try: + dev_size = device.attributes.asint("size") + except (AttributeError, UnicodeError, ValueError): + dev_size = None + + dev_links = list(sorted(it for it in device.device_links)) + child_parts = DiskParts.from_parent_device(device, partitions) + + return DiskInfos( + devname=device.device_node, + model=device.get("ID_MODEL", None), + serial=device.get("ID_SERIAL_SHORT", None), + size=dev_size, + links=dev_links, + partitions=child_parts or None, + ) + + +def infos(): + context = pyudev.Context() + partitions = {it.device: it for it in psutil.disk_partitions()} + result = OrderedDict() + + for it in sorted( + filter(filter_device, context.list_devices(subsystem="block", DEVTYPE="disk")), + key=lambda it: it.device_node, + ): + result[it.sys_name] = dataclasses.asdict( + DiskInfos.from_device(it, partitions), dict_factory=OrderedDict + ) + + return result diff --git a/src/storage.py b/src/storage.py new file mode 100644 index 0000000000..befd4d998f --- /dev/null +++ b/src/storage.py @@ -0,0 +1,4 @@ +def storage_disk_infos(): + from yunohost.disks import infos + + return infos() \ No newline at end of file diff --git a/src/utils/disks.py b/src/utils/disks.py new file mode 100644 index 0000000000..b2abacdeda --- /dev/null +++ b/src/utils/disks.py @@ -0,0 +1,12 @@ +import re + +IGNORE_DISKS = "sr", "md", "dm-", "loop", "zd", "pmem" +# regex: ^((sr)|(md)|...) +IGNORE_DISK_RE = re.compile(rf"""^({"|".join([f'({it})' for it in IGNORE_DISKS])})""") + + +def filter_device(device): + """ + Returns True if device has parents (e.g. USB device) and its name is not amongst + """ + return device.parent is not None and not IGNORE_DISK_RE.match(device.sys_name) From e83e7b36708f1f0c3f39a9f8a9cc53995232f872 Mon Sep 17 00:00:00 2001 From: Christophe Henry Date: Sun, 22 Sep 2024 16:59:40 +0200 Subject: [PATCH 2/2] Rewrite 'yunohost storage disk infos' using udisks --- debian/control | 2 +- src/disks.py | 166 +++++++++++++++++++++++++----------------- src/storage.py | 2 +- src/utils/__init__.py | 3 + src/utils/disks.py | 12 --- 5 files changed, 105 insertions(+), 80 deletions(-) delete mode 100644 src/utils/disks.py diff --git a/debian/control b/debian/control index dc95129334..a72c14d34f 100644 --- a/debian/control +++ b/debian/control @@ -15,7 +15,7 @@ Depends: ${python3:Depends}, ${misc:Depends} , python3-miniupnpc, python3-dbus, python3-jinja2 , python3-toml, python3-packaging, python3-publicsuffix2 , python3-ldap, python3-zeroconf (>= 0.36), python3-lexicon, - , python3-pyudev + , udisks2, , python-is-python3 , nginx, nginx-extras (>=1.18) , apt, apt-transport-https, apt-utils, aptitude, dirmngr diff --git a/src/disks.py b/src/disks.py index 9124a23b68..96766d5cea 100644 --- a/src/disks.py +++ b/src/disks.py @@ -1,18 +1,25 @@ +import operator from collections import OrderedDict import dataclasses from glob import glob -from typing import Optional +from typing import Optional, Any -import pyudev -import psutil +import dbus from moulinette.utils.log import getActionLogger +from yunohost.utils import bytearray_to_string -from yunohost.utils.disks import filter_device +logger = getActionLogger("yunohost.storage") -logger = getActionLogger("yunohost.storage") +UDISK_DRIVE_PATH = "/org/freedesktop/UDisks2/drives/" +UDISK_BLOCK_PATH = "/org/freedesktop/UDisks2/block_devices/" +UDISK_PART_TABLE_IFC = "org.freedesktop.UDisks2.PartitionTable" +UDISK_BLOCK_IFC = "org.freedesktop.UDisks2.Block" +UDISK_DRIVE_IFC = "org.freedesktop.UDisks2.Drive" +UDISK_ENCRYPTED_IFC = "org.freedesktop.UDisks2.Encrypted" +UDISK_FILESYSTEM_IFC = "org.freedesktop.UDisks2.Filesystem" @dataclasses.dataclass @@ -22,39 +29,6 @@ class DiskParts: encrypted: bool mountpoint: str - @staticmethod - def from_parent_device(device: pyudev.Device, partitions): - result = OrderedDict() - for child_dev in sorted( - filter(filter_device, device.children), key=lambda it: it.device_node - ): - encrypted_provider = glob(f"/sys/block/dm-*/slaves/{child_dev.sys_name}") - if encrypted_provider: - # retrive the dm-x part - dm = encrypted_provider[0].split("/")[3] - enc_dev = pyudev.Devices.from_name(device.context, "block", dm) - # This work for LUKS, what about other partition mecanisms? - partname = f"/dev/mapper/{enc_dev.properties['DM_NAME']}" - encrypted = True - else: - partname = child_dev.device_node - encrypted = False - - if partname not in partitions: - logger.warning( - f"{child_dev.device_node} not found by 'psutil.disk_partitions'" - ) - continue - - result[child_dev.sys_name] = DiskParts( - devname=device.device_node, - filesystem=partitions[partname].fstype, - encrypted=encrypted, - mountpoint=partitions[partname].mountpoint, - ) - - return result - @dataclasses.dataclass class DiskInfos: @@ -63,39 +37,99 @@ class DiskInfos: serial: str size: int links: list[str] - partitions: Optional[list[DiskParts]] - - @staticmethod - def from_device(device, partitions): - try: - dev_size = device.attributes.asint("size") - except (AttributeError, UnicodeError, ValueError): - dev_size = None - - dev_links = list(sorted(it for it in device.device_links)) - child_parts = DiskParts.from_parent_device(device, partitions) - - return DiskInfos( - devname=device.device_node, - model=device.get("ID_MODEL", None), - serial=device.get("ID_SERIAL_SHORT", None), - size=dev_size, - links=dev_links, - partitions=child_parts or None, - ) + partitions: Optional[dict[str, DiskParts]] def infos(): - context = pyudev.Context() - partitions = {it.device: it for it in psutil.disk_partitions()} result = OrderedDict() - for it in sorted( - filter(filter_device, context.list_devices(subsystem="block", DEVTYPE="disk")), - key=lambda it: it.device_node, - ): - result[it.sys_name] = dataclasses.asdict( - DiskInfos.from_device(it, partitions), dict_factory=OrderedDict + bus = dbus.SystemBus() + manager = bus.get_object("org.freedesktop.UDisks2", "/org/freedesktop/UDisks2") + + drives = {} + devices = {} + partitions = {} + + for k, v in manager.get_dbus_method( + "GetManagedObjects", "org.freedesktop.DBus.ObjectManager" + )().items(): + if k.startswith(UDISK_DRIVE_PATH): + # These are hard drives + drives[k.removeprefix(UDISK_DRIVE_PATH)] = v + elif UDISK_PART_TABLE_IFC in v: + # These are block container partition tables (/dev/sda, /dev/sdb, etc.) + devices[k.removeprefix(UDISK_BLOCK_PATH)] = v + elif UDISK_BLOCK_IFC in v: + # These are partitions (/dev/sda1, /dev/dm-1, etc.). Here, we try to + # associate partitions with as much keys as possible to easier search + # These will be, for instance sdb1 and /dev/sdb1, dm-1 and /dev/dm-1, etc. + _dev = bytearray_to_string(v[UDISK_BLOCK_IFC]["Device"]) + _pref_dev = bytearray_to_string(v[UDISK_BLOCK_IFC]["PreferredDevice"]) + partitions[_dev] = partitions[_dev.split("/")[-1]] = v + partitions[_pref_dev] = partitions[_pref_dev.split("/")[-1]] = v + partitions[k.removeprefix(UDISK_BLOCK_PATH)] = v + + for key, device in sorted(devices.items(), key=operator.itemgetter(0)): + drive = drives[device[UDISK_BLOCK_IFC]["Drive"].removeprefix(UDISK_DRIVE_PATH)][ + UDISK_DRIVE_IFC + ] + devname = bytearray_to_string(device[UDISK_BLOCK_IFC]["Device"]) + + device_partitions = OrderedDict() + + for partition_key in map( + lambda p: p.removeprefix(UDISK_BLOCK_PATH), + sorted(device[UDISK_PART_TABLE_IFC]["Partitions"]), + ): + partition_obj = partitions[partition_key] + partition_devname = bytearray_to_string( + partition_obj[UDISK_BLOCK_IFC]["Device"] + ) + encrypted = False + + if UDISK_ENCRYPTED_IFC in partition_obj: + encrypted = True + partition_obj = partitions[ + partition_obj[UDISK_ENCRYPTED_IFC]["CleartextDevice"].removeprefix( + UDISK_BLOCK_PATH + ) + ] + else: + # If partition is a device mapper, it's not easy to associate the + # virtual device with its underlying FS. If we can find an actual + # partition (i.e. sda5) in /sys/block/dm-*/slaves/, we can then + # search the FS using the corresponding dm-X in the partitions dict. + mapper = glob(f"/sys/block/dm-*/slaves/{partition_key}") + if mapper and (mapper_key := mapper[0].split("/")[3]) in partitions: + partition_obj = partitions[mapper_key] + + block = partition_obj[UDISK_BLOCK_IFC] + + if UDISK_FILESYSTEM_IFC in partition_obj: + device_partitions[partition_key] = DiskParts( + devname=partition_devname, + filesystem=block["IdType"], + encrypted=encrypted, + mountpoint=bytearray_to_string( + partition_obj[UDISK_FILESYSTEM_IFC]["MountPoints"][0] + ), + ) + + result[key] = dataclasses.asdict( + DiskInfos( + devname=devname, + model=drive["Model"], + serial=drive["Serial"], + size=drive["Size"], + links=list( + sorted( + bytearray_to_string(it) + for it in device[UDISK_BLOCK_IFC]["Symlinks"] + ) + ), + partitions=device_partitions or None, + ), + dict_factory=OrderedDict, ) return result diff --git a/src/storage.py b/src/storage.py index befd4d998f..17091bfa7d 100644 --- a/src/storage.py +++ b/src/storage.py @@ -1,4 +1,4 @@ def storage_disk_infos(): from yunohost.disks import infos - return infos() \ No newline at end of file + return infos() diff --git a/src/utils/__init__.py b/src/utils/__init__.py index 86d229abb0..6b7f8475ac 100644 --- a/src/utils/__init__.py +++ b/src/utils/__init__.py @@ -16,3 +16,6 @@ # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . # + +def bytearray_to_string(ba): + return bytearray(ba).decode("utf-8").removesuffix("\x00") diff --git a/src/utils/disks.py b/src/utils/disks.py deleted file mode 100644 index b2abacdeda..0000000000 --- a/src/utils/disks.py +++ /dev/null @@ -1,12 +0,0 @@ -import re - -IGNORE_DISKS = "sr", "md", "dm-", "loop", "zd", "pmem" -# regex: ^((sr)|(md)|...) -IGNORE_DISK_RE = re.compile(rf"""^({"|".join([f'({it})' for it in IGNORE_DISKS])})""") - - -def filter_device(device): - """ - Returns True if device has parents (e.g. USB device) and its name is not amongst - """ - return device.parent is not None and not IGNORE_DISK_RE.match(device.sys_name)