From 635f341260f08f40178825b0337dba31ff539787 Mon Sep 17 00:00:00 2001 From: Fabrice Normandin Date: Tue, 7 Nov 2023 14:27:23 -0500 Subject: [PATCH 1/3] Fix bug with outdated `disk-quota` output parsing Signed-off-by: Fabrice Normandin --- milatools/cli/commands.py | 65 ++++++++++++++++++++++++--------------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/milatools/cli/commands.py b/milatools/cli/commands.py index 8c350aff..ef7ab5eb 100644 --- a/milatools/cli/commands.py +++ b/milatools/cli/commands.py @@ -1034,32 +1034,49 @@ def _get_disk_quota_usage( Returns whether the quota is exceeded, in terms of storage space or number of files. Here is what the output of `disk-quota` looks like on the Mila cluster: - ```console - - Quota information for storage pool Default (ID: 1): + The mila cluster uses a lustre filesystem for $HOME. Here is the command we use: + ```bash + #!/bin/sh + # (disk-quota) + echo "==== HOME ====" + lfs quota -h -u "${1:-$USER}" /home/mila + + echo "" + echo "==== SCRATCH ====" + beegfs-ctl --cfgFile=/etc/beegfs/scratch.d/beegfs-client.conf --getquota --uid "${1:-$USER}" + ``` - user/group || size || chunk files - name | id || used | hard || used | hard - --------------|------||------------|------------||---------|--------- - normandf|1471600598|| 97.20 GiB| 100.00 GiB|| 806898| 1000000 + ```bash + $ lfs quota -h -u "${1:-$USER}" /home/mila + Disk quotas for usr normandf (uid 1471600598): + Filesystem used quota limit grace files quota limit grace + /home/mila 96.91G 0k 100G - 944570 0 1048576 - + uid 1471600598 is using default block quota setting + uid 1471600598 is using default file quota setting ``` """ - disk_quota_output = remote.get_output("disk-quota", hide=not print_command_output) - last_line_parts = disk_quota_output.splitlines()[-1] + home_disk_quota_output = remote.get_output( + "lfs quota -u $USER /home/mila", hide=not print_command_output + ) + lines = home_disk_quota_output.splitlines() ( - _username, - _id, - _, - used_gb, - max_gb, - _, - used_files, - max_files, - ) = last_line_parts.split("|") - used_gb = float(used_gb.replace("GiB", "").strip()) - max_gb = float(max_gb.replace("GiB", "").strip()) - used_files = int(used_files.strip()) - max_files = int(max_files.strip()) + filesystem, + used_kbytes, + _quota1, + limit_kbytes, + _grace1, + files, + _quota2, + limit_files, + _grace2, + ) = ( + lines[2].strip().split() + ) + + used_gb = float(int(used_kbytes.strip()) / (1024) ** 2) + max_gb = float(int(limit_kbytes.strip()) / (1024) ** 2) + used_files = int(files.strip()) + max_files = int(limit_files.strip()) return (used_gb, max_gb), (used_files, max_files) @@ -1072,12 +1089,12 @@ def check_disk_quota(remote: Remote) -> None: logger.debug("Checking disk quota on $HOME...") (used_gb, max_gb), (used_files, max_files) = _get_disk_quota_usage(remote) logger.debug( - f"Disk usage: {used_gb} / {max_gb} GiB and {used_files} / {max_files} files" + f"Disk usage: {used_gb:.1f} / {max_gb} GiB and {used_files} / {max_files} files" ) size_ratio = used_gb / max_gb files_ratio = used_files / max_files reason = ( - f"{used_gb} / {max_gb} GiB" + f"{used_gb:.1f} / {max_gb} GiB" if size_ratio > files_ratio else f"{used_files} / {max_files} files" ) From caff6cfba8bb3345114ef17bdf3950f7b42fbfcd Mon Sep 17 00:00:00 2001 From: Fabrice Normandin Date: Wed, 8 Nov 2023 12:19:24 -0500 Subject: [PATCH 2/3] Wrap the check_disk_quota in a try/except block Signed-off-by: Fabrice Normandin --- milatools/cli/commands.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/milatools/cli/commands.py b/milatools/cli/commands.py index ef7ab5eb..1d071b2a 100644 --- a/milatools/cli/commands.py +++ b/milatools/cli/commands.py @@ -542,7 +542,12 @@ def code( if command is None: command = os.environ.get("MILATOOLS_CODE_COMMAND", "code") - check_disk_quota(remote) + try: + check_disk_quota(remote) + except MilatoolsUserError: + raise + except Exception as exc: + logger.warning(f"Unable to check the disk-quota on the cluster: {exc}") cnode = _find_allocation( remote, job_name="mila-code", job=job, node=node, alloc=alloc From e3d7a90107f6a2e5f53741cc9425e354af5bbeed Mon Sep 17 00:00:00 2001 From: Fabrice Normandin Date: Wed, 8 Nov 2023 12:19:59 -0500 Subject: [PATCH 3/3] Remove leftover comment blocks in fn __doc__ Signed-off-by: Fabrice Normandin --- milatools/cli/commands.py | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/milatools/cli/commands.py b/milatools/cli/commands.py index 1d071b2a..89c70259 100644 --- a/milatools/cli/commands.py +++ b/milatools/cli/commands.py @@ -1037,35 +1037,23 @@ def _get_disk_quota_usage( """Checks the disk quota on the $HOME filesystem on the mila cluster. Returns whether the quota is exceeded, in terms of storage space or number of files. - - Here is what the output of `disk-quota` looks like on the Mila cluster: - The mila cluster uses a lustre filesystem for $HOME. Here is the command we use: - ```bash - #!/bin/sh - # (disk-quota) - echo "==== HOME ====" - lfs quota -h -u "${1:-$USER}" /home/mila - - echo "" - echo "==== SCRATCH ====" - beegfs-ctl --cfgFile=/etc/beegfs/scratch.d/beegfs-client.conf --getquota --uid "${1:-$USER}" - ``` - - ```bash - $ lfs quota -h -u "${1:-$USER}" /home/mila - Disk quotas for usr normandf (uid 1471600598): - Filesystem used quota limit grace files quota limit grace - /home/mila 96.91G 0k 100G - 944570 0 1048576 - - uid 1471600598 is using default block quota setting - uid 1471600598 is using default file quota setting - ``` """ + + # NOTE: This is what the output of the command looks like on the Mila cluster: + # + # $ lfs quota -u $USER /home/mila + # Disk quotas for usr normandf (uid 1471600598): + # Filesystem kbytes quota limit grace files quota limit grace + # /home/mila 101440844 0 104857600 - 936140 0 1048576 - + # uid 1471600598 is using default block quota setting + # uid 1471600598 is using default file quota setting + # home_disk_quota_output = remote.get_output( "lfs quota -u $USER /home/mila", hide=not print_command_output ) lines = home_disk_quota_output.splitlines() ( - filesystem, + _filesystem, used_kbytes, _quota1, limit_kbytes,