Skip to content

Commit

Permalink
Support GPU memory monitoring
Browse files Browse the repository at this point in the history
  • Loading branch information
marxin committed Feb 22, 2024
1 parent 3be859d commit 77ceaaa
Showing 1 changed file with 7 additions and 0 deletions.
7 changes: 7 additions & 0 deletions usage-wrapper.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,9 +155,14 @@ def difference_in_gb(self):
try:
import GPUtil
gpu_stats = DataStatistic(lambda: 100 * GPUtil.getGPUs()[0].load)
# the memory consumption is reported in MiBs
gpu_mem_stats = DataStatistic(lambda: GPUtil.getGPUs()[0].memoryUsed / 1024)
collectors.append(gpu_stats)
collectors.append(gpu_mem_stats)
except ImportError:
gpu_stats = None
gpu_mem_stats = None
print('WARNING: missing GPUtil package (pip install GPUtil)')


def get_process_name(proc):
Expand Down Expand Up @@ -311,6 +316,8 @@ def generate_graph():
mem_subplot.plot(timestamps, mem_stats.values, c='blue', lw=LW, label='total')
mem_subplot.set_title('Memory usage')
mem_subplot.set_ylabel('GiB')
if gpu_stats:
mem_subplot.plot(timestamps, gpu_mem_stats.values, c='fuchsia', lw=LW, label='GPU')

disk_subplot.plot(timestamps, disk_read_stats.values, c='green', lw=LW, label='read')
disk_subplot.plot(timestamps, disk_write_stats.values, c='red', lw=LW, label='write')
Expand Down

0 comments on commit 77ceaaa

Please sign in to comment.