ewm7213: Independent Parsing and Child Disk Monitoring (#5)

* Parse individually

* children process disk usage

* Add egg and version to gitignore

---------

Co-authored-by: Pete Peterson <[email protected]>

.gitignore

@@ -0,0 +1,2 @@
+src/mantidprofiler.egg-info/
+src/mantidprofiler/_version.py

README.md

@@ -10,7 +10,7 @@ It monitors CPU and RAM usage, and reports the time span of each algorithm (curr
 
 To profile the `SNSPowderReduction.py` workflow:
 ```
-python SNSPowderReduction.py & python path/to/mantid-profiler/mantid-profiler.py $!
+python SNSPowderReduction.py & python path/to/mantid-profiler/src/mantidprofiler/mantidprofiler.py $!
 ```
 The script attaches to the last spawned process, so you can also use the profiler if you are working with `MantidPlot`:
 ```

src/mantidprofiler/algorithm_tree.py

@@ -134,8 +134,14 @@ def apply_multiple_trees(trees, check, func):
 
 
 def parseLine(line):
-    res = re.search("ThreadID=([0-9]*), AlgorithmName=(.*), StartTime=([0-9]*), EndTime=([0-9]*)", line)
-    return {"thread_id": res.group(1), "name": res.group(2), "start": int(res.group(3)), "finish": int(res.group(4))}
+    res = {
+        "thread_id": re.search("ThreadID=([0-9]*)", line)[1],
+        "name": re.search("AlgorithmName=([a-zA-Z]*)", line)[1],
+        "start": int(re.search("StartTime=([0-9]*)", line)[1]),
+        "finish": int(re.search("EndTime=([0-9]*)", line)[1]),
+    }
+
+    return res
 
 
 def fromFile(fileName: Path, cleanup: bool = True):

src/mantidprofiler/children_util.py

@@ -0,0 +1,33 @@
+# children_util.py - helper functions for dealing with child processes
+#
+######################################################################
+
+
+import psutil
+
+
+def all_children(pr: psutil.Process) -> list[psutil.Process]:
+    try:
+        return pr.children(recursive=True)
+    except Exception:  # noqa: BLE001
+        return []
+
+
+def update_children(old_children: dict[int, psutil.Process], new_children: list[psutil.Process]):
+    new_dct = {}
+    for ch in new_children:
+        new_dct.update({ch.pid: ch})
+
+    todel = []
+    for pid in old_children.keys():
+        if pid not in new_dct.keys():
+            todel.append(pid)
+
+    for pid in todel:
+        del old_children[pid]
+
+    updct = {}
+    for pid in new_dct.keys():
+        if pid not in old_children.keys():
+            updct.update({pid: new_dct[pid]})
+    old_children.update(updct)

src/mantidprofiler/diskrecord.py

@@ -4,6 +4,7 @@
 
 import numpy as np
 import psutil
+from children_util import all_children
 from time_util import get_current_time, get_start_time
 
 
@@ -27,6 +28,10 @@ def monitor(pid: int, logfile: Path, interval: Optional[float], show_bytes: bool
 
     disk_before = process.io_counters()
 
+    children_before = {}
+    for ch in all_children(process):
+        children_before.update({ch.pid: {"process": ch, "disk": ch.io_counters()}})
+
     with open(logfile, "w") as handle:
         # add header
         handle.write(
@@ -72,6 +77,31 @@ def monitor(pid: int, logfile: Path, interval: Optional[float], show_bytes: bool
                         conversion_to_size * (disk_after.write_bytes - disk_before.write_bytes) / delta_time
                     )
 
+                    # get information from children
+                    children_after = {}
+                    for ch in all_children(process):
+                        # format the children dict
+                        children_after.update({ch.pid: {"process": ch, "disk": ch.io_counters()}})
+
+                        # initialize change with new child
+                        read_char_diff = children_after[ch.pid]["disk"].read_chars
+                        write_char_diff = children_after[ch.pid]["disk"].write_chars
+                        read_byte_diff = children_after[ch.pid]["disk"].read_bytes
+                        write_byte_diff = children_after[ch.pid]["disk"].write_bytes
+
+                        # subtract change from last iteration, if child already existed
+                        if ch.pid in children_before.keys():
+                            read_char_diff -= children_before[ch.pid]["disk"].read_chars
+                            write_char_diff -= children_before[ch.pid]["disk"].write_chars
+                            read_byte_diff -= children_before[ch.pid]["disk"].read_bytes
+                            write_byte_diff -= children_before[ch.pid]["disk"].write_bytes
+
+                        # add to totals
+                        read_char_per_sec += conversion_to_size * read_char_diff / delta_time
+                        write_char_per_sec += conversion_to_size * write_char_diff / delta_time
+                        read_byte_per_sec += conversion_to_size * read_byte_diff / delta_time
+                        write_byte_per_sec += conversion_to_size * write_byte_diff / delta_time
+
                     # write information to the log file
                     handle.write(
                         "{0:12.6f} {1:12.3f} {2:12.3f} {3:12.3f} {4}\n".format(
@@ -85,6 +115,7 @@ def monitor(pid: int, logfile: Path, interval: Optional[float], show_bytes: bool
 
                     # copy over information to new previous
                     disk_before = disk_after
+                    children_before = children_after
                     last_time = current_time
                 except (psutil.NoSuchProcess, psutil.AccessDenied):
                     break  # all done

src/mantidprofiler/psrecord.py

@@ -37,6 +37,7 @@
 
 import numpy as np
 import psutil
+from children_util import all_children, update_children
 from time_util import get_current_time, get_start_time
 
 
@@ -53,33 +54,6 @@ def get_threads(process):
     return process.threads()
 
 
-def all_children(pr: psutil.Process) -> list[psutil.Process]:
-    try:
-        return pr.children(recursive=True)
-    except Exception:  # noqa: BLE001
-        return []
-
-
-def update_children(old_children: dict[int, psutil.Process], new_children: list[psutil.Process]):
-    new_dct = {}
-    for ch in new_children:
-        new_dct.update({ch.pid: ch})
-
-    todel = []
-    for pid in old_children.keys():
-        if pid not in new_dct.keys():
-            todel.append(pid)
-
-    for pid in todel:
-        del old_children[pid]
-
-    updct = {}
-    for pid in new_dct.keys():
-        if pid not in old_children.keys():
-            updct.update({pid: new_dct[pid]})
-    old_children.update(updct)
-
-
 def monitor(pid: int, logfile: Path, interval: Optional[float]) -> None:
     # change None to reasonable default
     if interval is None: