Merge pull request #55 from bmwcarit/19-graphics

19 graphics

.gitignore

@@ -12,6 +12,10 @@ doc/test_project/results
 pypiemma.egg-info/
 site/
 
+# Exclude all .svg except in the doc/image folder
+*.svg
+!doc/images/*
+
 
 # Exclude all but .png files for UML/call graphs
 doc/images/call_graph_uml/*

.travis.yml

@@ -47,7 +47,7 @@ install:
     - pip3 install -U pyenchant         # needed for pylint spell checking
     - pip3 install -U coverage
     # Packages for Emma
-    - pip3 install -U Pygments Markdown matplotlib pandas "pypiscout>=2.0" graphviz
+    - pip3 install -U Pygments Markdown matplotlib pandas "pypiscout>=2.0" graphviz svgwrite
     # Packages for Emma reports + html doc
     - sudo apt-get update
     - sudo apt-get install graphviz

Emma/__init__.py

@@ -23,10 +23,9 @@ class SUBPARSER_STRINGS:
     DELTAS: str = "d"
 
 
-VERSION_MAJOR = "3"
-VERSION_MINOR = "4"
-VERSION_PATCH = "4"
-
+VERSION_MAJOR = "4"
+VERSION_MINOR = "0"
+VERSION_PATCH = "0"
 EMMA_VERSION = ".".join([VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH])
 EMMA_VISUALISER_VERSION = EMMA_VERSION
 EMMA_DELTAS_VERSION = EMMA_VERSION

Emma/emma.py

@@ -51,7 +51,7 @@ def main(arguments):
     if memoryManager.settings.createCategories or memoryManager.settings.dryRun:
         sc().info("No results were generated since categorisation or dryRun option is active.")
     else:
-        memoryManager.createReports(arguments.teamscale)
+        memoryManager.createReports(arguments.teamscale, arguments.memVis, arguments.memVisResolved, arguments.noprompt)
 
     # Stop and display time measurement
     TIME_END = timeit.default_timer()
@@ -141,14 +141,25 @@ def initParser():
         action="store_true",
         default=False
     )
+    parser.add_argument(
+        "--memVis",
+        help="Plot unresolved view of sections and objects for a specified address area",
+        default=False,
+        action="store_true"
+    )
+    parser.add_argument(
+        "--memVisResolved",
+        help="Plot figure visualising how Emma resolved the overlaps for a specified address area. Not possible if noResolveOverlap is active",
+        default=False,
+        action="store_true"
+    )
 
     parser.add_argument(
         "--teamscale",
         help="Create team scale reports",
         default=False,
         action="store_true",
     )
-
     parser.add_argument(
         "--dryRun",
         help="Do not store any standard reports",
@@ -171,7 +182,7 @@ def parseArgs(arguments=""):
 
 def processArguments(arguments):
     """
-    Function to extract the settings values from the command line arguments.
+    Extract the settings values from the command line arguments.
     :param arguments: The command line arguments, that is the result of the parser.parse_args().
     :return: The setting values.
     """
@@ -188,6 +199,13 @@ def processArguments(arguments):
     # Get paths straight (only forward slashes) or set it to empty if it was empty
     subDir = Emma.shared_libs.emma_helper.joinPath(arguments.subdir) if arguments.subdir is not None else ""
 
+    if arguments.memVis and arguments.memVisResolved:
+        sc().error("Select either `--memVis` or `--memVisResolved`")
+    if arguments.memVisResolved and arguments.noResolveOverlap:
+        sc().warning("Incompatible arguments `--noResolveOverlap` and `--memVisResolved` were found. SVG figure will depict the unresolved scenario.")
+        arguments.memVisResolved = False
+        arguments.memVis = True
+
     outputPath = Emma.shared_libs.emma_helper.joinPath(directory, subDir, OUTPUT_DIR)
     analyseDebug = arguments.analyseDebug
     createCategories = arguments.createCategories
@@ -196,9 +214,12 @@ def processArguments(arguments):
     noResolveOverlap = arguments.noResolveOverlap
     teamscale = arguments.teamscale
     dryRun = arguments.dryRun
+    memVis = arguments.memVis
+    memVisResolved = arguments.memVisResolved
+
     # TODO: It would be more convenient if arguments which are not modified are passed without manually modifying the code (MSc)
 
-    return projectName, configurationPath, mapfilesPath, outputPath, analyseDebug, createCategories, removeUnmatched, noPrompt, noResolveOverlap, teamscale, dryRun
+    return projectName, configurationPath, mapfilesPath, outputPath, analyseDebug, createCategories, removeUnmatched, noPrompt, noResolveOverlap, teamscale, dryRun, memVis, memVisResolved
 
 
 def runEmma():

Emma/emma_libs/memoryMap.py

@@ -16,7 +16,6 @@
 along with this program.  If not, see <https://www.gnu.org/licenses/>
 """
 
-
 import csv
 import bisect
 import copy
@@ -356,6 +355,6 @@ def writeReportToDisk(reportPath, consumerCollection):
                 # FQN
                 row.getFQN()
             ])
-
             # Writing the data to the file
+
             writer.writerow(rowData)

Emma/emma_vis_libs/dataReports.py

@@ -27,6 +27,7 @@
 
 import pandas
 import matplotlib.pyplot
+from Emma import shared_libs
 
 from Emma.shared_libs.stringConstants import *                           # pylint: disable=unused-wildcard-import,wildcard-import
 import Emma.shared_libs.emma_helper

Emma/emma_vis_libs/helper.py

@@ -71,5 +71,5 @@ def getLastModFileOrPrompt(subStringIdentifier: str, inOutPath: str, quiet: bool
                 pypiscout.SCout.warning("Invalid input.")
 
     if fileToUse is None:
-        sc().error("No file containing '" + subStringIdentifier + "' found in " + path)
+        sc().error(f"No file containing  `{subStringIdentifier}` found in {path}")
     return fileToUse

README.md

@@ -166,6 +166,7 @@ emma-dev (. at) googlegroups.com
 | Pygments (v2.3.1+)   | [Pygments](https://pypi.org/project/Pygments/)      | BSD-2-Clause                         | [https://bitbucket.org/birkenfeld/pygments-main/src/default/](https://bitbucket.org/birkenfeld/pygments-main/src/default/); [http://pygments.org/download/](http://pygments.org/download/)   |
 | Matplotlib (v3.0.0+) | [matplotlib](https://pypi.org/project/matplotlib/)  | Matplotlib License (BSD compatible)  | [https://matplotlib.org/users/installing.html](https://matplotlib.org/users/installing.html); [https://github.com/matplotlib/matplotlib](https://github.com/matplotlib/matplotlib)           |
 | SCout (v2.0+)        | [pypiscout](https://pypi.org/project/pypiscout/)    | MIT                                  | [https://github.com/holzkohlengrill/SCout](https://github.com/holzkohlengrill/SCout)                                                                                                         |
+| svgwrite (v1.4+)    | [svgwrite](https://pypi.org/project/svgwrite/)          | MIT License (MIT License)           | [https://github.com/mozman/svgwrite](https://github.com/mozman/svgwrite); [https://svgwrite.readthedocs.io/en/latest/](https://svgwrite.readthedocs.io/en/latest/)                           |
 
 
 **Optional dependencies:**
@@ -188,8 +189,8 @@ Utility scripts used to build GitHub pages documentation. As a normal user you c
 
 | Library (version)             | pip package name                                                  | Licence                              | URL                                                                                                                                                |
 |-------------------------------|-------------------------------------------------------------------|--------------------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------|
-| MkDocs (v1.1.2+)              | [mkdocs](https://pypi.org/project/mkdocs/)                        | BSD-3Clause                          | [https://github.com/mkdocs/mkdocs](https://github.com/mkdocs/mkdocs)                                                                               |
-| Material for MkDocs (v5.2.1+) | [mkdocs-material](https://pypi.org/project/mkdocs-material/)      | MIT                                  | [https://github.com/squidfunk/mkdocs-material](https://github.com/squidfunk/mkdocs-material)                                                       |
+| MkDocs (v1.0.4+)              | [mkdocs](https://pypi.org/project/mkdocs/)                        | BSD-3Clause                          | [https://github.com/mkdocs/mkdocs](https://github.com/mkdocs/mkdocs)                                                                               |
+| Material for MkDocs (v4.4.1+) | [mkdocs-material](https://pypi.org/project/mkdocs-material/)      | MIT                                  | [https://github.com/squidfunk/mkdocs-material](https://github.com/squidfunk/mkdocs-material)                                                       |
 
 
 

doc/readme-emma.md

@@ -44,20 +44,33 @@ The devices must have a single linear physical address space:
 
     --mapfiles MAPFILES, --map MAPFILES
 
-### Optional Arguments
+### Some Optional Arguments
+This section will provide a more in-depth description about selected command line arguments when a short description (like in `--help`) might be to short, the behaviour is too complex or background knowledge might assist you to understand the whole picture. For the full list execute Emma with `--help`.
 
 * `--dir`
     * User defined path for the top folder holding the `memStats`/output files. Per default it uses the same directory as the configuration files.
 * `--stats_dir`
-  * User defined path inside the folder given in the `--dir` argument. This is usefull when batch analysing mapfiles from various development stages. Every analysis output gets it's own directory.
+    * User defined path inside the folder given in the `--dir` argument. This is usefull when batch analysing mapfiles from various development stages. Every analysis output gets it's own directory.
 * `--create_categories`
-  * Create `categories*.json` from `categories*Keywords.json` for easier categorisation.
+    * Create `categories*.json` from `categories*Keywords.json` for easier categorisation.
 * `--remove_unmatched`,
-  * Remove unmatched entries from `categories*.json`. This is useful when a `categories*.json` from another project is used.
+    * Remove unmatched entries from `categories*.json`. This is useful when a `categories*.json` from another project is used.
 * `--analyse_debug`, `--dbg`
-  * Normally we remove DWARF debug sections from the analysis to show the relevant information for a possible release software. This can be prevented if this argument is set. DWARF section names are defined in `stringConstants.py`. `.unused_ram` is always excluded (regardless of this flag)
+    * Normally we remove DWARF debug sections from the analysis to show the relevant information for a possible release software. This can be prevented if this argument is set. DWARF section names are defined in `stringConstants.py`. `.unused_ram` is always excluded (regardless of this flag)
 * `--noprompt`
-  * Exit and fail on user prompt. Normally this happens when some files or configurations are ambiguous. This is useful when running Emma on CI systems.
+    * Exit and fail on user prompt. Normally this happens when some files or configurations are ambiguous. This is useful when running Emma on CI systems.
+* `--memVis`
+    * This is a visualisation based on data you actually see in the map files (i.e. the data *before* the containment/duplicate/overlap resolution)
+    * Prompts for a start and end address (and x/y scaling) for which memory region a visualisation should be created (as `.svg`)
+    * This visualisation allows to better see complex overlaps/alignments of objects/sections (e.g. check your linker configuration, ...)
+    * Note that huge address ranges containing many objects/sections may cause your viewer to get slow/unresponsive due to the high amount of objects/sections; it is recommended to keep your viewing area small
+    * For huge `.svg`s the authors made good experiences with Inkscape and Google Chrome
+    * Usually you detect an interesting scenario in the `.csv` reports. It might be hard to see what is actually happening (e.g. many overlaps/containments, ...). That is where a visualisation is helpful
+    * If `--noPrompt` is active you will get a weak warning that no `.svg` reports will be generated
+        <div align="left"> <img src="./images/memVis.png" width="50%"> </div>
+* `--memVisResolved`
+    * Basically the same as `--memVis` but plots the *resolved* view (i.e. after Emma resolved the containment/duplicate overlap -> basically you will see what stands in `Objects_in_Sections`)
+    * Skipped if `--noResolveOverlap` is active
 
 
 ## Project Configuration

setup.py

@@ -63,7 +63,8 @@
                       "matplotlib",
                       "pandas",
                       "pypiscout>=2.0",
-                      "graphviz"
+                      "graphviz",
+                      "svgwrite"
                       ],
     extras_require={"dev":                                      # Install dev version via `pip3 install pypiemma[dev]`
                         ["gprof2dot",