Merge pull request #1306 from volatilityfoundation/release/v2.8.0

Release/v2.8.0

.github/workflows/black.yml

@@ -6,8 +6,10 @@ jobs:
   lint:
     runs-on: ubuntu-20.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: psf/black@stable
         with:
           options: "--check --diff --verbose"
           src: "./volatility3"
+          # FIXME: Remove when Volatility3 minimum Python version is >3.8
+          version: "24.8.0"

.github/workflows/build-pypi.yml

@@ -20,24 +20,23 @@ jobs:
       matrix:
         python-version: ["3.7"]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel
+        pip install build
 
     - name: Build PyPi packages
       run: |
-        python setup.py sdist --formats=gztar,zip
-        python setup.py bdist_wheel
+        python -m build
 
     - name: Archive dist
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: volatility3-pypi
         path: |

.github/workflows/install.yml

@@ -10,7 +10,7 @@ jobs:
         host: [ ubuntu-latest, windows-latest ]
         python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v4

.github/workflows/test.yaml

@@ -8,23 +8,22 @@ jobs:
       matrix:
         python-version: ["3.7"]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
         pip install Cmake
-        pip install setuptools wheel
+        pip install build
         pip install -r ./test/requirements-testing.txt
 
     - name: Build PyPi packages
       run: |
-        python setup.py sdist --formats=gztar,zip
-        python setup.py bdist_wheel
+        python -m build
 
     - name: Download images
       run: |

.gitignore

@@ -27,6 +27,7 @@ config*.json
 # Pyinstaller files
 build
 dist
+*.egg-info
 
 # Environments
 .env

CITATION.cff

@@ -14,7 +14,7 @@ authors:
 identifiers:
   - type: url
     value: 'https://github.com/volatilityfoundation/volatility3'
-    description: Volatility 3 source code respository
+    description: Volatility 3 source code repository
 repository-code: 'https://github.com/volatilityfoundation/volatility3'
 url: 'https://github.com/volatilityfoundation/volatility3'
 abstract: >-

README.md

@@ -20,17 +20,16 @@ more details.
 
 ## Requirements
 
-Volatility 3 requires Python 3.7.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as:
+Volatility 3 requires Python 3.7.3 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as:
 
 ```shell
 pip3 install -r requirements-minimal.txt
 ```
 
-Alternately, the minimal packages will be installed automatically when Volatility 3 is installed using setup.py. However, as noted in the Quick Start section below, Volatility 3 does not *need* to be installed via setup.py prior to using it.
+Alternately, the minimal packages will be installed automatically when Volatility 3 is installed using pip. However, as noted in the Quick Start section below, Volatility 3 does not *need* to be installed prior to using it.
 
 ```shell
-python3 setup.py build 
-python3 setup.py install
+pip3 install .
 ```
 
 To enable the full range of Volatility 3 functionality, use a command like the one below. For partial functionality, comment out any unnecessary packages in [requirements.txt](requirements.txt) prior to running the command.

pyproject.toml

@@ -0,0 +1,32 @@
+[project]
+name = "volatility3"
+description = "Memory forensics framework"
+keywords = ["volatility", "memory", "forensics", "framework", "windows", "linux", "volshell"]
+readme = "README.md"
+authors = [
+    { name = "Volatility Foundation", email = "[email protected]" },
+]
+requires-python = ">=3.7.3"
+license = { text = "VSL" }
+dynamic = ["dependencies", "optional-dependencies", "version"]
+
+[project.urls]
+Homepage = "https://github.com/volatilityfoundation/volatility3/"
+"Bug Tracker" = "https://github.com/volatilityfoundation/volatility3/issues"
+Documentation = "https://volatility3.readthedocs.io/"
+"Source Code" = "https://github.com/volatilityfoundation/volatility3"
+
+[project.scripts]
+vol = "volatility3.cli:main"
+volshell = "volatility3.cli.volshell:main"
+
+[tool.setuptools.dynamic]
+version = { attr = "volatility3.framework.constants._version.PACKAGE_VERSION" }
+dependencies = { file = "requirements-minimal.txt" }
+
+[tool.setuptools.packages.find]
+include = ["volatility3*"]
+
+[build-system]
+requires = ["setuptools>=68"]
+build-backend = "setuptools.build_meta"

requirements.txt

@@ -15,7 +15,7 @@ capstone>=3.0.5
 pycryptodome
 
 # This is required for memory acquisition via leechcore/pcileech.
-leechcorepyc>=2.4.0
+leechcorepyc>=2.4.0; sys_platform != 'darwin'
 
 # This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage
 gcsfs>=2023.1.0

setup.py

@@ -4,50 +4,21 @@
 
 import setuptools
 
-from volatility3.framework import constants
 
-with open("README.md", "r", encoding="utf-8") as fh:
-    long_description = fh.read()
-
-
-def get_install_requires():
+def get_requires(filename):
     requirements = []
-    with open("requirements-minimal.txt", "r", encoding="utf-8") as fh:
+    with open(filename, "r", encoding="utf-8") as fh:
         for line in fh.readlines():
             stripped_line = line.strip()
-            if stripped_line == "" or stripped_line.startswith("#"):
+            if stripped_line == "" or stripped_line.startswith(("#", "-r")):
                 continue
             requirements.append(stripped_line)
     return requirements
 
 
 setuptools.setup(
-    name="volatility3",
-    description="Memory forensics framework",
-    version=constants.PACKAGE_VERSION,
-    license="VSL",
-    keywords="volatility memory forensics framework windows linux volshell",
-    author="Volatility Foundation",
-    long_description=long_description,
-    long_description_content_type="text/markdown",
-    author_email="[email protected]",
-    url="https://github.com/volatilityfoundation/volatility3/",
-    project_urls={
-        "Bug Tracker": "https://github.com/volatilityfoundation/volatility3/issues",
-        "Documentation": "https://volatility3.readthedocs.io/",
-        "Source Code": "https://github.com/volatilityfoundation/volatility3",
-    },
-    packages=setuptools.find_namespace_packages(
-        include=["volatility3", "volatility3.*"]
-    ),
-    package_dir={"volatility3": "volatility3"},
-    python_requires=">=3.7.0",
-    include_package_data=True,
-    entry_points={
-        "console_scripts": [
-            "vol = volatility3.cli:main",
-            "volshell = volatility3.cli.volshell:main",
-        ],
+    extras_require={
+        "dev": get_requires("requirements-dev.txt"),
+        "full": get_requires("requirements.txt"),
     },
-    install_requires=get_install_requires(),
 )

vol.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python3
+# PYTHON_ARGCOMPLETE_OK
 
 # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0
 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0

volatility3/cli/__init__.py

@@ -22,6 +22,13 @@
 from typing import Any, Dict, List, Tuple, Type, Union
 from urllib import parse, request
 
+try:
+    import argcomplete
+
+    HAS_ARGCOMPLETE = True
+except ImportError:
+    HAS_ARGCOMPLETE = False
+
 from volatility3.cli import text_filter
 import volatility3.plugins
 import volatility3.symbols
@@ -351,6 +358,10 @@ def run(self):
         # Hand the plugin requirements over to the CLI (us) and let it construct the config tree
 
         # Run the argparser
+        if HAS_ARGCOMPLETE:
+            # The autocompletion line must be after the partial_arg handling, so that it doesn't trip it
+            # before all the plugins have been added
+            argcomplete.autocomplete(parser)
         args = parser.parse_args()
         if args.plugin is None:
             parser.error("Please select a plugin to run")
@@ -716,19 +727,17 @@ def _get_final_filename(self):
                 """Gets the final filename"""
                 if output_dir is None:
                     raise TypeError("Output directory is not a string")
+
                 os.makedirs(output_dir, exist_ok=True)
 
-                pref_name_array = self.preferred_filename.split(".")
-                filename, extension = (
-                    os.path.join(output_dir, ".".join(pref_name_array[:-1])),
-                    pref_name_array[-1],
-                )
-                output_filename = f"{filename}.{extension}"
+                output_filename = os.path.join(output_dir, self.preferred_filename)
+                filename, extension = os.path.splitext(output_filename)
 
                 counter = 1
                 while os.path.exists(output_filename):
-                    output_filename = f"{filename}-{counter}.{extension}"
+                    output_filename = f"{filename}-{counter}{extension}"
                     counter += 1
+
                 return output_filename
 
         class CLIMemFileHandler(io.BytesIO, CLIFileHandler):
@@ -791,8 +800,16 @@ def close(self):
                 if self._file.closed:
                     return None
 
-                self._file.close()
                 output_filename = self._get_final_filename()
+
+                # Update the filename, which may have changed if a file with
+                # the same name already existed. This needs to be done before
+                # closing the file, otherwise FileHandlerInterface will raise
+                # an exception. Also, the preferred_filename setter only allows
+                #  a specific set of characters, where '/' is not in that list
+                self.preferred_filename = os.path.basename(output_filename)
+
+                self._file.close()
                 os.rename(self._name, output_filename)
 
         if direct:

volatility3/cli/text_renderer.py

@@ -25,7 +25,7 @@
     vollog.debug("Disassembly library capstone not found")
 
 
-def hex_bytes_as_text(value: bytes) -> str:
+def hex_bytes_as_text(value: bytes, width: int = 16) -> str:
     """Renders HexBytes as text.
 
     Args:
@@ -36,19 +36,24 @@ def hex_bytes_as_text(value: bytes) -> str:
     """
     if not isinstance(value, bytes):
         raise TypeError(f"hex_bytes_as_text takes bytes not: {type(value)}")
-    ascii = []
-    hex = []
-    count = 0
-    output = ""
-    for byte in value:
-        hex.append(f"{byte:02x}")
-        ascii.append(chr(byte) if 0x20 < byte <= 0x7E else ".")
-        if (count % 8) == 7:
-            output += "\n"
-            output += " ".join(hex[count - 7 : count + 1])
-            output += "\t"
-            output += "".join(ascii[count - 7 : count + 1])
-        count += 1
+
+    printables = ""
+    output = "\n"
+    for count, byte in enumerate(value):
+        output += f"{byte:02x} "
+        char = chr(byte)
+        printables += char if 0x20 <= byte <= 0x7E else "."
+        if count % width == width - 1:
+            output += printables
+            if count < len(value) - 1:
+                output += "\n"
+            printables = ""
+
+    # Handle leftovers when the lenght is not mutiple of width
+    if printables:
+        output += "   " * (width - len(printables))
+        output += printables
+
     return output
 
 

volatility3/cli/volargparse.py

@@ -21,8 +21,6 @@ class HelpfulSubparserAction(argparse._SubParsersAction):
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
-        # We don't want the action self-check to kick in, so we remove the choices list, the check happens in __call__
-        self.choices = None
 
     def __call__(
         self,
@@ -100,3 +98,20 @@ def _match_argument(self, action, arg_strings_pattern) -> int:
 
         # return the number of arguments matched
         return len(match.group(1))
+
+    def _check_value(self, action: argparse.Action, value: Any) -> None:
+        """This is called to ensure a value is correct/valid
+
+        In normal operation, it would check that a value provided is valid and return None
+        If it was not valid, it would throw an ArgumentError
+
+        When people provide a partial plugin name, we want to look for a matching plugin name
+        which happens in the HelpfulSubparserAction's __call_method
+
+        To get there without tripping the check_value failure, we have to prevent the exception
+        being thrown when the value is a HelpfulSubparserAction.  This therefore affects no other
+        checks for normal parameters.
+        """
+        if not isinstance(action, HelpfulSubparserAction):
+            super()._check_value(action, value)
+        return None

volatility3/cli/volshell/__init__.py

@@ -21,6 +21,14 @@
     plugins,
 )
 
+try:
+    import argcomplete
+
+    HAS_ARGCOMPLETE = True
+except ImportError:
+    HAS_ARGCOMPLETE = False
+
+
 # Make sure we log everything
 
 rootlog = logging.getLogger()
@@ -276,6 +284,10 @@ def run(self):
         # Hand the plugin requirements over to the CLI (us) and let it construct the config tree
 
         # Run the argparser
+        if HAS_ARGCOMPLETE:
+            # The autocompletion line must be after the partial_arg handling, so that it doesn't trip it
+            # before all the plugins have been added
+            argcomplete.autocomplete(parser)
         args = parser.parse_args()
 
         vollog.log(