diff --git a/.github/workflows/install.yml b/.github/workflows/install.yml new file mode 100644 index 0000000000..cc2a7fd3ec --- /dev/null +++ b/.github/workflows/install.yml @@ -0,0 +1,31 @@ +name: Install Volatility3 test +on: [push, pull_request] +jobs: + + install_test: + runs-on: ${{ matrix.host }} + strategy: + fail-fast: false + matrix: + host: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + steps: + - uses: actions/checkout@v3 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + + - name: Setup python-pip + run: python -m pip install --upgrade pip + + - name: Install dependencies + run: | + pip install -r requirements.txt + + - name: Install volatility3 + run: pip install . + + - name: Run volatility3 + run: vol --help \ No newline at end of file diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000000..925a9f624e --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,23 @@ +name: Close inactive issues +on: + schedule: + - cron: "30 1 * * *" + +jobs: + close-issues: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/stale@v5 + with: + days-before-issue-stale: 200 + days-before-issue-close: 60 + stale-issue-label: "stale" + stale-issue-message: "This issue is stale because it has been open for 200 days with no activity." + close-issue-message: "This issue was closed because it has been inactive for 60 days since being marked as stale." + days-before-pr-stale: -1 + days-before-pr-close: -1 + repo-token: ${{ secrets.GITHUB_TOKEN }} + exempt-issue-labels: "enhancement,plugin-request,question" diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 0000000000..c36c3b7d5c --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,37 @@ +# This CITATION.cff file was generated with cffinit. +# Visit https://bit.ly/cffinit to generate yours today! + +cff-version: 1.2.0 +title: Volatility 3 +message: >- + If you reference this software, please feel free to cite + it using the information below. +type: software +authors: + - name: Volatility Foundation + country: US + website: 'https://www.volatilityfoundation.org/' +identifiers: + - type: url + value: 'https://github.com/volatilityfoundation/volatility3' + description: Volatility 3 source code respository +repository-code: 'https://github.com/volatilityfoundation/volatility3' +url: 'https://github.com/volatilityfoundation/volatility3' +abstract: >- + Volatility is the world's most widely used framework for + extracting digital artifacts from volatile memory (RAM) + samples. The extraction techniques are performed + completely independent of the system being investigated + but offer visibility into the runtime state of the system. + The framework is intended to introduce people to the + techniques and complexities associated with extracting + digital artifacts from volatile memory samples and provide + a platform for further work into this exciting area of + research. +keywords: + - malware + - forensics + - memory + - python + - ram + - volatility diff --git a/README.md b/README.md index 471735af8b..f69acb3bb5 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,7 @@ The latest generated copy of the documentation can be found at: + constants.PLUGINS_PATH failures = framework.import_files(volatility3.plugins, True) +.. note:: + + Volatility uses the `volatility3.plugins` namespace for all plugins (including those in `volatility3.framework.plugins`). + Please ensure you only use `volatility3.plugins` and only ever import plugins from this namespace. + This ensures the ability of users to override core plugins without needing write access to the framework directory. + Once the plugins have been imported, we can interrogate which plugins are available. The :py:func:`~volatility3.framework.list_plugins` call will return a dictionary of plugin names and the plugin classes. diff --git a/requirements-dev.txt b/requirements-dev.txt index 9db14d4411..c9b615cd84 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ # The following packages are required for core functionality. -pefile>=2017.8.1 +pefile>=2023.2.7 # The following packages are optional. # If certain packages are not necessary, place a comment (#) at the start of the line. diff --git a/requirements-minimal.txt b/requirements-minimal.txt index 31ac028148..c030b332dd 100644 --- a/requirements-minimal.txt +++ b/requirements-minimal.txt @@ -1,2 +1,2 @@ # These packages are required for core functionality. -pefile>=2017.8.1 #foo \ No newline at end of file +pefile>=2023.2.7 #foo \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 99e0786cc9..4d09ff82ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # The following packages are required for core functionality. -pefile>=2017.8.1 +pefile>=2023.2.7 # The following packages are optional. # If certain packages are not necessary, place a comment (#) at the start of the line. @@ -16,3 +16,7 @@ pycryptodome # This is required for memory acquisition via leechcore/pcileech. leechcorepyc>=2.4.0 + +# This is required for memory analysis on a Amazon/MinIO S3 and Google Cloud object storage +gcsfs>=2023.1.0 +s3fs>=2023.1.0 \ No newline at end of file diff --git a/setup.py b/setup.py index 936a12af22..c2c55067dc 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ def get_install_requires(): requirements = [] - with open("requirements-minimal.txt", "r", encoding = "utf-8") as fh: + with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: for line in fh.readlines(): stripped_line = line.strip() if stripped_line == "" or stripped_line.startswith("#"): @@ -20,6 +20,7 @@ def get_install_requires(): requirements.append(stripped_line) return requirements + setuptools.setup( name="volatility3", description="Memory forensics framework", @@ -36,12 +37,12 @@ def get_install_requires(): "Documentation": "https://volatility3.readthedocs.io/", "Source Code": "https://github.com/volatilityfoundation/volatility3", }, - python_requires=">=3.7.0", - include_package_data=True, - exclude_package_data={"": ["development", "development.*"], "development": ["*"]}, packages=setuptools.find_namespace_packages( - exclude=["development", "development.*"] + include=["volatility3", "volatility3.*"] ), + package_dir={"volatility3": "volatility3"}, + python_requires=">=3.7.0", + include_package_data=True, entry_points={ "console_scripts": [ "vol = volatility3.cli:main", diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 9bfd14c6c6..91bda7c665 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -662,7 +662,7 @@ def __init__(self, filename: str): def close(self): # Don't overcommit if self.closed: - return + return None self.seek(0) @@ -712,7 +712,7 @@ def close(self): """Closes and commits the file (by moving the temporary file to the correct name""" # Don't overcommit if self._file.closed: - return + return None self._file.close() output_filename = self._get_final_filename() diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index ffb8d516bc..6e58ee68d7 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -389,9 +389,11 @@ class JsonRenderer(CLIRenderer): interfaces.renderers.Disassembly: quoted_optional(display_disassembly), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), - datetime.datetime: lambda x: x.isoformat() - if not isinstance(x, interfaces.renderers.BaseAbsentValue) - else None, + datetime.datetime: lambda x: ( + x.isoformat() + if not isinstance(x, interfaces.renderers.BaseAbsentValue) + else None + ), "default": lambda x: x, } diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index ea9e65d9b5..b95129d19e 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -108,7 +108,7 @@ def help(self, *args): """Describes the available commands""" if args: help(*args) - return + return None variables = [] print("\nMethods:") @@ -325,7 +325,7 @@ def display_type( (str, interfaces.objects.ObjectInterface, interfaces.objects.Template), ): print("Cannot display information about non-type object") - return + return None if not isinstance(object, str): # Mypy requires us to order things this way @@ -453,7 +453,7 @@ def display_symbols(self, symbol_table: str = None): """Prints an alphabetical list of symbols for a symbol table""" if symbol_table is None: print("No symbol table provided") - return + return None longest_offset = longest_name = 0 table = self.context.symbol_space[symbol_table] diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 8c23bbec3e..c5e555ec7d 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -35,9 +35,9 @@ def change_task(self, pid=None): process_layer = task.add_process_layer() if process_layer is not None: self.change_layer(process_layer) - return + return None print(f"Layer for task ID {pid} could not be constructed") - return + return None print(f"No task with task ID {pid} found") def list_tasks(self): diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index b709511b14..2b32ad6776 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -35,9 +35,9 @@ def change_task(self, pid=None): process_layer = task.add_process_layer() if process_layer is not None: self.change_layer(process_layer) - return + return None print(f"Layer for task ID {pid} could not be constructed") - return + return None print(f"No task with task ID {pid} found") def list_tasks(self, method=None): diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index 652b2e66b0..5c2190c027 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -32,7 +32,7 @@ def change_process(self, pid=None): if process.UniqueProcessId == pid: process_layer = process.add_process_layer() self.change_layer(process_layer) - return + return None print(f"No process with process ID {pid} found") def list_processes(self): diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index 9c17846a80..1565b22670 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -206,9 +206,9 @@ def _zipwalk(path: str): if not file.is_dir(): dirlist = zip_results.get(os.path.dirname(file.filename), []) dirlist.append(os.path.basename(file.filename)) - zip_results[ - os.path.join(path, os.path.dirname(file.filename)) - ] = dirlist + zip_results[os.path.join(path, os.path.dirname(file.filename))] = ( + dirlist + ) for value in zip_results: yield value, zip_results[value] diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index aa75fbc3d7..e517531394 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -138,9 +138,9 @@ def stack( config_path = join("automagic", "MacIntelHelper", new_layer_name) context.config[join(config_path, "memory_layer")] = layer_name context.config[join(config_path, "page_map_offset")] = dtb - context.config[ - join(config_path, MacSymbolFinder.banner_config_key) - ] = str(banner, "latin-1") + context.config[join(config_path, MacSymbolFinder.banner_config_key)] = ( + str(banner, "latin-1") + ) new_layer = intel.Intel32e( context, diff --git a/volatility3/framework/automagic/module.py b/volatility3/framework/automagic/module.py index 2bdaf3f625..ff13db9055 100644 --- a/volatility3/framework/automagic/module.py +++ b/volatility3/framework/automagic/module.py @@ -29,21 +29,21 @@ def __call__( requirement.requirements[req], progress_callback, ) - return + return None if not requirement.unsatisfied(context, config_path): - return + return None # The requirement is unfulfilled and is a ModuleRequirement - context.config[ - interfaces.configuration.path_join(new_config_path, "class") - ] = "volatility3.framework.contexts.Module" + context.config[interfaces.configuration.path_join(new_config_path, "class")] = ( + "volatility3.framework.contexts.Module" + ) for req in requirement.requirements: if ( requirement.requirements[req].unsatisfied(context, new_config_path) and req != "offset" ): - return + return None # We now just have the offset requirement, but the layer requirement has been fulfilled. # Unfortunately we don't know the layer name requirement's exact name diff --git a/volatility3/framework/automagic/stacker.py b/volatility3/framework/automagic/stacker.py index e611b5f067..c251d3c46b 100644 --- a/volatility3/framework/automagic/stacker.py +++ b/volatility3/framework/automagic/stacker.py @@ -103,7 +103,7 @@ def stack( appropriate_config_path, layer_name = result context.config.merge(appropriate_config_path, subconfig) context.config[appropriate_config_path] = top_layer_name - return + return None self._cached = None new_context = context.clone() @@ -156,6 +156,9 @@ def stack( self._cached = context.config.get(path, None), context.config.branch( path ) + vollog.debug( + f"physical_layer maximum_address: {physical_layer.maximum_address}" + ) vollog.debug(f"Stacked layers: {stacked_layers}") @classmethod diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index 29f2cfd08b..22f1c94f34 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -429,7 +429,7 @@ def update(self, progress_callback=None): progress_callback(0, "Reading remote ISF list") cursor = self._database.cursor() cursor.execute( - f"SELECT cached FROM cache WHERE local = 0 and cached < datetime('now', {self.cache_period})" + f"SELECT cached FROM cache WHERE local = 0 and cached < datetime('now', '{self.cache_period}')" ) remote_identifiers = RemoteIdentifierFormat(constants.REMOTE_ISF_URL) progress_callback(50, "Reading remote ISF list") @@ -438,9 +438,13 @@ def update(self, progress_callback=None): {}, operating_system=operating_system ) for identifier, location in identifiers: + identifier = identifier.rstrip() + identifier = ( + identifier[:-1] if identifier.endswith(b"\x00") else identifier + ) # Linux banners dumped by dwarf2json end with "\x00\n". If not stripped, the banner cannot match. cursor.execute( "INSERT OR REPLACE INTO cache(identifier, location, operating_system, local, cached) VALUES (?, ?, ?, ?, datetime('now'))", - (location, identifier, operating_system, False), + (identifier, location, operating_system, False), ) progress_callback(100, "Reading remote ISF list") self._database.commit() diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index f30dff456c..21e594549e 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -69,7 +69,7 @@ def __call__( # Bomb out early if our details haven't been configured if self.symbol_class is None: - return + return None self._requirements = self.find_requirements( context, @@ -120,7 +120,7 @@ def _banner_scan( # Bomb out early if there's no banners if not self.banners: - return + return None mss = scanners.MultiStringScanner([x for x in self.banners if x is not None]) @@ -150,12 +150,12 @@ def _banner_scan( clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join - context.config[ - path_join(config_path, requirement.name, "class") - ] = clazz - context.config[ - path_join(config_path, requirement.name, "isf_url") - ] = isf_path + context.config[path_join(config_path, requirement.name, "class")] = ( + clazz + ) + context.config[path_join(config_path, requirement.name, "isf_url")] = ( + isf_path + ) context.config[ path_join(config_path, requirement.name, "symbol_mask") ] = layer.address_mask diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index a8530829ba..52296f5ad5 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -402,19 +402,19 @@ def __call__( if swap_location: context.config[current_layer_path] = current_layer_name try: - context.config[ - layer_loc_path - ] = requirements.URIRequirement.location_from_file( - swap_location + context.config[layer_loc_path] = ( + requirements.URIRequirement.location_from_file( + swap_location + ) ) except ValueError: vollog.warning( f"Volatility swap_location {swap_location} could not be validated - swap layer disabled" ) continue - context.config[ - layer_class_path - ] = "volatility3.framework.layers.physical.FileLayer" + context.config[layer_class_path] = ( + "volatility3.framework.layers.physical.FileLayer" + ) # Add the requirement new_req = requirements.TranslationLayerRequirement( @@ -424,9 +424,9 @@ def __call__( ) swap_req.add_requirement(new_req) - context.config[ - path_join(swap_sub_config, "number_of_elements") - ] = counter + context.config[path_join(swap_sub_config, "number_of_elements")] = ( + counter + ) context.config[swap_sub_config] = True swap_req.construct(context, swap_config) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index abdffdbe49..1c0622574e 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -550,9 +550,9 @@ def unsatisfied( config_path = interfaces.configuration.path_join(config_path, self.name) if not self.matches_required(self._version, self._component.version): return {config_path: self} - context.config[ - interfaces.configuration.path_join(config_path, self.name) - ] = True + context.config[interfaces.configuration.path_join(config_path, self.name)] = ( + True + ) return {} @classmethod diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index de1674885f..09dded076e 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -45,7 +45,7 @@ # We use the SemVer 2.0.0 versioning scheme VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 5 # Number of changes that only add to the interface -VERSION_PATCH = 0 # Number of changes that do not change the interface +VERSION_PATCH = 2 # Number of changes that do not change the interface VERSION_SUFFIX = "" # TODO: At version 2.0.0, remove the symbol_shift feature diff --git a/volatility3/framework/constants/linux/__init__.py b/volatility3/framework/constants/linux/__init__.py index a802e0adaf..6e8883f195 100644 --- a/volatility3/framework/constants/linux/__init__.py +++ b/volatility3/framework/constants/linux/__init__.py @@ -279,3 +279,5 @@ "bpf", "checkpoint_restore", ) + +ELF_MAX_EXTRACTION_SIZE = 1024 * 1024 * 1024 * 4 - 1 diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index 68592f8cbc..e2a68780a7 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -678,16 +678,12 @@ def del_layer(self, name: str) -> None: name: The name of the layer to delete """ for layer in self._layers: - depend_list = [ - superlayer - for superlayer in self._layers - if name in self._layers[layer].dependencies - ] - if depend_list: + if name in self._layers[layer].dependencies: raise exceptions.LayerException( self._layers[layer].name, - f"Layer {self._layers[layer].name} is depended upon: {', '.join(depend_list)}", + f"Layer {self._layers[layer].name} is depended upon by {layer}", ) + # Otherwise, wipe out the layer self._layers[name].destroy() del self._layers[name] diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 0de109c5e8..29395aadf3 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -43,7 +43,7 @@ def preferred_filename(self): return self._preferred_filename @preferred_filename.setter - def preferred_filename(self, filename): + def preferred_filename(self, filename: str): """Sets the preferred filename""" if self.closed: raise IOError("FileHandler name cannot be changed once closed") @@ -57,6 +57,18 @@ def preferred_filename(self, filename): def close(self): """Method that commits the file and fixes the final filename for use""" + @staticmethod + def sanitize_filename(filename: str) -> str: + """Sanititizes the filename to ensure only a specific whitelist of characters is allowed through""" + allowed = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789.- ()[]\{\}!$%^:#~?<>,|" + result = "" + for char in filename: + if char in allowed: + result += char + else: + result += "?" + return result + def __enter__(self): return self diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py index c825464ccc..2e55721923 100644 --- a/volatility3/framework/layers/avml.py +++ b/volatility3/framework/layers/avml.py @@ -224,7 +224,7 @@ def stack( except exceptions.LayerException: return None new_name = context.layers.free_layer_name("AVMLLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return AVMLLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/cloudstorage.py b/volatility3/framework/layers/cloudstorage.py new file mode 100644 index 0000000000..3f88ef34fc --- /dev/null +++ b/volatility3/framework/layers/cloudstorage.py @@ -0,0 +1,57 @@ +# This file is Copyright 2022 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import urllib.parse +from typing import Optional, Any, List + +try: + import s3fs + + HAS_S3FS = True +except ImportError: + HAS_S3FS = False + +try: + import gcsfs + + HAS_GCSFS = True +except ImportError: + HAS_GCSFS = False + +from volatility3.framework import exceptions +from volatility3.framework.layers import resources + +vollog = logging.getLogger(__file__) + +if HAS_S3FS: + + class S3FileSystemHandler(resources.VolatilityHandler): + @classmethod + def non_cached_schemes(cls) -> List[str]: + return ["s3"] + + @staticmethod + def default_open(req: urllib.request.Request) -> Optional[Any]: + """Handles the request if it's the s3 scheme.""" + if req.type == "s3": + object_uri = "://".join(req.full_url.split("://")[1:]) + return s3fs.S3FileSystem().open(object_uri) + return None + + +if HAS_GCSFS: + + class GSFileSystemHandler(resources.VolatilityHandler): + @classmethod + def non_cached_schemes(cls) -> List[str]: + return ["gs"] + + @staticmethod + def default_open(req: urllib.request.Request) -> Optional[Any]: + """Handles the request if it's the gs scheme.""" + if req.type == "gs": + object_uri = "://".join(req.full_url.split("://")[1:]) + return gcsfs.GCSFileSystem().open(object_uri) + return None diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index a10d365926..b2fd6d4d18 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -115,9 +115,9 @@ def stack( vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("Elf64Layer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) try: return Elf64Layer(context, new_name, new_name) diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index 046203fa68..ae477854d6 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -277,9 +277,9 @@ def mapping( This allows translation layers to provide maps of contiguous regions in one layer """ - stashed_offset = ( - stashed_mapped_offset - ) = stashed_size = stashed_mapped_size = stashed_map_layer = None + stashed_offset = stashed_mapped_offset = stashed_size = stashed_mapped_size = ( + stashed_map_layer + ) = None for offset, size, mapped_offset, mapped_size, map_layer in self._mapping( offset, length, ignore_errors ): @@ -331,9 +331,9 @@ def _mapping( except exceptions.InvalidAddressException: if not ignore_errors: raise - return + return None yield offset, length, mapped_offset, length, layer_name - return + return None while length > 0: try: chunk_offset, page_size, layer_name = self._translate(offset) diff --git a/volatility3/framework/layers/lime.py b/volatility3/framework/layers/lime.py index 28d646640b..8b93932ab2 100644 --- a/volatility3/framework/layers/lime.py +++ b/volatility3/framework/layers/lime.py @@ -104,7 +104,7 @@ def stack( except LimeFormatException: return None new_name = context.layers.free_layer_name("LimeLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return LimeLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/msf.py b/volatility3/framework/layers/msf.py index 76c645e92b..8d84a774b8 100644 --- a/volatility3/framework/layers/msf.py +++ b/volatility3/framework/layers/msf.py @@ -47,7 +47,7 @@ def pdb_symbol_table(self) -> str: def read_streams(self): # Shortcut in case they've already been read if self._streams: - return + return None # Recover the root table, by recovering the root table index table... module = self.context.module(self.pdb_symbol_table, self._base_layer, offset=0) diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 501b8655e3..ff483291c9 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -486,9 +486,9 @@ def stack( except exceptions.LayerException: return None new_name = context.layers.free_layer_name("QemuSuspendLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) layer = QemuSuspendLayer(context, new_name, new_name) cls.stacker_slow_warning() return layer diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index cc8ce1f4ca..9841d2bb08 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -171,7 +171,15 @@ def get_key( node (default) or a list of nodes from root to the current node (if return_list is true). """ - node_key = [self.get_node(self.root_cell_offset)] + root_node = self.get_node(self.root_cell_offset) + if not root_node.vol.type_name.endswith(constants.BANG + "_CM_KEY_NODE"): + raise RegistryFormatException( + self.name, + "Encountered {} instead of _CM_KEY_NODE".format( + root_node.vol.type_name + ), + ) + node_key = [root_node] if key.endswith("\\"): key = key[:-1] key_array = key.split("\\") diff --git a/volatility3/framework/layers/segmented.py b/volatility3/framework/layers/segmented.py index beb667436e..0d29d8bff5 100644 --- a/volatility3/framework/layers/segmented.py +++ b/volatility3/framework/layers/segmented.py @@ -126,9 +126,9 @@ def mapping( current_offset = logical_offset # If it starts too late then we're done if logical_offset > offset + length: - return + return None except exceptions.InvalidAddressException: - return + return None # Crop it to the amount we need left chunk_size = min(size, length + offset - logical_offset) yield logical_offset, chunk_size, mapped_offset, mapped_size, self._base_layer diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index 0bc1a350b4..622ff02509 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -4,6 +4,7 @@ import contextlib import logging import struct +import os from typing import Any, Dict, List, Optional from volatility3.framework import constants, exceptions, interfaces @@ -232,6 +233,11 @@ def stack( ) if not vmss_success and not vmsn_success: + vmem_file_basename = os.path.basename(location) + example_vmss_file_basename = os.path.basename(vmss) + vollog.warning( + f"No metadata file found alongside VMEM file. A VMSS or VMSN file may be required to correctly process a VMEM file. These should be placed in the same directory with the same file name, e.g. {vmem_file_basename} and {example_vmss_file_basename}.", + ) return None new_layer_name = context.layers.free_layer_name("VmwareLayer") context.config[ diff --git a/volatility3/framework/layers/xen.py b/volatility3/framework/layers/xen.py index f7881a0911..927b304301 100644 --- a/volatility3/framework/layers/xen.py +++ b/volatility3/framework/layers/xen.py @@ -173,8 +173,8 @@ def stack( vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("XenCoreDumpLayer") - context.config[ - interfaces.configuration.path_join(new_name, "base_layer") - ] = layer_name + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = ( + layer_name + ) return XenCoreDumpLayer(context, new_name, new_name) diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 3b1745718c..316a30becd 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -593,17 +593,19 @@ def _generate_inverse_choices(cls, choices: Dict[str, int]) -> Dict[int, str]: inverse_choices: Dict[int, str] = {} for k, v in choices.items(): if v in inverse_choices: - # Technically this shouldn't be a problem, but since we inverse cache - # and can't map one value to two possibilities we throw an exception during build - # We can remove/work around this if it proves a common issue - raise ValueError( - f"Enumeration value {v} duplicated as {k} and {inverse_choices[v]}" + vollog.log( + constants.LOGLEVEL_VVV, + f"Enumeration value {v} duplicated as {k}. Keeping name {inverse_choices[v]}", ) + continue inverse_choices[v] = k return inverse_choices def lookup(self, value: int = None) -> str: - """Looks up an individual value and returns the associated name.""" + """Looks up an individual value and returns the associated name. + + If multiple identifiers map to the same value, the first matching identifier will be returned + """ if value is None: return self.lookup(self) if value in self._inverse_choices: @@ -640,7 +642,10 @@ class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): @classmethod def lookup(cls, template: interfaces.objects.Template, value: int) -> str: - """Looks up an individual value and returns the associated name.""" + """Looks up an individual value and returns the associated name. + + If multiple identifiers map to the same value, the first matching identifier will be returned + """ _inverse_choices = Enumeration._generate_inverse_choices( template.vol["choices"] ) @@ -763,12 +768,10 @@ def child_template( raise IndexError(f"Member not present in array template: {child}") @overload - def __getitem__(self, i: int) -> interfaces.objects.Template: - ... + def __getitem__(self, i: int) -> interfaces.objects.Template: ... @overload - def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: - ... + def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: ... def __getitem__(self, i): """Returns the i-th item from the array.""" diff --git a/volatility3/framework/objects/utility.py b/volatility3/framework/objects/utility.py index 177074cd7a..0292608c12 100644 --- a/volatility3/framework/objects/utility.py +++ b/volatility3/framework/objects/utility.py @@ -44,8 +44,9 @@ def array_of_pointers( raise TypeError( "Subtype must be a valid template (or string name of an object template)" ) + # We have to clone the pointer class, or we'll be defining the pointer subtype for all future pointers subtype_pointer = context.symbol_space.get_type( symbol_table + constants.BANG + "pointer" - ) + ).clone() subtype_pointer.update_vol(subtype=subtype) return array.cast("array", count=count, subtype=subtype_pointer) diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index b7dc2c16bf..ce4567ca60 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -75,11 +75,16 @@ def _generator(self, tasks): bang_addrs = [] + # get task memory sections to be used by scanners + task_memory_sections = [ + section for section in task.get_process_memory_sections(heap_only=True) + ] + # find '#' values on the heap for address in proc_layer.scan( self.context, scanners.BytesScanner(b"#"), - sections=task.get_process_memory_sections(heap_only=True), + sections=task_memory_sections, ): bang_addrs.append(struct.pack(pack_format, address)) @@ -89,7 +94,7 @@ def _generator(self, tasks): for address, _ in proc_layer.scan( self.context, scanners.MultiStringScanner(bang_addrs), - sections=task.get_process_memory_sections(heap_only=True), + sections=task_memory_sections, ): hist = self.context.object( bash_table_name + constants.BANG + "hist_entry", diff --git a/volatility3/framework/plugins/linux/capabilities.py b/volatility3/framework/plugins/linux/capabilities.py index 518f526039..bfdb69aba2 100644 --- a/volatility3/framework/plugins/linux/capabilities.py +++ b/volatility3/framework/plugins/linux/capabilities.py @@ -88,7 +88,7 @@ def _check_capabilities_support( kernel_cap_last_cap = vmlinux.object_from_symbol(symbol_name="cap_last_cap") except exceptions.SymbolError: # It should be a kernel < 3.2 - return + return None vol2_last_cap = extensions.kernel_cap_struct.get_last_cap_value() if kernel_cap_last_cap > vol2_last_cap: diff --git a/volatility3/framework/plugins/linux/check_afinfo.py b/volatility3/framework/plugins/linux/check_afinfo.py index 90e714eaab..7fced6acd1 100644 --- a/volatility3/framework/plugins/linux/check_afinfo.py +++ b/volatility3/framework/plugins/linux/check_afinfo.py @@ -51,10 +51,22 @@ def _check_members(self, var_ops, var_name, members): yield check, addr def _check_afinfo(self, var_name, var, op_members, seq_members): - for hooked_member, hook_address in self._check_members( - var.seq_fops, var_name, op_members - ): - yield var_name, hooked_member, hook_address + # check if object has a least one of the members used for analysis by this function + required_members = ["seq_fops", "seq_ops", "seq_show"] + has_required_member = any( + [var.has_member(member) for member in required_members] + ) + if not has_required_member: + vollog.debug( + f"{var_name} object at {hex(var.vol.offset)} had none of the required members: {', '.join([member for member in required_members])}" + ) + raise exceptions.PluginRequirementException + + if var.has_member("seq_fops"): + for hooked_member, hook_address in self._check_members( + var.seq_fops, var_name, op_members + ): + yield var_name, hooked_member, hook_address # newer kernels if var.has_member("seq_ops"): @@ -64,8 +76,10 @@ def _check_afinfo(self, var_name, var, op_members, seq_members): yield var_name, hooked_member, hook_address # this is the most commonly hooked member by rootkits, so a force a check on it - elif not self._is_known_address(var.seq_show): - yield var_name, "show", var.seq_show + else: + if var.has_member("seq_show"): + if not self._is_known_address(var.seq_show): + yield var_name, "show", var.seq_show def _generator(self): vmlinux = self.context.modules[self.config["kernel"]] @@ -85,6 +99,12 @@ def _generator(self): ) protocols = [tcp, udp] + # used to track the calls to _check_afinfo and the + # number of errors produced due to missing members + symbols_checked = set() + symbols_with_errors = set() + + # loop through all symbols for struct_type, global_vars in protocols: for global_var_name in global_vars: # this will lookup fail for the IPv6 protocols on kernels without IPv6 support @@ -97,10 +117,20 @@ def _generator(self): object_type=struct_type, offset=global_var.address ) - for name, member, address in self._check_afinfo( - global_var_name, global_var, op_members, seq_members - ): - yield 0, (name, member, format_hints.Hex(address)) + symbols_checked.add(global_var_name) + try: + for name, member, address in self._check_afinfo( + global_var_name, global_var, op_members, seq_members + ): + yield 0, (name, member, format_hints.Hex(address)) + except exceptions.PluginRequirementException: + symbols_with_errors.add(global_var_name) + + # if every call to _check_afinfo failed show a warning + if symbols_checked == symbols_with_errors: + vollog.warning( + "This plugin was not able to check for hooks. This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt." + ) def run(self): return renderers.TreeGrid( diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index b1d2919f9c..b6634d612d 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -145,7 +145,7 @@ def _generator(self): table_info = self._get_table_info(vmlinux, "sys_call_table", ptr_sz) except exceptions.SymbolError: vollog.error("Unable to find the system call table. Exiting.") - return + return None tables = [(table_name, table_info)] diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 2ff5eb591e..e688ecb424 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -4,20 +4,26 @@ """A module containing a collection of plugins that produce data typically found in Linux's /proc file system.""" -from typing import List +import logging +from typing import List, Optional, Type -from volatility3.framework import renderers, interfaces +from volatility3.framework import constants, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.linux.extensions import elf from volatility3.plugins.linux import pslist +vollog = logging.getLogger(__name__) + class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" _required_framework_version = (2, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -36,9 +42,93 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] element_type=int, optional=True, ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed processes", + default=False, + optional=True, + ), ] + @classmethod + def elf_dump( + cls, + context: interfaces.context.ContextInterface, + layer_name: str, + elf_table_name: str, + vma: interfaces.objects.ObjectInterface, + task: interfaces.objects.ObjectInterface, + open_method: Type[interfaces.plugins.FileHandlerInterface], + ) -> Optional[interfaces.plugins.FileHandlerInterface]: + """Extracts an ELF as a FileHandlerInterface + Args: + context: the context to operate upon + layer_name: The name of the layer on which to operate + elf_table_name: the name for the symbol table containing the symbols for ELF-files + vma: virtual memory allocation of ELF + task: the task object whose memory should be output + open_method: class to provide context manager for opening the file + Returns: + An open FileHandlerInterface object containing the complete data for the task or None in the case of failure + """ + + proc_layer = context.layers[layer_name] + file_handle = None + + elf_object = context.object( + elf_table_name + constants.BANG + "Elf", + offset=vma.vm_start, + layer_name=layer_name, + ) + + if not elf_object.is_valid(): + return None + + sections = {} + # TODO: Apply more effort to reconstruct ELF, e.g.: https://github.com/enbarberis/core2ELF64 ? + for phdr in elf_object.get_program_headers(): + if phdr.p_type != 1: # PT_LOAD = 1 + continue + + start = phdr.p_vaddr + size = phdr.p_memsz + end = start + size + + # Use complete memory pages for dumping + # If start isn't a multiple of 4096, stick to the highest multiple < start + # If end isn't a multiple of 4096, stick to the lowest multiple > end + if start % 4096: + start = start & ~0xFFF + + if end % 4096: + end = (end & ~0xFFF) + 4096 + + real_size = end - start + + # Check if ELF has a legitimate size + if real_size < 0 or real_size > constants.linux.ELF_MAX_EXTRACTION_SIZE: + raise ValueError(f"The claimed size of the ELF is invalid: {real_size}") + + sections[start] = real_size + + elf_data = b"" + for section_start in sorted(sections.keys()): + read_size = sections[section_start] + + buf = proc_layer.read(vma.vm_start + section_start, read_size, pad=True) + elf_data = elf_data + buf + + file_handle = open_method( + f"pid.{task.pid}.{utility.array_to_string(task.comm)}.{vma.vm_start:#x}.dmp" + ) + file_handle.write(elf_data) + + return file_handle + def _generator(self, tasks): + elf_table_name = intermed.IntermediateSymbolTable.create( + self.context, self.config_path, "linux", "elf", class_types=elf.class_types + ) for task in tasks: proc_layer_name = task.add_process_layer() if not proc_layer_name: @@ -60,6 +150,21 @@ def _generator(self, tasks): path = vma.get_name(self.context, task) + file_output = "Disabled" + if self.config["dump"]: + file_handle = self.elf_dump( + self.context, + proc_layer_name, + elf_table_name, + vma, + task, + self.open, + ) + file_output = "Error outputting file" + if file_handle: + file_handle.close() + file_output = str(file_handle.preferred_filename) + yield ( 0, ( @@ -68,6 +173,7 @@ def _generator(self, tasks): format_hints.Hex(vma.vm_start), format_hints.Hex(vma.vm_end), path, + file_output, ), ) @@ -81,6 +187,7 @@ def run(self): ("Start", format_hints.Hex), ("End", format_hints.Hex), ("File Path", str), + ("File Output", str), ], self._generator( pslist.PsList.list_tasks( diff --git a/volatility3/framework/plugins/linux/iomem.py b/volatility3/framework/plugins/linux/iomem.py index 785405ef3c..6732084dbc 100644 --- a/volatility3/framework/plugins/linux/iomem.py +++ b/volatility3/framework/plugins/linux/iomem.py @@ -16,7 +16,7 @@ class IOMem(interfaces.plugins.PluginInterface): """Generates an output similar to /proc/iomem on a running system.""" _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -53,7 +53,7 @@ def parse_resource( # create the resource object with protection against memory smear try: - resource = vmlinux.object("resource", resource_offset) + resource = vmlinux.object("resource", resource_offset, absolute=True) except exceptions.InvalidAddressException: vollog.warning( f"Unable to create resource object at {resource_offset:#x}. This resource, " diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py index 3f7345bdc5..5136a00f65 100644 --- a/volatility3/framework/plugins/linux/kmsg.py +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -64,10 +64,8 @@ def __init__( ): self._context = context self._config = config - vmlinux = context.modules[self._config["kernel"]] - self.layer_name = vmlinux.layer_name # type: ignore - symbol_table_name = vmlinux.symbol_table_name # type: ignore - self.vmlinux = contexts.Module.create(context, symbol_table_name, self.layer_name, 0) # type: ignore + self.vmlinux = context.modules[self._config["kernel"]] + self.layer_name = self.vmlinux.layer_name # type: ignore self.long_unsigned_int_size = self.vmlinux.get_type("long unsigned int").size @classmethod @@ -358,19 +356,24 @@ def run(self) -> Iterator[Tuple[str, str, str, str, str]]: desc_ring = ringbuffers.desc_ring text_data_ring = ringbuffers.text_data_ring - desc_count = 1 << desc_ring.count_bits - desc_arr = self.vmlinux.object( - object_type="array", + + array_type = self.vmlinux.symbol_table_name + constants.BANG + "array" + + desc_arr = self._context.object( + array_type, offset=desc_ring.descs, subtype=self.vmlinux.get_type("prb_desc"), count=desc_count, + layer_name=self.layer_name, ) - info_arr = self.vmlinux.object( - object_type="array", + + info_arr = self._context.object( + array_type, offset=desc_ring.infos, subtype=self.vmlinux.get_type("printk_info"), count=desc_count, + layer_name=self.layer_name, ) # See kernel/printk/printk_ringbuffer.h @@ -409,7 +412,7 @@ class Kmsg(plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 8a21afc03f..cf06ee0ccb 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -44,7 +44,7 @@ def _list_injections(self, task): proc_layer_name = task.add_process_layer() if not proc_layer_name: - return + return None proc_layer = self.context.layers[proc_layer_name] diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index af260a772c..9afd13e5a3 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -1,12 +1,15 @@ # This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from typing import Callable, Iterable, List, Any, Tuple +from typing import Any, Callable, Iterable, List, Tuple -from volatility3.framework import renderers, interfaces +from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.linux.extensions import elf +from volatility3.plugins.linux import elfs class PsList(interfaces.plugins.PluginInterface): @@ -14,7 +17,7 @@ class PsList(interfaces.plugins.PluginInterface): _required_framework_version = (2, 0, 0) - _version = (2, 1, 0) + _version = (2, 2, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -24,6 +27,9 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] description="Linux kernel", architectures=["Intel32", "Intel64"], ), + requirements.PluginRequirement( + name="elfs", plugin=elfs.Elfs, version=(2, 0, 0) + ), requirements.ListRequirement( name="pid", description="Filter on specific process IDs", @@ -42,6 +48,12 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] optional=True, default=False, ), + requirements.BooleanRequirement( + name="dump", + description="Extract listed processes", + optional=True, + default=False, + ), ] @classmethod @@ -66,11 +78,11 @@ def filter_func(x): else: return lambda _: False - def _get_task_fields( - self, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False + @classmethod + def get_task_fields( + cls, task: interfaces.objects.ObjectInterface, decorate_comm: bool = False ) -> Tuple[int, int, int, str]: """Extract the fields needed for the final output - Args: task: A task object from where to get the fields. decorate_comm: If True, it decorates the comm string of @@ -90,14 +102,54 @@ def _get_task_fields( elif task.is_user_thread: name = f"{{{name}}}" - task_fields = (format_hints.Hex(task.vol.offset), pid, tid, ppid, name) + task_fields = (task.vol.offset, pid, tid, ppid, name) return task_fields + def _get_file_output(self, task: interfaces.objects.ObjectInterface) -> str: + """Extract the elf for the process if requested + Args: + task: A task object to extract from. + Returns: + A string showing the results of the extraction, either + the filename used or an error. + """ + elf_table_name = intermed.IntermediateSymbolTable.create( + self.context, + self.config_path, + "linux", + "elf", + class_types=elf.class_types, + ) + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + # if we can't build a proc layer we can't + # extract the elf + return renderers.NotApplicableValue() + else: + # Find the vma that belongs to the main ELF of the process + file_output = "Error outputting file" + for v in task.mm.get_mmap_iter(): + if v.vm_start == task.mm.start_code: + file_handle = elfs.Elfs.elf_dump( + self.context, + proc_layer_name, + elf_table_name, + v, + task, + self.open, + ) + if file_handle: + file_output = str(file_handle.preferred_filename) + file_handle.close() + break + return file_output + def _generator( self, pid_filter: Callable[[Any], bool], include_threads: bool = False, decorate_comm: bool = False, + dump: bool = False, ): """Generates the tasks list. @@ -110,14 +162,29 @@ def _generator( - User threads: in curly brackets, - Kernel threads: in square brackets Defaults to False. + dump: If True, the main executable of the process is written to a file + Defaults to False. Yields: Each rows """ for task in self.list_tasks( self.context, self.config["kernel"], pid_filter, include_threads ): - row = self._get_task_fields(task, decorate_comm) - yield (0, row) + if dump: + file_output = self._get_file_output(task) + else: + file_output = "Disabled" + + offset, pid, tid, ppid, name = self.get_task_fields(task, decorate_comm) + + yield 0, ( + format_hints.Hex(offset), + pid, + tid, + ppid, + name, + file_output, + ) @classmethod def list_tasks( @@ -155,6 +222,7 @@ def run(self): pids = self.config.get("pid") include_threads = self.config.get("threads") decorate_comm = self.config.get("decorate_comm") + dump = self.config.get("dump") filter_func = self.create_pid_filter(pids) columns = [ @@ -163,7 +231,8 @@ def run(self): ("TID", int), ("PPID", int), ("COMM", str), + ("File output", str), ] return renderers.TreeGrid( - columns, self._generator(filter_func, include_threads, decorate_comm) + columns, self._generator(filter_func, include_threads, decorate_comm, dump) ) diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index e07a8aced5..efe5223dfd 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -2,18 +2,49 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints from volatility3.plugins.linux import pslist -class PsTree(pslist.PsList): +class PsTree(interfaces.plugins.PluginInterface): """Plugin for listing processes in a tree based on their parent process ID.""" - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self._tasks = {} - self._levels = {} - self._children = {} + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 2, 0) + ), + requirements.ListRequirement( + name="pid", + description="Filter on specific process IDs", + element_type=int, + optional=True, + ), + requirements.BooleanRequirement( + name="threads", + description="Include user threads", + optional=True, + default=False, + ), + requirements.BooleanRequirement( + name="decorate_comm", + description="Show `user threads` comm in curly brackets, and `kernel threads` comm in square brackets", + optional=True, + default=False, + ), + ] def find_level(self, pid: int) -> None: """Finds how deep the PID is in the tasks hierarchy. @@ -39,15 +70,14 @@ def find_level(self, pid: int) -> None: self._levels[pid] = level def _generator( - self, pid_filter, include_threads: bool = False, decorate_com: bool = False + self, + tasks: list, + decorate_comm: bool = False, ): """Generates the tasks hierarchy tree. Args: - pid_filter: A function which takes a process object and returns True if the process should be ignored/filtered - include_threads: If True, the output will also show the user threads - If False, only the thread group leaders will be shown - Defaults to False. + tasks: A list of task objects to be displayed decorate_comm: If True, it decorates the comm string of - User threads: in curly brackets, - Kernel threads: in square brackets @@ -55,13 +85,12 @@ def _generator( Yields: Each rows """ - vmlinux = self.context.modules[self.config["kernel"]] - for proc in self.list_tasks( - self.context, - vmlinux.name, - filter_func=pid_filter, - include_threads=include_threads, - ): + + self._tasks = {} + self._levels = {} + self._children = {} + + for proc in tasks: self._tasks[proc.pid] = proc # Build the child/level maps @@ -71,7 +100,10 @@ def _generator( def yield_processes(pid): task = self._tasks[pid] - row = self._get_task_fields(task, decorate_com) + row = pslist.PsList.get_task_fields(task, decorate_comm) + # update the first element, the offset, in the row tuple to use format_hints.Hex + # as a simple int is returned from get_task_fields. + row = (format_hints.Hex(row[0]),) + row[1:] tid = task.pid yield (self._levels[tid] - 1, row) @@ -82,3 +114,27 @@ def yield_processes(pid): for pid, level in self._levels.items(): if level == 1: yield from yield_processes(pid) + + def run(self): + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + include_threads = self.config.get("threads") + decorate_comm = self.config.get("decorate_comm") + + return renderers.TreeGrid( + [ + ("OFFSET (V)", format_hints.Hex), + ("PID", int), + ("TID", int), + ("PPID", int), + ("COMM", str), + ], + self._generator( + pslist.PsList.list_tasks( + self.context, + self.config["kernel"], + filter_func=filter_func, + include_threads=include_threads, + ), + decorate_comm=decorate_comm, + ), + ) diff --git a/volatility3/framework/plugins/linux/sockstat.py b/volatility3/framework/plugins/linux/sockstat.py index fa67122baa..e9c98a2274 100644 --- a/volatility3/framework/plugins/linux/sockstat.py +++ b/volatility3/framework/plugins/linux/sockstat.py @@ -147,7 +147,7 @@ def _extract_socket_filter_info( socket_filter["bpf_filter_type"] = "cBPF" if not sock_filter.has_member("prog") or not sock_filter.prog: - return + return None bpfprog = sock_filter.prog @@ -158,13 +158,13 @@ def _extract_socket_filter_info( return # cBPF filter except AttributeError: # kernel < 3.18.140, it's a cBPF filter - return + return None BPF_PROG_TYPE_SOCKET_FILTER = 1 # eBPF filter if bpfprog_type != BPF_PROG_TYPE_SOCKET_FILTER: socket_filter["bpf_filter_type"] = f"UNK({bpfprog_type})" vollog.warning(f"Unexpected BPF type {bpfprog_type} for a socket") - return + return None socket_filter["bpf_filter_type"] = "eBPF" if not bpfprog.has_member("aux") or not bpfprog.aux: @@ -329,17 +329,17 @@ def _xdp_sock( xdp_sock = sock.cast("xdp_sock") device = xdp_sock.dev if not device: - return + return None src_addr = utility.array_to_string(device.name) src_port = dst_addr = dst_port = None bpfprog = device.xdp_prog if not bpfprog: - return + return None if not bpfprog.has_member("aux") or not bpfprog.aux: - return + return None bpfprog_aux = bpfprog.aux if bpfprog_aux.has_member("id"): diff --git a/volatility3/framework/plugins/linux/vmayarascan.py b/volatility3/framework/plugins/linux/vmayarascan.py new file mode 100644 index 0000000000..eda0d7dcaf --- /dev/null +++ b/volatility3/framework/plugins/linux/vmayarascan.py @@ -0,0 +1,113 @@ +# This file is Copyright 2023 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from typing import Iterable, List, Tuple + +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.plugins import yarascan +from volatility3.plugins.linux import pslist + + +class VmaYaraScan(interfaces.plugins.PluginInterface): + """Scans all virtual memory areas for tasks using yara.""" + + _required_framework_version = (2, 4, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + # create a list of requirements for vmayarascan + vmayarascan_requirements = [ + requirements.ListRequirement( + name="pid", + element_type=int, + description="Process IDs to include (all other processes are excluded)", + optional=True, + ), + requirements.PluginRequirement( + name="pslist", plugin=pslist.PsList, version=(2, 0, 0) + ), + requirements.PluginRequirement( + name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0) + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + requirements.ModuleRequirement( + name="kernel", + description="Linux kernel", + architectures=["Intel32", "Intel64"], + ), + ] + + # get base yarascan requirements for command line options + yarascan_requirements = yarascan.YaraScan.get_yarascan_option_requirements() + + # return the combined requirements + return yarascan_requirements + vmayarascan_requirements + + def _generator(self): + # use yarascan to parse the yara options provided and create the rules + rules = yarascan.YaraScan.process_yara_options(dict(self.config)) + + # filter based on the pid option if provided + filter_func = pslist.PsList.create_pid_filter(self.config.get("pid", None)) + for task in pslist.PsList.list_tasks( + context=self.context, + vmlinux_module_name=self.config["kernel"], + filter_func=filter_func, + ): + # attempt to create a process layer for each task and skip those + # that cannot (e.g. kernel threads) + proc_layer_name = task.add_process_layer() + if not proc_layer_name: + continue + + # get the proc_layer object from the context + proc_layer = self.context.layers[proc_layer_name] + + # scan the process layer with the yarascanner + for offset, rule_name, name, value in proc_layer.scan( + context=self.context, + scanner=yarascan.YaraScanner(rules=rules), + sections=self.get_vma_maps(task), + ): + yield 0, ( + format_hints.Hex(offset), + task.tgid, + rule_name, + name, + value, + ) + + @staticmethod + def get_vma_maps( + task: interfaces.objects.ObjectInterface, + ) -> Iterable[Tuple[int, int]]: + """Creates a map of start/end addresses for each virtual memory area in a task. + + Args: + task: The task object of which to read the vmas from + + Returns: + An iterable of tuples containing start and end addresses for each descriptor + """ + if task.mm: + for vma in task.mm.get_vma_iter(): + vm_size = vma.vm_end - vma.vm_start + yield (vma.vm_start, vm_size) + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("PID", int), + ("Rule", str), + ("Component", str), + ("Value", bytes), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/mac/check_sysctl.py b/volatility3/framework/plugins/mac/check_sysctl.py index 165aad436b..4f64eaed80 100644 --- a/volatility3/framework/plugins/mac/check_sysctl.py +++ b/volatility3/framework/plugins/mac/check_sysctl.py @@ -69,7 +69,7 @@ def _process_sysctl_list(self, kernel, sysctl_list, recursive=0): try: sysctl = sysctl.oid_link.sle_next.dereference() except exceptions.InvalidAddressException: - return + return None while sysctl: try: diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index 3b996bc0af..2a8692b772 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -116,7 +116,7 @@ def _walk_klist_array(cls, kernel, fdp, array_pointer_member, array_size_member) ) except exceptions.InvalidAddressException: - return + return None for klist in klist_array: for kn in mac.MacUtilities.walk_slist(klist, "kn_link"): @@ -140,7 +140,7 @@ def _get_task_kevents(cls, kernel, task): try: p_klist = task.p_klist except exceptions.InvalidAddressException: - return + return None for kn in mac.MacUtilities.walk_slist(p_klist, "kn_link"): yield kn diff --git a/volatility3/framework/plugins/mac/lsmod.py b/volatility3/framework/plugins/mac/lsmod.py index 2979e374be..c6f57f8892 100644 --- a/volatility3/framework/plugins/mac/lsmod.py +++ b/volatility3/framework/plugins/mac/lsmod.py @@ -75,7 +75,7 @@ def list_modules( try: kmod = kmod.next except exceptions.InvalidAddressException: - return + return None return # Generation finished def _generator(self): diff --git a/volatility3/framework/plugins/mac/malfind.py b/volatility3/framework/plugins/mac/malfind.py index 98b282e24b..3094ada852 100644 --- a/volatility3/framework/plugins/mac/malfind.py +++ b/volatility3/framework/plugins/mac/malfind.py @@ -40,7 +40,7 @@ def _list_injections(self, task): proc_layer_name = task.add_process_layer() if proc_layer_name is None: - return + return None proc_layer = self.context.layers[proc_layer_name] diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 88045a277e..9b570f3f9c 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -49,9 +49,7 @@ def get_requirements(cls): ] @classmethod - def get_list_tasks( - cls, method: str - ) -> Callable[ + def get_list_tasks(cls, method: str) -> Callable[ [interfaces.context.ContextInterface, str, Callable[[int], bool]], Iterable[interfaces.objects.ObjectInterface], ]: diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index a9b669add4..6e667984a2 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -108,12 +108,12 @@ def _generator(self, syshive, sechive): vollog.warning("Unable to locate SYSTEM hive") if sechive is None: vollog.warning("Unable to locate SECURITY hive") - return + return None bootkey = hashdump.Hashdump.get_bootkey(syshive) if not bootkey: vollog.warning("Unable to find bootkey") - return + return None kernel = self.context.modules[self.config["kernel"]] @@ -124,17 +124,17 @@ def _generator(self, syshive, sechive): lsakey = lsadump.Lsadump.get_lsa_key(sechive, bootkey, vista_or_later) if not lsakey: vollog.warning("Unable to find lsa key") - return + return None nlkm = self.get_nlkm(sechive, lsakey, vista_or_later) if not nlkm: vollog.warning("Unable to find nlkma key") - return + return None cache = hashdump.Hashdump.get_hive_key(sechive, "Cache") if not cache: vollog.warning("Unable to find cache key") - return + return None for cache_item in cache.get_values(): if cache_item.Name == "NL$Control": diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 48b2e7c620..fcc333b9fa 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -157,7 +157,7 @@ def _list_registry_callbacks_legacy( ) if callback_count == 0: - return + return None fast_refs = ntkrnlmp.object( object_type="array", @@ -199,7 +199,7 @@ def _list_registry_callbacks_new( ) if callback_count == 0: - return + return None callback_list = ntkrnlmp.object(object_type="_LIST_ENTRY", offset=symbol_offset) for callback in callback_list.to_list(full_type_name, "Link"): @@ -256,7 +256,7 @@ def list_registry_callbacks( symbol_status = "exists" vollog.debug(f"symbol {symbol_name} {symbol_status}.") - return + return None @classmethod def list_bugcheck_reason_callbacks( @@ -287,7 +287,7 @@ def list_bugcheck_reason_callbacks( ).address except exceptions.SymbolError: vollog.debug("Cannot find KeBugCheckReasonCallbackListHead") - return + return None full_type_name = ( callback_table_name + constants.BANG + "_KBUGCHECK_REASON_CALLBACK_RECORD" @@ -343,7 +343,7 @@ def list_bugcheck_callbacks( list_offset = ntkrnlmp.get_symbol("KeBugCheckCallbackListHead").address except exceptions.SymbolError: vollog.debug("Cannot find KeBugCheckCallbackListHead") - return + return None full_type_name = ( callback_table_name + constants.BANG + "_KBUGCHECK_CALLBACK_RECORD" diff --git a/volatility3/framework/plugins/windows/crashinfo.py b/volatility3/framework/plugins/windows/crashinfo.py index 4ecd850872..862eb60807 100644 --- a/volatility3/framework/plugins/windows/crashinfo.py +++ b/volatility3/framework/plugins/windows/crashinfo.py @@ -46,9 +46,9 @@ def _generator(self, layer: crash.WindowsCrashDump32Layer): bitmap_size = format_hints.Hex(summary_header.BitmapSize) bitmap_pages = format_hints.Hex(summary_header.Pages) else: - bitmap_header_size = ( - bitmap_size - ) = bitmap_pages = renderers.NotApplicableValue() + bitmap_header_size = bitmap_size = bitmap_pages = ( + renderers.NotApplicableValue() + ) yield ( 0, diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 38d55d15dc..dd82d897ea 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -130,7 +130,7 @@ def process_file_object( constants.LOGLEVEL_VVV, f"The file object at {file_obj.vol.offset:#x} is not a file on disk", ) - return + return None # Depending on the type of object (DataSection, ImageSection, SharedCacheMap) we may need to # read from the memory layer or the primary layer. diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index dd7c90860a..ddd9cb78e1 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -285,7 +285,7 @@ def _make_handle_array(self, offset, level, depth=0): count = 0x1000 / subtype.size if not self.context.layers[virtual].is_valid(offset): - return + return None table = ntkrnlmp.object( object_type="array", @@ -335,7 +335,7 @@ def handles(self, handle_table): constants.LOGLEVEL_VVV, "Handle table parsing was aborted due to an invalid address exception", ) - return + return None for handle_table_entry in self._make_handle_array(TableCode, table_levels): yield handle_table_entry diff --git a/volatility3/framework/plugins/windows/lsadump.py b/volatility3/framework/plugins/windows/lsadump.py index 12589b07e5..da8dee3258 100644 --- a/volatility3/framework/plugins/windows/lsadump.py +++ b/volatility3/framework/plugins/windows/lsadump.py @@ -168,16 +168,16 @@ def _generator( lsakey = self.get_lsa_key(sechive, bootkey, vista_or_later) if not bootkey: vollog.warning("Unable to find bootkey") - return + return None if not lsakey: vollog.warning("Unable to find lsa key") - return + return None secrets_key = hashdump.Hashdump.get_hive_key(sechive, "Policy\\Secrets") if not secrets_key: vollog.warning("Unable to find secrets key") - return + return None for key in secrets_key.get_subkeys(): sec_val_key = hashdump.Hashdump.get_hive_key( diff --git a/volatility3/framework/plugins/windows/malfind.py b/volatility3/framework/plugins/windows/malfind.py index 4249259555..6ed078996c 100644 --- a/volatility3/framework/plugins/windows/malfind.py +++ b/volatility3/framework/plugins/windows/malfind.py @@ -110,7 +110,7 @@ def list_injections( proc_id, excp.invalid_address, excp.layer_name ) ) - return + return None proc_layer = context.layers[proc_layer_name] diff --git a/volatility3/framework/plugins/windows/mftscan.py b/volatility3/framework/plugins/windows/mftscan.py index 87416d274d..a266fd864d 100644 --- a/volatility3/framework/plugins/windows/mftscan.py +++ b/volatility3/framework/plugins/windows/mftscan.py @@ -67,9 +67,8 @@ def _generator(self): ) # We will update this on each pass in the next loop and use it as the new offset. attr_base_offset = mft_record.FirstAttrOffset - - attr_header = self.context.object( - header_object, + attr = self.context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) @@ -77,17 +76,8 @@ def _generator(self): # There is no field that has a count of Attributes # Keep Attempting to read attributes until we get an invalid attr_header.AttrType - while attr_header.AttrType.is_valid_choice: - vollog.debug(f"Attr Type: {attr_header.AttrType.lookup()}") - - # Offset past the headers to the attribute data - attr_data_offset = ( - offset - + attr_base_offset - + self.context.symbol_space.get_type( - attribute_object - ).relative_child_offset("Attr_Data") - ) + while attr.Attr_Header.AttrType.is_valid_choice: + vollog.debug(f"Attr Type: {attr.Attr_Header.AttrType.lookup()}") # MFT Flags determine the file type or dir # If we don't have a valid enum, coerce to hex so we can keep the record @@ -97,19 +87,16 @@ def _generator(self): mft_flag = hex(mft_record.Flags) # Standard Information Attribute - if attr_header.AttrType.lookup() == "STANDARD_INFORMATION": - attr_data = self.context.object( - si_object, offset=attr_data_offset, layer_name=layer.name - ) - + if attr.Attr_Header.AttrType.lookup() == "STANDARD_INFORMATION": + attr_data = attr.Attr_Data.cast(si_object) yield 0, ( - format_hints.Hex(attr_data_offset), + format_hints.Hex(attr_data.vol.offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, renderers.NotApplicableValue(), - attr_header.AttrType.lookup(), + attr.Attr_Header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -118,10 +105,8 @@ def _generator(self): ) # File Name Attribute - if attr_header.AttrType.lookup() == "FILE_NAME": - attr_data = self.context.object( - fn_object, offset=attr_data_offset, layer_name=layer.name - ) + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + attr_data = attr.Attr_Data.cast(fn_object) file_name = attr_data.get_full_name() # If we don't have a valid enum, coerce to hex so we can keep the record @@ -131,13 +116,13 @@ def _generator(self): permissions = hex(attr_data.Flags) yield 1, ( - format_hints.Hex(attr_data_offset), + format_hints.Hex(attr_data.vol.offset), mft_record.get_signature(), mft_record.RecordNumber, mft_record.LinkCount, mft_flag, permissions, - attr_header.AttrType.lookup(), + attr.Attr_Header.AttrType.lookup(), conversion.wintime_to_datetime(attr_data.CreationTime), conversion.wintime_to_datetime(attr_data.ModifiedTime), conversion.wintime_to_datetime(attr_data.UpdatedTime), @@ -146,14 +131,13 @@ def _generator(self): ) # If there's no advancement the loop will never end, so break it now - if attr_header.Length == 0: + if attr.Attr_Header.Length == 0: break # Update the base offset to point to the next attribute - attr_base_offset += attr_header.Length - # Get the next attribute - attr_header = self.context.object( - header_object, + attr_base_offset += attr.Attr_Header.Length + attr = self.context.object( + attribute_object, offset=offset + attr_base_offset, layer_name=layer.name, ) @@ -189,3 +173,139 @@ def run(self): ], self._generator(), ) + + +class ADS(interfaces.plugins.PluginInterface): + """Scans for Alternate Data Stream""" + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement( + name="primary", + description="Memory layer for the kernel", + architectures=["Intel32", "Intel64"], + ), + requirements.VersionRequirement( + name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) + ), + ] + + def _generator(self): + layer = self.context.layers[self.config["primary"]] + + # Yara Rule to scan for MFT Header Signatures + rules = yarascan.YaraScan.process_yara_options( + {"yara_rules": "/FILE0|FILE\*|BAAD/"} + ) + + # Read in the Symbol File + symbol_table = intermed.IntermediateSymbolTable.create( + context=self.context, + config_path=self.config_path, + sub_path="windows", + filename="mft", + class_types={ + "MFT_ENTRY": mft.MFTEntry, + "FILE_NAME_ENTRY": mft.MFTFileName, + "ATTRIBUTE": mft.MFTAttribute, + }, + ) + + # get each of the individual Field Sets + mft_object = symbol_table + constants.BANG + "MFT_ENTRY" + attribute_object = symbol_table + constants.BANG + "ATTRIBUTE" + fn_object = symbol_table + constants.BANG + "FILE_NAME_ENTRY" + + # Scan the layer for Raw MFT records and parse the fields + for offset, _rule_name, _name, _value in layer.scan( + context=self.context, scanner=yarascan.YaraScanner(rules=rules) + ): + with contextlib.suppress(exceptions.PagedInvalidAddressException): + mft_record = self.context.object( + mft_object, offset=offset, layer_name=layer.name + ) + # We will update this on each pass in the next loop and use it as the new offset. + attr_base_offset = mft_record.FirstAttrOffset + + attr = self.context.object( + attribute_object, + offset=offset + attr_base_offset, + layer_name=layer.name, + ) + + # There is no field that has a count of Attributes + # Keep Attempting to read attributes until we get an invalid attr.AttrType + is_ads = False + file_name = renderers.NotAvailableValue + # The First $DATA Attr is the 'principal' file itself not the ADS + while attr.Attr_Header.AttrType.is_valid_choice: + if attr.Attr_Header.AttrType.lookup() == "FILE_NAME": + attr_data = attr.Attr_Data.cast(fn_object) + file_name = attr_data.get_full_name() + if attr.Attr_Header.AttrType.lookup() == "DATA": + if is_ads: + if not attr.Attr_Header.NonResidentFlag: + # Resident files are the most interesting. + if attr.Attr_Header.NameLength > 0: + ads_name = attr.get_resident_filename() + if not ads_name: + ads_name = renderers.NotAvailableValue + + content = attr.get_resident_filecontent() + if content: + # Preparing for Disassembly + disasm = interfaces.renderers.BaseAbsentValue + architecture = layer.metadata.get( + "architecture", None + ) + if architecture: + disasm = interfaces.renderers.Disassembly( + content, 0, architecture.lower() + ) + else: + content = renderers.NotAvailableValue + disasm = interfaces.renderers.BaseAbsentValue + + yield 0, ( + format_hints.Hex(attr_data.vol.offset), + mft_record.get_signature(), + mft_record.RecordNumber, + attr.Attr_Header.AttrType.lookup(), + file_name, + ads_name, + format_hints.HexBytes(content), + disasm, + ) + else: + is_ads = True + + # If there's no advancement the loop will never end, so break it now + if attr.Attr_Header.Length == 0: + break + + # Update the base offset to point to the next attribute + attr_base_offset += attr.Attr_Header.Length + # Get the next attribute + attr = self.context.object( + attribute_object, + offset=offset + attr_base_offset, + layer_name=layer.name, + ) + + def run(self): + return renderers.TreeGrid( + [ + ("Offset", format_hints.Hex), + ("Record Type", str), + ("Record Number", int), + ("MFT Type", str), + ("Filename", str), + ("ADS Filename", str), + ("Hexdump", format_hints.HexBytes), + ("Disasm", interfaces.renderers.Disassembly), + ], + self._generator(), + ) diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index d0bbd5cbdd..62ead3ab7d 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -487,10 +487,12 @@ def generate_timeline(self): if not isinstance(row_data[9], datetime.datetime): continue row_data = [ - "N/A" - if isinstance(i, renderers.UnreadableValue) - or isinstance(i, renderers.UnparsableValue) - else i + ( + "N/A" + if isinstance(i, renderers.UnreadableValue) + or isinstance(i, renderers.UnparsableValue) + else i + ) for i in row_data ] description = ( diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py index d3ce3fd2e2..24eb02018b 100644 --- a/volatility3/framework/plugins/windows/netstat.py +++ b/volatility3/framework/plugins/windows/netstat.py @@ -154,7 +154,7 @@ def enumerate_structures_by_port( ) else: # invalid argument. - return + return None vollog.debug(f"Current Port: {port}") # the given port serves as a shifted index into the port pool lists @@ -175,7 +175,7 @@ def enumerate_structures_by_port( assignment = inpa.InPaBigPoolBase.Assignments[truncated_port] if not assignment: - return + return None # the value within assignment.Entry is a) masked and b) points inside of the network object # first decode the pointer diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index 88697e71ae..e7a0d5dd46 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -90,9 +90,19 @@ def process_dump( offset=peb.ImageBaseAddress, layer_name=proc_layer_name, ) + + process_name = proc.ImageFileName.cast( + "string", + max_length=proc.ImageFileName.vol.count, + errors="replace", + ) + file_handle = open_method( - f"pid.{proc.UniqueProcessId}.{peb.ImageBaseAddress:#x}.dmp" + open_method.sanitize_filename( + f"{proc.UniqueProcessId}.{process_name}.{peb.ImageBaseAddress:#x}.dmp" + ) ) + for offset, data in dos_header.reconstruct(): file_handle.seek(offset) file_handle.write(data) diff --git a/volatility3/framework/plugins/windows/pstree.py b/volatility3/framework/plugins/windows/pstree.py index 5c78d16828..a39fe7485f 100644 --- a/volatility3/framework/plugins/windows/pstree.py +++ b/volatility3/framework/plugins/windows/pstree.py @@ -108,13 +108,13 @@ def _generator( def yield_processes(pid, descendant: bool = False): if pid in process_pids: vollog.debug(f"Pid cycle: already processed pid {pid}") - return + return None process_pids.add(pid) if pid not in self._ancestors and not descendant: vollog.debug(f"Pid cycle: pid {pid} not in filtered tree") - return + return None proc, offset = self._processes[pid] row = ( diff --git a/volatility3/framework/plugins/windows/registry/hivelist.py b/volatility3/framework/plugins/windows/registry/hivelist.py index 91798de405..1cc76dad68 100644 --- a/volatility3/framework/plugins/windows/registry/hivelist.py +++ b/volatility3/framework/plugins/windows/registry/hivelist.py @@ -30,7 +30,7 @@ def __iter__(self): ): if not hive.is_valid(): self._invalid = hive.vol.offset - return + return None yield hive @property diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index 537bfc943a..180f8f9d9f 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -74,7 +74,7 @@ def key_iterator( node_path = [hive.get_node(hive.root_cell_offset)] if not isinstance(node_path, list) or len(node_path) < 1: vollog.warning("Hive walker was not passed a valid node_path (or None)") - return + return None node = node_path[-1] key_path_items = [hive] + node_path[1:] key_path = "\\".join([k.get_name() for k in key_path_items]) @@ -153,6 +153,11 @@ def _printkey_iterator( vollog.debug(excp) key_node_name = renderers.UnreadableValue() + # if the item is a subkey, use the LastWriteTime of that subkey + last_write_time = conversion.wintime_to_datetime( + node.LastWriteTime.QuadPart + ) + yield ( depth, ( @@ -188,9 +193,9 @@ def _printkey_iterator( vollog.debug( "Couldn't read registry value type, so data is unreadable" ) - value_data: Union[ - interfaces.renderers.BaseAbsentValue, bytes - ] = renderers.UnreadableValue() + value_data: Union[interfaces.renderers.BaseAbsentValue, bytes] = ( + renderers.UnreadableValue() + ) else: try: value_data = node.decode_data() diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index f90724f66c..70c75b50b1 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -173,11 +173,11 @@ def list_userassist( if not userassist_node_path: vollog.warning("list_userassist did not find a valid node_path (or None)") - return + return None if not isinstance(userassist_node_path, list): vollog.warning("userassist_node_path did not return a list as expected") - return + return None userassist_node = userassist_node_path[-1] # iterate through the GUIDs under the userassist key for guidkey in userassist_node.get_subkeys(): diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py index b697774cb7..d321c2cc0c 100644 --- a/volatility3/framework/plugins/windows/skeleton_key_check.py +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -601,21 +601,21 @@ def _generator(self, procs): if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name): vollog.info("This plugin only supports 64bit Windows memory samples") - return + return None lsass_proc, proc_layer_name = self._find_lsass_proc(procs) if not lsass_proc: vollog.info( "Unable to find a valid lsass.exe process in the process list. This should never happen. Analysis cannot proceed." ) - return + return None cryptdll_base, cryptdll_size = self._find_cryptdll(lsass_proc) if not cryptdll_base: vollog.info( "Unable to find the location of cryptdll.dll inside of lsass.exe. Analysis cannot proceed." ) - return + return None # the custom type information from binary analysis cryptdll_types = self._get_cryptdll_types( @@ -649,7 +649,7 @@ def _generator(self, procs): vollog.info( "Unable to find CSystems inside of cryptdll.dll. Analysis cannot proceed." ) - return + return None for csystem in csystems: if not self.context.layers[proc_layer_name].is_valid( diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index 4b30a9d8b4..d795818e93 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -18,47 +18,26 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" _required_framework_version = (2, 4, 0) - _version = (1, 0, 0) + _version = (1, 0, 1) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: - return [ + # create a list of requirements for vadyarascan + vadyarascan_requirements = [ requirements.ModuleRequirement( name="kernel", description="Windows kernel", architectures=["Intel32", "Intel64"], ), - requirements.BooleanRequirement( - name="wide", - description="Match wide (unicode) strings", - default=False, - optional=True, - ), - requirements.StringRequirement( - name="yara_rules", description="Yara rules (as a string)", optional=True - ), - requirements.URIRequirement( - name="yara_file", description="Yara rules (as a file)", optional=True - ), - # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code - # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later - requirements.URIRequirement( - name="yara_compiled_file", - description="Yara compiled rules (as a file)", - optional=True, - ), - requirements.IntRequirement( - name="max_size", - default=0x40000000, - description="Set the maximum size (default is 1GB)", - optional=True, - ), requirements.PluginRequirement( name="pslist", plugin=pslist.PsList, version=(2, 0, 0) ), requirements.VersionRequirement( name="yarascanner", component=yarascan.YaraScanner, version=(2, 0, 0) ), + requirements.PluginRequirement( + name="yarascan", plugin=yarascan.YaraScan, version=(1, 2, 0) + ), requirements.ListRequirement( name="pid", element_type=int, @@ -67,6 +46,12 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] ), ] + # get base yarascan requirements for command line options + yarascan_requirements = yarascan.YaraScan.get_yarascan_option_requirements() + + # return the combined requirements + return yarascan_requirements + vadyarascan_requirements + def _generator(self): kernel = self.context.modules[self.config["kernel"]] diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 1c84676895..11c7086073 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -61,19 +61,31 @@ class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" _required_framework_version = (2, 0, 0) - _version = (1, 1, 0) + _version = (1, 2, 0) # TODO: When the major version is bumped, take the opportunity to rename the yara_rules config to yara_string # or something that makes more sense @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: - return [ + """Returns the requirements needed to run yarascan directly, combining the TranslationLayerRequirement + and the requirements from get_yarascan_option_requirements.""" + return cls.get_yarascan_option_requirements() + [ requirements.TranslationLayerRequirement( name="primary", description="Memory layer for the kernel", architectures=["Intel32", "Intel64"], - ), + ) + ] + + @classmethod + def get_yarascan_option_requirements( + cls, + ) -> List[interfaces.configuration.RequirementInterface]: + """Returns the requirements needed for the command lines options used by yarascan. This can + then also be used by other plugins that are using yarascan. This does not include a + TranslationLayerRequirement or a ModuleRequirement.""" + return [ requirements.BooleanRequirement( name="insensitive", description="Makes the search case insensitive", diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index 534686022c..43bb59a210 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -10,7 +10,7 @@ import collections.abc import datetime import logging -from typing import Any, Callable, Iterable, List, Optional, Tuple, TypeVar, Union +from typing import Any, Callable, Dict, Iterable, List, Optional, Tuple, TypeVar, Union from volatility3.framework import interfaces from volatility3.framework.interfaces import renderers @@ -96,6 +96,10 @@ def _validate_values(self, values: List[interfaces.renderers.BaseTypes]) -> None # if isinstance(val, datetime.datetime): # tznaive = val.tzinfo is None or val.tzinfo.utcoffset(val) is None + def asdict(self) -> Dict[str, Any]: + """Returns the contents of the node as a dictionary""" + return self._values._asdict() + @property def values(self) -> List[interfaces.renderers.BaseTypes]: """Returns the list of values from the particular node, based on column diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index bf7da9ecb2..bb18fcc8a6 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -28,9 +28,9 @@ def wintime_to_datetime( def unixtime_to_datetime( unixtime: int, ) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: - ret: Union[ - interfaces.renderers.BaseAbsentValue, datetime.datetime - ] = renderers.UnparsableValue() + ret: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] = ( + renderers.UnparsableValue() + ) if unixtime > 0: with contextlib.suppress(ValueError): diff --git a/volatility3/framework/renderers/format_hints.py b/volatility3/framework/renderers/format_hints.py index 6ec9ebab97..6120b77c93 100644 --- a/volatility3/framework/renderers/format_hints.py +++ b/volatility3/framework/renderers/format_hints.py @@ -59,7 +59,8 @@ def __init__( def __eq__(self, other): return ( - super(self) == super(other) + isinstance(other, self.__class__) + and super() == super(self.__class__, other) and self.converted_int == other.converted_int and self.encoding == other.encoding and self.split_nulls == other.split_nulls diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index 10cf39cf19..d1e7a104d6 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -35,9 +35,9 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): def __init__(self) -> None: super().__init__() - self._dict: Dict[ - str, interfaces.symbols.BaseSymbolTableInterface - ] = collections.OrderedDict() + self._dict: Dict[str, interfaces.symbols.BaseSymbolTableInterface] = ( + collections.OrderedDict() + ) # Permanently cache all resolved symbols self._resolved: Dict[str, interfaces.objects.Template] = {} self._resolved_symbols: Dict[str, interfaces.objects.Template] = {} @@ -73,9 +73,9 @@ def get_symbols_by_location( self, offset: int, size: int = 0, table_name: str = None ) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" - table_list: Iterable[ - interfaces.symbols.BaseSymbolTableInterface - ] = self._dict.values() + table_list: Iterable[interfaces.symbols.BaseSymbolTableInterface] = ( + self._dict.values() + ) if table_name is not None: if table_name in self._dict: table_list = [self._dict[table_name]] @@ -179,15 +179,15 @@ def _iterative_resolve(self, traverse_list): if child.vol.type_name not in self._resolved: traverse_list.append(child.vol.type_name) try: - self._resolved[ - child.vol.type_name - ] = self._weak_resolve( - SymbolType.TYPE, child.vol.type_name + self._resolved[child.vol.type_name] = ( + self._weak_resolve( + SymbolType.TYPE, child.vol.type_name + ) ) except exceptions.SymbolError: - self._resolved[ - child.vol.type_name - ] = self.UnresolvedTemplate(child.vol.type_name) + self._resolved[child.vol.type_name] = ( + self.UnresolvedTemplate(child.vol.type_name) + ) # Stash the replacement replacements.add((traverser, child)) elif child.children: diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 5c42a436d6..c4e2587f49 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -29,10 +29,11 @@ def __init__(self, *args, **kwargs) -> None: self.set_type_class("files_struct", extensions.files_struct) self.set_type_class("kobject", extensions.kobject) self.set_type_class("cred", extensions.cred) - self.set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) # Might not exist in the current symbols self.optional_set_type_class("module", extensions.module) self.optional_set_type_class("bpf_prog", extensions.bpf_prog) + self.optional_set_type_class("kernel_cap_struct", extensions.kernel_cap_struct) + self.optional_set_type_class("kernel_cap_t", extensions.kernel_cap_t) # Mount self.set_type_class("vfsmount", extensions.vfsmount) @@ -243,17 +244,17 @@ def files_descriptors_for_process( ): # task.files can be null if not task.files: - return + return None fd_table = task.files.get_fds() if fd_table == 0: - return + return None max_fds = task.files.get_max_fds() # corruption check if max_fds > 500000: - return + return None file_type = symbol_table + constants.BANG + "file" @@ -378,7 +379,7 @@ def container_of( """ if not addr: - return + return None type_dec = vmlinux.get_type(type_name) member_offset = type_dec.relative_child_offset(member_name) diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 616e54e703..d8a2867cce 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -71,11 +71,11 @@ def get_name(self): def _get_sect_count(self, grp): """Try to determine the number of valid sections""" arr = self._context.object( - self.get_symbol_table().name + constants.BANG + "array", + self.get_symbol_table_name() + constants.BANG + "array", layer_name=self.vol.layer_name, offset=grp.attrs, subtype=self._context.symbol_space.get_type( - self.get_symbol_table().name + constants.BANG + "pointer" + self.get_symbol_table_name() + constants.BANG + "pointer" ), count=25, ) @@ -92,11 +92,11 @@ def get_sections(self): else: num_sects = self._get_sect_count(self.sect_attrs.grp) arr = self._context.object( - self.get_symbol_table().name + constants.BANG + "array", + self.get_symbol_table_name() + constants.BANG + "array", layer_name=self.vol.layer_name, offset=self.sect_attrs.attrs.vol.offset, subtype=self._context.symbol_space.get_type( - self.get_symbol_table().name + constants.BANG + "module_sect_attr" + self.get_symbol_table_name() + constants.BANG + "module_sect_attr" ), count=num_sects, ) @@ -104,41 +104,82 @@ def get_sections(self): for attr in arr: yield attr - def get_symbols(self): - if symbols.symbol_table_is_64bit(self._context, self.get_symbol_table().name): - prefix = "Elf64_" - else: - prefix = "Elf32_" + def get_elf_table_name(self): elf_table_name = intermed.IntermediateSymbolTable.create( - self.context, - self.config_path, + self._context, + "elf_symbol_table", "linux", "elf", native_types=None, class_types=elf.class_types, ) + return elf_table_name + + def get_symbols(self): + """Get symbols of the module + + Yields: + A symbol object + """ + if not hasattr(self, "_elf_table_name"): + self._elf_table_name = self.get_elf_table_name() + if symbols.symbol_table_is_64bit(self._context, self.get_symbol_table_name()): + prefix = "Elf64_" + else: + prefix = "Elf32_" syms = self._context.object( - self.get_symbol_table().name + constants.BANG + "array", + self.get_symbol_table_name() + constants.BANG + "array", layer_name=self.vol.layer_name, offset=self.section_symtab, subtype=self._context.symbol_space.get_type( - elf_table_name + constants.BANG + prefix + "Sym" + self._elf_table_name + constants.BANG + prefix + "Sym" ), count=self.num_symtab + 1, ) if self.section_strtab: for sym in syms: - sym.set_cached_strtab(self.section_strtab) yield sym - def get_symbol(self, wanted_sym_name): - """Get value for a given symbol name""" + def get_symbols_names_and_addresses(self) -> Tuple[str, int]: + """Get names and addresses for each symbol of the module + + Yields: + A tuple for each symbol containing the symbol name and its corresponding value + """ + for sym in self.get_symbols(): - sym_name = sym.get_name() - sym_addr = sym.st_value + sym_arr = self._context.object( + self.get_symbol_table_name() + constants.BANG + "array", + layer_name=self.vol.native_layer_name, + offset=self.section_strtab + sym.st_name, + ) + try: + sym_name = utility.array_to_string( + sym_arr, 512 + ) # 512 is the value of KSYM_NAME_LEN kernel constant + except exceptions.InvalidAddressException: + continue + if sym_name != "": + # Normalize sym.st_value offset, which is an address pointing to the symbol value + mask = self._context.layers[self.vol.layer_name].address_mask + sym_address = sym.st_value & mask + yield (sym_name, sym_address) + + def get_symbol(self, wanted_sym_name): + """Get symbol value for a given symbol name""" + for sym_name, sym_address in self.get_symbols_names_and_addresses(): if wanted_sym_name == sym_name: - return sym_addr + return sym_address + + return None + + def get_symbol_by_address(self, wanted_sym_address): + """Get symbol name for a given symbol address""" + for sym_name, sym_address in self.get_symbols_names_and_addresses(): + if wanted_sym_address == sym_address: + return sym_name + return None @property @@ -203,7 +244,7 @@ def get_process_memory_sections( ) -> Generator[Tuple[int, int], None, None]: """Returns a list of sections based on the memory manager's view of this task's virtual memory.""" - for vma in self.mm.get_mmap_iter(): + for vma in self.mm.get_vma_iter(): start = int(vma.vm_start) end = int(vma.vm_end) @@ -309,19 +350,30 @@ def _parse_maple_tree_node( maple_tree_entry, parent, expected_maple_tree_depth, - seen=set(), + seen=None, current_depth=1, ): """Recursively parse Maple Tree Nodes and yield all non empty slots""" + # Create seen set if it does not exist, e.g. on the first call into this recursive function. This + # must be None or an existing set of addresses for MTEs that have already been processed or that + # should otherwise be ignored. If parsing from the root node for example this should be None on the + # first call. If you needed to parse all nodes downwards from part of the tree this should still be + # None. If however you wanted to parse from a node, but ignore some parts of the tree below it then + # this could be populated with the addresses of the nodes you wish to ignore. + + if seen == None: + seen = set() + # protect against unlikely loop if maple_tree_entry in seen: vollog.warning( f"The mte {hex(maple_tree_entry)} has all ready been seen, no further results will be produced for this node." ) - return + return None else: seen.add(maple_tree_entry) + # check if we have exceeded the expected depth of this maple tree. # e.g. when current_depth is larger than expected_maple_tree_depth there may be an issue. # it is normal that expected_maple_tree_depth is equal to current_depth. @@ -330,6 +382,7 @@ def _parse_maple_tree_node( f"The depth for the maple tree at {hex(self.vol.offset)} is {expected_maple_tree_depth}, however when parsing the nodes " f"a depth of {current_depth} was reached. This is unexpected and may lead to incorrect results." ) + # parse the mte to extract the pointer value, node type, and leaf status pointer = maple_tree_entry & ~(self.MAPLE_NODE_POINTER_MASK) node_type = ( @@ -402,7 +455,7 @@ def get_mmap_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: "get_mmap_iter called on mm_struct where no mmap member exists." ) if not self.mmap: - return + return None yield self.mmap seen = {self.mmap.vol.offset} @@ -723,7 +776,7 @@ def to_list( try: link = getattr(self, direction).dereference() except exceptions.InvalidAddressException: - return + return None if not sentinel: yield self._context.object( symbol_type, layer, offset=self.vol.offset - relative_offset @@ -1132,7 +1185,7 @@ def get_devname(self) -> str: class kobject(objects.StructType): def reference_count(self): refcnt = self.kref.refcount - if self.has_member("counter"): + if refcnt.has_member("counter"): ret = refcnt.counter else: ret = refcnt.refs.counter @@ -1218,7 +1271,7 @@ def get_inode(self): return self.sk_socket.get_inode() def get_protocol(self): - return + return None def get_state(self): # Return the generic socket state @@ -1230,13 +1283,13 @@ def get_state(self): class unix_sock(objects.StructType): def get_name(self): if not self.addr: - return + return None sockaddr_un = self.addr.name.cast("sockaddr_un") saddr = str(utility.array_to_string(sockaddr_un.sun_path)) return saddr def get_protocol(self): - return + return None def get_state(self): """Return a string representing the sock state.""" @@ -1295,7 +1348,7 @@ def get_dst_port(self): elif hasattr(sk_common, "skc_dport"): dport_le = sk_common.skc_dport else: - return + return None return socket_module.htons(dport_le) def get_src_addr(self): @@ -1313,7 +1366,7 @@ def get_src_addr(self): addr_size = 16 saddr = self.pinet6.saddr else: - return + return None parent_layer = self._context.layers[self.vol.layer_name] try: addr_bytes = parent_layer.read(saddr.vol.offset, addr_size) @@ -1321,7 +1374,7 @@ def get_src_addr(self): vollog.debug( f"Unable to read socket src address from {saddr.vol.offset:#x}" ) - return + return None return socket_module.inet_ntop(family, addr_bytes) def get_dst_addr(self): @@ -1342,7 +1395,7 @@ def get_dst_addr(self): daddr = sk_common.skc_v6_daddr addr_size = 16 else: - return + return None parent_layer = self._context.layers[self.vol.layer_name] try: addr_bytes = parent_layer.read(daddr.vol.offset, addr_size) @@ -1350,7 +1403,7 @@ def get_dst_addr(self): vollog.debug( f"Unable to read socket dst address from {daddr.vol.offset:#x}" ) - return + return None return socket_module.inet_ntop(family, addr_bytes) @@ -1388,7 +1441,7 @@ def get_dst_portid(self): class vsock_sock(objects.StructType): def get_protocol(self): # The protocol should always be 0 for vsocks - return + return None def get_state(self): # Return the generic socket state @@ -1399,7 +1452,7 @@ class packet_sock(objects.StructType): def get_protocol(self): eth_proto = socket_module.htons(self.num) if eth_proto == 0: - return + return None elif eth_proto in ETH_PROTOCOLS: return ETH_PROTOCOLS[eth_proto] else: @@ -1425,7 +1478,7 @@ def get_state(self): class xdp_sock(objects.StructType): def get_protocol(self): # The protocol should always be 0 for xdp_sock - return + return None def get_state(self): # xdp_sock.state is an enum @@ -1490,13 +1543,13 @@ def euid(self): class kernel_cap_struct(objects.StructType): - # struct kernel_cap_struct was added in kernels 2.5.0 + # struct kernel_cap_struct exists from 2.1.92 <= kernels < 6.3 @classmethod def get_last_cap_value(cls) -> int: """Returns the latest capability ID supported by the framework. Returns: - int: The latest supported capability ID supported by the framework. + int: The latest capability ID supported by the framework. """ return len(CAPABILITIES) - 1 @@ -1504,7 +1557,7 @@ def get_kernel_cap_full(self) -> int: """Return the maximum value allowed for this kernel for a capability Returns: - int: _description_ + int: The capability full bitfield mask """ vmlinux = linux.LinuxUtilities.get_module_from_volobj_type(self._context, self) try: @@ -1540,17 +1593,29 @@ def get_capabilities(self) -> int: int: The capability bitfield value. """ + if not self.has_member("cap"): + raise exceptions.VolatilityException( + "Unsupported kernel capabilities implementation" + ) + if isinstance(self.cap, objects.Array): - # In 2.6.25.x <= kernels < 6.3 kernel_cap_struct::cap is a two - # elements __u32 array that constitutes a 64bit bitfield. - # Technically, it can also be an array of 1 element if - # _KERNEL_CAPABILITY_U32S = _LINUX_CAPABILITY_U32S_1 - # However, in the source code, that never happens. - # From 2.6.24 to 2.6.25 cap became an array of 2 elements. - cap_value = (self.cap[1] << 32) | self.cap[0] + if len(self.cap) == 1: + # At least in the vanilla kernel, from 2.6.24 to 2.6.25 + # kernel_cap_struct::cap become a two elements array. + # However, in some distros or custom kernel can technically + # be _KERNEL_CAPABILITY_U32S = _LINUX_CAPABILITY_U32S_1 + # Leaving this code here for the sake of ensuring completeness. + cap_value = self.cap[0] + elif len(self.cap) == 2: + # In 2.6.25.x <= kernels < 6.3 kernel_cap_struct::cap is a two + # elements __u32 array that constitutes a 64bit bitfield. + cap_value = (self.cap[1] << 32) | self.cap[0] + else: + raise exceptions.VolatilityException( + "Unsupported kernel capabilities implementation" + ) else: - # In kernels < 2.6.25.x kernel_cap_struct::cap was a __u32 - # In kernels >= 6.3 kernel_cap_struct::cap is a u64 + # In kernels < 2.6.25.x kernel_cap_struct::cap is a __u32 cap_value = self.cap return cap_value & self.get_kernel_cap_full() @@ -1581,3 +1646,23 @@ def has_capability(self, capability: str) -> bool: cap_value = 1 << CAPABILITIES.index(capability) return cap_value & self.get_capabilities() != 0 + + +class kernel_cap_t(kernel_cap_struct): + # In kernels 6.3 kernel_cap_struct became the kernel_cap_t typedef + def get_capabilities(self) -> int: + """Returns the capability bitfield value + + Returns: + int: The capability bitfield value. + """ + + if self.has_member("val"): + # In kernels >= 6.3 kernel_cap_t::val is a u64 + cap_value = self.val + else: + raise exceptions.VolatilityException( + "Unsupported kernel capabilities implementation" + ) + + return cap_value & self.get_kernel_cap_full() diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 416a7e4d28..fe85b194fd 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -3,9 +3,12 @@ # from typing import Dict, Tuple +import logging from volatility3.framework import constants -from volatility3.framework import objects, interfaces +from volatility3.framework import objects, interfaces, exceptions + +vollog = logging.getLogger(__name__) class elf(objects.StructType): @@ -33,14 +36,23 @@ def __init__( layer_name = self.vol.layer_name symbol_table_name = self.get_symbol_table_name() # We read the MAGIC: (0x0 to 0x4) 0x7f 0x45 0x4c 0x46 - magic = self._context.object( - symbol_table_name + constants.BANG + "unsigned long", - layer_name=layer_name, - offset=object_info.offset, - ) + try: + magic = self._context.object( + symbol_table_name + constants.BANG + "unsigned long", + layer_name=layer_name, + offset=object_info.offset, + ) + except ( + exceptions.PagedInvalidAddressException, + exceptions.InvalidAddressException, + ) as excp: + vollog.debug( + f"Unable to check magic bytes for ELF file at offset {hex(object_info.offset)} in layer {layer_name}: {excp}" + ) + return None # Check validity - if magic != 0x464C457F: + if magic != 0x464C457F: # e.g. ELF return None # We need to read the EI_CLASS (0x4 offset) @@ -72,7 +84,10 @@ def is_valid(self): """ Determine whether it is a valid object """ - return self._type_prefix is not None and self._hdr is not None + if hasattr(self, "_type_prefix") and hasattr(self, "_hdr"): + return self._type_prefix is not None and self._hdr is not None + else: + return False def __getattr__(self, name): # Just redirect to the corresponding header @@ -171,7 +186,7 @@ def get_symbols(self): self._find_symbols() if self._cached_symtab is None: - return + return None symtab_arr = self._context.object( self.get_symbol_table_name() + constants.BANG + "array", diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index 56ac96633f..bc98e5bdc9 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -169,7 +169,7 @@ def files_descriptors_for_process( try: table_addr = task.p_fd.fd_ofiles.dereference() except exceptions.InvalidAddressException: - return + return None fds = objects.utility.array_of_pointers( table_addr, count=num_fds, subtype=file_type, context=context @@ -204,7 +204,7 @@ def _walk_iterable( try: current = queue.member(attr=list_head_member) except exceptions.InvalidAddressException: - return + return None while current: if current.vol.offset in seen: diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index c89b527e62..bf0b3d775d 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -50,7 +50,7 @@ def get_map_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: task = self.get_task() current_map = task.map.hdr.links.next except exceptions.InvalidAddressException: - return + return None seen: Set[int] = set() @@ -138,13 +138,13 @@ def get_map_object(self): class vnode(objects.StructType): def _do_calc_path(self, ret, vnodeobj, vname): if vnodeobj is None: - return + return None if vname: try: ret.append(utility.pointer_to_string(vname, 255)) except exceptions.InvalidAddressException: - return + return None if int(vnodeobj.v_flag) & 0x000001 != 0 and int(vnodeobj.v_mount) != 0: if int(vnodeobj.v_mount.mnt_vnodecovered) != 0: @@ -158,7 +158,7 @@ def _do_calc_path(self, ret, vnodeobj, vname): parent = vnodeobj.v_parent parent_name = parent.v_name except exceptions.InvalidAddressException: - return + return None self._do_calc_path(ret, parent, parent_name) @@ -502,7 +502,7 @@ def walk_list( yielded = yielded + 1 if yielded == max_size: - return + return None n = ( getattr(n.member(attr=member_name), attr) diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index ba00a40533..846e5bd90d 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -91,7 +91,7 @@ def traverse(self, visited=None, depth=0): if vad_address in visited: vollog.log(constants.LOGLEVEL_VVV, "VAD node already seen!") - return + return None visited.add(vad_address) tag = self.get_tag() @@ -111,7 +111,7 @@ def traverse(self, visited=None, depth=0): constants.LOGLEVEL_VVV, f"Skipping VAD at {self.vol.offset} depth {depth} with tag {tag}", ) - return + return None if target: vad_object = self.cast(target) @@ -452,9 +452,9 @@ def is_valid(self) -> bool: ].is_valid(self.FileName.Buffer) def file_name_with_device(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: - name: Union[ - str, interfaces.renderers.BaseAbsentValue - ] = renderers.UnreadableValue() + name: Union[str, interfaces.renderers.BaseAbsentValue] = ( + renderers.UnreadableValue() + ) # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. @@ -665,7 +665,7 @@ def load_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: ): yield entry except exceptions.InvalidAddressException: - return + return None def init_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: """Generator for DLLs in the order that they were initialized""" @@ -678,7 +678,7 @@ def init_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: ): yield entry except exceptions.InvalidAddressException: - return + return None def mem_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: """Generator for DLLs in the order that they appear in memory""" @@ -691,7 +691,7 @@ def mem_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: ): yield entry except exceptions.InvalidAddressException: - return + return None def get_handle_count(self): try: @@ -841,11 +841,11 @@ def to_list( try: is_valid = trans_layer.is_valid(self.vol.offset) if not is_valid: - return + return None link = getattr(self, direction).dereference() except exceptions.InvalidAddressException: - return + return None if not sentinel: yield self._context.object( @@ -860,7 +860,7 @@ def to_list( obj_offset = link.vol.offset - relative_offset if not trans_layer.is_valid(obj_offset): - return + return None obj = self._context.object( symbol_type, @@ -875,7 +875,7 @@ def to_list( try: link = getattr(link, direction).dereference() except exceptions.InvalidAddressException: - return + return None def __iter__(self) -> Iterator[interfaces.objects.ObjectInterface]: return self.to_list(self.vol.parent.vol.type_name, self.vol.member_name) @@ -905,10 +905,10 @@ def get_sids(self) -> Iterable[str]: sid = sid_and_attr.Sid.dereference().cast("_SID") # catch invalid pointers (UserAndGroupCount is too high) if sid is None: - return + return None # this mimics the windows API IsValidSid if sid.Revision & 0xF != 1 or sid.SubAuthorityCount > 15: - return + return None id_auth = "" for i in sid.IdentifierAuthority.Value: id_auth = i diff --git a/volatility3/framework/symbols/windows/extensions/mft.py b/volatility3/framework/symbols/windows/extensions/mft.py index 17b6c83253..14c1f08d63 100644 --- a/volatility3/framework/symbols/windows/extensions/mft.py +++ b/volatility3/framework/symbols/windows/extensions/mft.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework import objects +from volatility3.framework import objects, constants, exceptions class MFTEntry(objects.StructType): @@ -21,3 +21,36 @@ def get_full_name(self) -> str: "string", encoding="utf16", max_length=self.NameLength * 2, errors="replace" ) return output + + +class MFTAttribute(objects.StructType): + """This represents an MFT ATTRIBUTE""" + + def get_resident_filename(self) -> str: + # To get the resident name, we jump to relative name offset and read name length * 2 bytes of data + try: + name = self._context.object( + self.vol.type_name.split(constants.BANG)[0] + constants.BANG + "string", + layer_name=self.vol.layer_name, + offset=self.vol.offset + self.Attr_Header.NameOffset, + max_length=self.Attr_Header.NameLength * 2, + errors="replace", + encoding="utf16", + ) + return name + except exceptions.InvalidAddressException: + return None + + def get_resident_filecontent(self) -> bytes: + # To get the resident content, we jump to relative content offset and read name length * 2 bytes of data + try: + bytesobj = self._context.object( + self.vol.type_name.split(constants.BANG)[0] + constants.BANG + "bytes", + layer_name=self.vol.layer_name, + offset=self.vol.offset + self.Attr_Header.ContentOffset, + native_layer_name=self.vol.native_layer_name, + length=self.Attr_Header.ContentLength, + ) + return bytesobj + except exceptions.InvalidAddressException: + return None diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index fbd3ead8e9..51be0841cf 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -162,7 +162,7 @@ def _get_subkeys_recursive( try: signature = node.cast("string", max_length=2, encoding="latin-1") except (exceptions.InvalidAddressException, RegistryFormatException): - return + return None listjump = None if signature == "ri": @@ -220,7 +220,7 @@ def get_values(self) -> Iterable[interfaces.objects.ObjectInterface]: yield node except (exceptions.InvalidAddressException, RegistryFormatException) as excp: vollog.debug(f"Invalid address in get_values iteration: {excp}") - return + return None def get_name(self) -> interfaces.objects.ObjectInterface: """Gets the name for the current key node""" diff --git a/volatility3/framework/symbols/windows/extensions/services.py b/volatility3/framework/symbols/windows/extensions/services.py index 00fb1cc4ec..e14de761d8 100644 --- a/volatility3/framework/symbols/windows/extensions/services.py +++ b/volatility3/framework/symbols/windows/extensions/services.py @@ -110,7 +110,7 @@ def traverse(self): yield rec rec = rec.ServiceList.Blink.dereference() except exceptions.InvalidAddressException: - return + return None class SERVICE_HEADER(objects.StructType): diff --git a/volatility3/framework/symbols/windows/mft.json b/volatility3/framework/symbols/windows/mft.json index e5de8f3fa7..d4f2aef7a9 100644 --- a/volatility3/framework/symbols/windows/mft.json +++ b/volatility3/framework/symbols/windows/mft.json @@ -230,21 +230,21 @@ "offset": 0, "type": { "kind": "struct", - "name": "mft!ATTR_HEADER" + "name": "ATTR_HEADER" } }, "Resident_Header": { "offset": 16, "type": { "kind": "struct", - "name": "mft!RESIDENT_HEADER" + "name": "RESIDENT_HEADER" } }, "Attr_Data": { "offset": 24, "type": { "kind": "struct", - "name": "mft!ATTR_HEADER" + "name": "ATTR_HEADER" } } }, @@ -300,10 +300,24 @@ "kind": "base", "name": "unsigned short" } + }, + "ContentLength": { + "offset": 16, + "type": { + "kind": "base", + "name": "unsigned int" + } + }, + "ContentOffset": { + "offset": 20, + "type": { + "kind": "base", + "name": "unsigned short" + } } }, "kind": "struct", - "size": 16 + "size": 24 },"RESIDENT_HEADER": { "fields": { "AttrSize": { diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index a43933ccfb..3816312cd5 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -2,13 +2,13 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -import binascii import json import logging import lzma import os import re import struct +from pathlib import PureWindowsPath from typing import Any, Dict, Generator, List, Optional, Tuple, Union from urllib import parse, request @@ -226,13 +226,12 @@ def get_guid_from_mz( return None pdb_name = debug_entry.PdbFileName.decode("utf-8").strip("\x00") + + # Let pathlib do the filename extraction. This will likely always be a Windows path though. + pdb_name = PureWindowsPath(pdb_name).name + age = debug_entry.Age - guid = "{:08x}{:04x}{:04x}{}".format( - debug_entry.Signature_Data1, - debug_entry.Signature_Data2, - debug_entry.Signature_Data3, - binascii.hexlify(debug_entry.Signature_Data4).decode("utf-8"), - ) + guid = debug_entry.Signature_String[:32] # Removes the Age from the GUID return guid, age, pdb_name @classmethod diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index 9915312e39..7f56b75f8a 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -31,13 +31,9 @@ def _generator(self): # Do mass mapping and determine the number of different layers and how many pages go to each one layer = self.context.layers[self.config["primary"]] - page_count = ( - swap_count - ) = ( - invalid_page_count - ) = ( - large_page_count - ) = large_swap_count = large_invalid_count = other_invalid = 0 + page_count = swap_count = invalid_page_count = large_page_count = ( + large_swap_count + ) = large_invalid_count = other_invalid = 0 if isinstance(layer, intel.Intel): page_addr = 0