diff --git a/API_CHANGES.md b/API_CHANGES.md new file mode 100644 index 0000000000..4a65de04be --- /dev/null +++ b/API_CHANGES.md @@ -0,0 +1,15 @@ +API Changes +=========== + +When an addition to the existing API is made, the minor version is bumped. +When an API feature or function is removed or changed, the major version is bumped. + + +1.2.0 +===== +* Added support for module collections +* Added context.modules +* Added ModuleRequirement +* Added get\_symbols\_by\_absolute\_location + + diff --git a/README.md b/README.md index f48bf728f7..9f9c1bbb70 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Volatility 3: The volatile memory extraction framework -Volatility is the world’s most widely used framework for extracting digital +Volatility is the world's most widely used framework for extracting digital artifacts from volatile memory (RAM) samples. The extraction techniques are performed completely independent of the system being investigated but offer visibility into the runtime state of the system. The framework is intended @@ -14,21 +14,34 @@ technical and performance challenges associated with the original code base that became apparent over the previous 10 years. Another benefit of the rewrite is that Volatility 3 could be released under a custom license that was more aligned with the goals of the Volatility community, -the Volatility Software License (VSL). See the [LICENSE](LICENSE.txt) file for more details. +the Volatility Software License (VSL). See the +[LICENSE](https://www.volatilityfoundation.org/license/vsl-v1.0) file for +more details. ## Requirements -- Python 3.5.3 or later. -- Pefile 2017.8.1 or later. +Volatility 3 requires Python 3.6.0 or later. To install the most minimal set of dependencies (some plugins will not work) use a command such as: -## Optional Dependencies +```shell +pip3 install -r requirements-minimal.txt +``` + +Alternately, the minimal packages will be installed automatically when Volatility 3 is installed using setup.py. However, as noted in the Quick Start section below, Volatility 3 does not *need* to be installed via setup.py prior to using it. -- yara-python 3.8.0 or later. -- capstone 3.0.0 or later. +```shell +python3 setup.py build +python3 setup.py install +``` + +To enable the full range of Volatility 3 functionality, use a command like the one below. For partial functionality, comment out any unnecessary packages in [requirements.txt](requirements.txt) prior to running the command. + +```shell +pip3 install -r requirements.txt +``` ## Downloading Volatility -The latest stable version of Volatility will always be the master branch of the GitHub repository. You can get the latest version of the code using the following command: +The latest stable version of Volatility will always be the stable branch of the GitHub repository. You can get the latest version of the code using the following command: ```shell git clone https://github.com/volatilityfoundation/volatility3.git @@ -45,7 +58,7 @@ git clone https://github.com/volatilityfoundation/volatility3.git 2. See available options: ```shell - python3 vol.py —h + python3 vol.py -h ``` 3. To get more information on a Windows memory sample and to make sure @@ -55,10 +68,10 @@ Volatility supports that sample type, run Example: ```shell - python3 vol.py —f /home/user/samples/stuxnet.vmem windows.info + python3 vol.py -f /home/user/samples/stuxnet.vmem windows.info ``` -4. Run some other plugins. The `-f` or `—-single-location` is not strictly +4. Run some other plugins. The `-f` or `--single-location` is not strictly required, but most plugins expect a single sample. Some also require/accept other options. Run `python3 vol.py -h` for more information on a particular command. @@ -91,7 +104,7 @@ The latest generated copy of the documentation can be found at: ${JSON_DIR}/${KERNEL_DIR}.json.xz -if [ $? == 0 ]; then - ${DWARF2JSON} mac --arch i386 --macho "${UNPACK_DIR}/${KERNEL_DIR}/kernel.dSYM" --macho-symbols "${UNPACK_DIRECTORY}/${KERNEL_DIR}/kernel" | xz -9 > ${JSON_DIR}/${KERNEL_DIR}.json.xz +echo "Running ${DWARF2JSON} mac --macho "${UNPACK_DIR}/${KERNEL_DIR}/kernel.dSYM" --macho-symbols "${UNPACK_DIR}/${KERNEL_DIR}/kernel" | xz -9 > ${JSON_DIR}/${KERNEL_DIR}.json.xz" +${DWARF2JSON} mac --macho "${UNPACK_DIR}/${KERNEL_DIR}/kernel.dSYM" --macho-symbols "${UNPACK_DIR}/${KERNEL_DIR}/kernel" | xz -9 > ${JSON_DIR}/${KERNEL_DIR}.json.xz +if [ $? != 0 ]; then + ${DWARF2JSON} mac --arch i386 --macho "${UNPACK_DIR}/${KERNEL_DIR}/kernel.dSYM" --macho-symbols "${UNPACK_DIR}/${KERNEL_DIR}/kernel" | xz -9 > ${JSON_DIR}/${KERNEL_DIR}.json.xz fi diff --git a/development/pdbparse-to-json.py b/development/pdbparse-to-json.py index 88c8427fb0..819e44e15d 100644 --- a/development/pdbparse-to-json.py +++ b/development/pdbparse-to-json.py @@ -27,17 +27,17 @@ def retreive_pdb(self, guid: str, file_name: str) -> Optional[str]: logger.info("Download PDB file...") file_name = ".".join(file_name.split(".")[:-1] + ['pdb']) for sym_url in ['http://msdl.microsoft.com/download/symbols']: - url = sym_url + "/{}/{}/".format(file_name, guid) + url = sym_url + f"/{file_name}/{guid}/" result = None for suffix in [file_name[:-1] + '_', file_name]: try: - logger.debug("Attempting to retrieve {}".format(url + suffix)) + logger.debug(f"Attempting to retrieve {url + suffix}") result, _ = request.urlretrieve(url + suffix) except request.HTTPError as excp: - logger.debug("Failed with {}".format(excp)) + logger.debug(f"Failed with {excp}") if result: - logger.debug("Successfully written to {}".format(result)) + logger.debug(f"Successfully written to {result}") break return result @@ -116,7 +116,7 @@ def __init__(self, filename: str): self._filename = filename logger.info("Parsing PDB...") self._pdb = pdbparse.parse(filename) - self._seen_ctypes = set([]) # type: Set[str] + self._seen_ctypes: Set[str] = set([]) def lookup_ctype(self, ctype: str) -> str: self._seen_ctypes.add(ctype) @@ -169,7 +169,7 @@ def generate_metadata(self) -> Dict[str, Any]: def read_enums(self) -> Dict: """Reads the Enumerations from the PDB file""" logger.info("Reading enums...") - output = {} # type: Dict[str, Any] + output: Dict[str, Any] = {} stream = self._pdb.STREAM_TPI for type_index in stream.types: user_type = stream.types[type_index] @@ -231,7 +231,7 @@ def read_usertypes(self) -> Dict: def _format_usertype(self, usertype, kind) -> Dict: """Produces a single usertype""" - fields = {} # type: Dict[str, Dict[str, Any]] + fields: Dict[str, Dict[str, Any]] = {} [fields.update(self._format_field(s)) for s in usertype.fieldlist.substructs] return {usertype.name: {'fields': fields, 'kind': kind, 'size': usertype.size}} @@ -257,7 +257,7 @@ def _determine_size(self, field): if output is None: import pdb pdb.set_trace() - raise ValueError("Unknown size for field: {}".format(field.name)) + raise ValueError(f"Unknown size for field: {field.name}") return output def _format_kind(self, kind): @@ -355,6 +355,6 @@ def read_basetypes(self) -> Dict: json.dump(convertor.read_pdb(), f, indent = 2, sort_keys = True) if args.keep: - print("Temporary PDB file: {}".format(filename)) + print(f"Temporary PDB file: {filename}") elif delfile: os.remove(filename) diff --git a/development/schema_validate.py b/development/schema_validate.py index ece56cdec8..0908e934f3 100644 --- a/development/schema_validate.py +++ b/development/schema_validate.py @@ -35,7 +35,7 @@ for filename in args.filenames: try: if os.path.exists(filename): - print("[?] Validating file: {}".format(filename)) + print(f"[?] Validating file: {filename}") with open(filename, 'r') as t: test = json.load(t) @@ -45,14 +45,14 @@ result = schemas.validate(test, False) if result: - print("[+] Validation successful: {}".format(filename)) + print(f"[+] Validation successful: {filename}") else: - print("[-] Validation failed: {}".format(filename)) + print(f"[-] Validation failed: {filename}") failures.append(filename) else: - print("[x] File not found: {}".format(filename)) + print(f"[x] File not found: {filename}") except Exception as e: failures.append(filename) - print("[x] Exception occurred: {} ({})".format(filename, repr(e))) + print(f"[x] Exception occurred: {filename} ({repr(e)})") print("Failures", failures) diff --git a/development/stock-linux-json.py b/development/stock-linux-json.py index 51bcc7ecd1..c863d41e49 100644 --- a/development/stock-linux-json.py +++ b/development/stock-linux-json.py @@ -30,7 +30,7 @@ def download_lists(self, keep = False): def download_list(self, urls: List[str]) -> Dict[str, str]: processed_files = {} for url in urls: - print(" - Downloading {}".format(url)) + print(f" - Downloading {url}") data = requests.get(url) with tempfile.NamedTemporaryFile() as archivedata: archivedata.write(data.content) @@ -48,14 +48,14 @@ def process_rpm(self, archivedata) -> Optional[str]: extracted = None for member in rpm.getmembers(): if 'vmlinux' in member.name or 'System.map' in member.name: - print(" - Extracting {}".format(member.name)) + print(f" - Extracting {member.name}") extracted = rpm.extractfile(member) break if not member or not extracted: return None with tempfile.NamedTemporaryFile(delete = False, prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: - print(" - Writing to {}".format(output.name)) + print(f" - Writing to {output.name}") output.write(extracted.read()) return output.name @@ -65,14 +65,14 @@ def process_deb(self, archivedata) -> Optional[str]: extracted = None for member in deb.data.tgz().getmembers(): if member.name.endswith('vmlinux') or 'System.map' in member.name: - print(" - Extracting {}".format(member.name)) + print(f" - Extracting {member.name}") extracted = deb.data.get_file(member.name) break if not member or not extracted: return None with tempfile.NamedTemporaryFile(delete = False, prefix = 'vmlinux' if 'vmlinux' in member.name else 'System.map') as output: - print(" - Writing to {}".format(output.name)) + print(f" - Writing to {output.name}") output.write(extracted.read()) return output.name @@ -81,7 +81,7 @@ def process_files(self, named_files: Dict[str, str]): print("Processing Files...") for i in named_files: if named_files[i] is None: - print("FAILURE: None encountered for {}".format(i)) + print(f"FAILURE: None encountered for {i}") return args = [DWARF2JSON, 'linux'] output_filename = 'unknown-kernel.json' @@ -91,10 +91,10 @@ def process_files(self, named_files: Dict[str, str]): prefix = '--elf' output_filename = './' + '-'.join((named_file.split('/')[-1]).split('-')[2:])[:-4] + '.json.xz' args += [prefix, named_files[named_file]] - print(" - Running {}".format(args)) + print(f" - Running {args}") proc = subprocess.run(args, capture_output = True) - print(" - Writing to {}".format(output_filename)) + print(f" - Writing to {output_filename}") with lzma.open(output_filename, 'w') as f: f.write(proc.stdout) diff --git a/doc/requirements.txt b/doc/requirements.txt new file mode 100644 index 0000000000..d646d22ce4 --- /dev/null +++ b/doc/requirements.txt @@ -0,0 +1,4 @@ +# These packages are required for building the documentation. +sphinx>=1.8.2 +sphinx_autodoc_typehints>=1.4.0 +sphinx-rtd-theme>=0.4.3 \ No newline at end of file diff --git a/doc/source/glossary.rst b/doc/source/glossary.rst new file mode 100644 index 0000000000..68bc41e4c1 --- /dev/null +++ b/doc/source/glossary.rst @@ -0,0 +1,192 @@ +Glossary +======== +There are many terms when talking about memory forensics, this list hopes to define the common ones and +provide some commonality on how to refer to particular ideas within the field. + +A +- +.. _Address: + An address is another name for an :ref:`offset`, specifically an offset within memory. Offsets can be + both relative or absolute, whereas addresses are almost always absolute. + +.. _Address Space: + +Address Space + This is the name in volatility 2 for what's referred to as a :ref:`Translation Layer`. It + encompasses all values that can be addresses, usually in reference to addresses in memory. + +.. _Alignment: + +Alignment + This value is what all data :ref:`offsets` will typically be a multiple of within a :ref:`type`. + +.. _Array: + +Array + This represents a list of items, which can be access by an index, which is zero-based (meaning the first + element has index 0). Items in arrays are almost always the same size (it is not a generic list, as in python) + even if they are :ref:`pointers` to different sized objects. + +D +- +.. _Data Layer: + +Data Layer + A group of bytes, where each byte can be addressed by a specific offset. Data layers are usually contiguous + chunks of data. + +.. _Dereference: + +Dereference + The act of taking the value of a pointer, and using it as an offset to another object, as a reference. + +.. _Domain: + +Domain + This the grouping for input values for a mapping or mathematical function. + +M +- +.. _Map: + +Map, mapping + A mapping is a relationship between two sets (where elements of the :ref:`Domain` map to elements + of the :ref:`Range`). Mappings can be seen as a mathematical function, and therefore volatility 3 + attempts to use mathematical functional notation where possible. Within volatility a mapping is most often + used to refer to the function for translating addresses from a higher layer (domain) to a lower layer (range). + For further information, please see + `Function (mathematics) in wikipedia https://en.wikipedia.org/wiki/Function_(mathematics)` + + +.. _Member: + +Member + The name of subcomponents of a type, similar to attributes of objects in common programming parlance. These + are usually recorded as :ref:`offset` and :ref:`type` pairs within a :ref:`structure`. + +O +- +.. _Object: + +Object + This has a specific meaning within computer programming (as in Object Oriented Programming), but within the world + of Volatility it is used to refer to a type that has been associated with a chunk of data, or a specific instance + of a type. See also :ref:`Type`. + +.. _Offset: + +Offset + A numeric value that identifies a distance within a group of bytes, to uniquely identify a single byte, or the + start of a run of bytes. An offset is often relative (offset from another object/item) but can be absolute (offset from + the start of a region of data). + +P +- +.. _Packed: + +Packed + Structures are often :ref:`aligned` meaning that the various members (subtypes) are always aligned at + particular values (usually multiples of 2, 4 or 8). Thus if the data used to represent a particular value has + an odd number of bytes, not a multiple of the chosen number, there will be :ref:`padding` between it and + the next member. In packed structs, no padding is used and the offset of the next member depends on the length of + the previous one. + +.. _Padding: + +Padding + Data that (usually) contains no useful information. The typical value used for padding is 0 (sometimes called + a null byte). As an example, if a string :ref:`object` that has been allocated a particular number of + bytes, actually contains fewer bytes, the rest of the data (to make up the original length) will be padded with + null (0) bytes. + +.. _Page: + +Page + A specific chunk of contiguous data. It is an organizational quantity of memory (usually 0x1000, or 4096 bytes). + Pages, like pages in a book, make up the whole, but allow for specific chunks to be allocated and used as necessary. + Operating systems uses pages as a means to have granular control over chunks of memory. This allows them to be + reordered and reused as necessary (without having to move large chunks of data around), and allows them to have + access controls placed upon them, limiting actions such as reading and writing. + +.. _Page Table: + +Page Table + A table that points to a series of :ref:`pages`. Each page table is typically the size of a single page, + and page tables can point to pages that are in fact other page tables. Using tables that point to tables, it's + possible to use them as a way to map a particular address within a (potentially larger, but sparsely populated) + virtual space to a concrete (and usually contiguous) physical space, through the process of :ref:`mapping`. + +.. _Pointer: + +Pointer + A value within memory that points to a different area of memory. This allows objects to contain references to + other objects without containing all the data of the other object. Following a pointer is known as :ref:`dereferencing` + a pointer. Pointers are usually the same length as the maximum address of the address space, since they + should be able to point to any address within the space. + +R +- +.. _Range: + +Range + This is the set of the possible output values for a mapping or mathematical function. + +S +- +.. _Struct: + +Struct, Structure + A means of containing multiple different :ref:`type` associated together. A struct typically contains + other :ref:`type`, usually :ref:`aligned` (unless :ref:`packing` is involved). In this way + the :ref:`members` of a type can be accessed by finding the data at the relative :ref:`offset` to + the start of the structure. + +.. _Symbol: + +Symbol + This is used in many different contexts, as a short term for many things. Within Volatility, a symbol is a + construct that usually encompasses a specific type :ref:`type` at a specfific :ref:`offset`, + representing a particular instance of that type within the memory of a compiled and running program. An example + would be the location in memory of a list of active tcp endpoints maintained by the networking stack + within an operating system. + +T +- +.. _Template: + +Template + Within volatility 3, the term template applies to a :ref:`type` that has not yet been instantiated or linked + to any data or a specific location within memory. Once a type has been tied to a particular chunk of data, it is + called an :ref:`object`. + +.. _Translation Layer: + +Translation Layer + This is a type of data layer which allows accessing data from lower layers using addresses different to those + used by the lower layers themselves. When accessing data in a translation layer, it translates (or :ref:`maps`) + addresses from its own :ref:`address space
` to the address space of the lower layer and returns the + corresponding data from the lower layer. Note that multiple addresses in the higher layer might refer to the same + address in the lower layer. Conversely, some addresses in the higher layer might have no corresponding address in the + lower layer at all. Translation layers most commonly handle the translation from virtual to physical addresses, + but can be used to translate data to and from a compressed form or translate data from a particular file format + into another format. + +.. _Type: + +Type + This is a structure definition of multiple elements that expresses how data is laid out. Basic types define how + the data should be interpreted in terms of a run of bits (or more commonly a collection of 8 bits at a time, + called bytes). New types can be constructed by combining other types at specific relative offsets, forming something + called a :ref:`struct`, or by repeating the same type, known as an :ref:`array`. They can even + contain other types at the same offset depending on the data itself, known as :ref:`Unions`. Once a type + has been linked to a specific chunk of data, the result is referred to as an :ref:`object`. + +U +- +.. _Union: + +Union + A union is a type that can hold multiple different subtypes, whose relative offsets specifically overlap. + A union is a means for holding multiple different types within the same size of data, the relative offsets of the + types within the union specifically overlap. This means that the data in a union object is interpreted differently + based on the types of the union used to access it. diff --git a/doc/source/index.rst b/doc/source/index.rst index f8cdb690fd..0dc3b50258 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -17,6 +17,8 @@ Here are some guidelines for using Volatility 3 effectively: complex-plugin using-as-a-library symbol-tables + volshell + glossary Python Packages =============== diff --git a/doc/source/simple-plugin.rst b/doc/source/simple-plugin.rst index 52dabfd4fb..4e499b1869 100644 --- a/doc/source/simple-plugin.rst +++ b/doc/source/simple-plugin.rst @@ -147,7 +147,7 @@ it does not. The :py:func:`~volatility3.plugins.windows.pslist.PsList.create_pi identifiers that are included in the list. If the list is empty, all processes are returned. The next line specifies the columns by their name and type. The types are simple types (int, str, bytes, float, and bool) -but can also provide hints as to how the output should be displayed (such as a hexidecimal number, using +but can also provide hints as to how the output should be displayed (such as a hexadecimal number, using :py:class:`volatility3.framework.renderers.format_hints.Hex`). This indicates to user interfaces that the value should be displayed in a particular way, but does not guarantee that the value will be displayed that way (for example, if it doesn't make sense to do so in a particular interface). diff --git a/doc/source/symbol-tables.rst b/doc/source/symbol-tables.rst index fd94dea4d4..31329b9ebc 100644 --- a/doc/source/symbol-tables.rst +++ b/doc/source/symbol-tables.rst @@ -47,7 +47,32 @@ banner), which Volatility's automagic can detect. Volatility caches the mapping tables they come from, meaning the precise file names don't matter and can be organized under any necessary hierarchy under the operating system directory. -Linux and Mac symbol tables can be generated from a DWARF file using a tool called `dwarf2json `_. Currently a kernel -with debugging symbols is the only suitable means for recovering all the information required by most Volatility plugins. +Linux and Mac symbol tables can be generated from a DWARF file using a tool called `dwarf2json `_. +Currently a kernel with debugging symbols is the only suitable means for recovering all the information required by +most Volatility plugins. Note that in most linux distributions, the standard kernel is stripped of debugging information +and the kernel with debugging information is stored in a package that must be acquired separately. + +A generic table isn't guaranteed to produce accurate results, and would reduce the number of structures +that all plugins could rely on. As such, and because linux kernels with different configurations can produce different structures, +volatility 3 requires that the banners in the JSON file match the banners found in the image *exactly*, not just the version +number. This can include elements such as the compilation time and even the version of gcc used for the compilation. +The exact match is required to ensure that the results volatility returns are accurate, therefore there is no simple means +provided to get the wrong JSON ISF file to easily match. + +To determine the string for a particular memory image, use the `banners` plugin. Once the specific banner is known, +try to locate that exact kernel debugging package for the operating system. Unfortunately each distribution provides +its debugging packages under different package names and there are so many that the distribution may not keep all old +versions of the debugging symbols, and therefore **it may not be possible to find the right symbols to analyze a linux +memory image with volatlity**. With Macs there are far fewer kernels and only one distribution, making it easier to +ensure that the right symbols can be found. + Once a kernel with debugging symbols/appropriate DWARF file has been located, `dwarf2json `_ will convert it into an -appropriate JSON file. +appropriate JSON file. Example code for automatically creating a JSON from URLs for the kernel debugging package and +the package containing the System.map, can be found in `stock-linux-json.py `_ . +The System.map file is recommended for completeness, but a kernel with debugging information often contains the same +symbol offsets within the DWARF data, which dwarf2json can extract into the JSON ISF file. + +The banners available for volatility to use can be found using the `isfinfo` plugin, but this will potentially take a +long time to run depending on the number of JSON files available. This will list all the JSON (ISF) files that +volatility3 is aware of, and for linux/mac systems what banner string they search for. For volatility to use the JSON +file, the banners must match exactly (down to the compilation date). diff --git a/doc/source/using-as-a-library.rst b/doc/source/using-as-a-library.rst index ded861de45..95d2b10808 100644 --- a/doc/source/using-as-a-library.rst +++ b/doc/source/using-as-a-library.rst @@ -161,9 +161,16 @@ If unsatisfied is an empty list, then the plugin has been given everything it re Dictionary of the hierarchy paths and their associated requirements that weren't satisfied. The plugin can then be instantiated with the context (containing the plugin's configuration) and the path that the -plugin can find its configuration at. A progress_callback can also be provided to give users feedback whilst the -plugin is running. Also, should the plugin produce files, an open_method can be set on the plugin, which will -be called whenever a plugin produces an auxiliary file. +plugin can find its configuration at. This configuration path only needs to be a unique value to identify where the +configuration details can be found, similar to a registry key in Windows. + +A progress_callback can also be provided to give users feedback whilst the plugin is running. A progress callback +is a function (callable) that takes a percentage and a descriptive string. User interfaces implementing these can +therefore provide progress feedback to a user, as the framework will call these every so often during intensive actions, +to update the user as to how much has been completed so far. + +Also, should the plugin produce files, an open_method can be set on the plugin, which will be called whenever a plugin +produces an auxiliary file. :: @@ -171,7 +178,11 @@ be called whenever a plugin produces an auxiliary file. constructed.set_open_method(file_handler) The file_handler must adhere to the :py:class:`~volatility3.framework.interfaces.plugins.FileHandlerInterface`, -which represents an IO[bytes] object but also contains a `preferred_filename` attribute as a hint. +which represents an IO[bytes] object but also contains a `preferred_filename` attribute as a hint indicating what the +file being produced should be called. When a plugin produces a new file, rather than opening it with the python `open` +method, it will use the `FileHandlerInterface` and construct it with a descriptive filename, and then write bytes to it +using the `write` method, just like other python file-like objects. This allows web user interfaces to offer the files +for download, whilst CLIs to write them to disk and other UIs to handle files however they need. All of this functionality has been condensed into a framework method called `construct_plugin` which will take and run the automagics, and instantiate the plugin on the provided `base_config_path`. It also diff --git a/doc/source/vol-cli.rst b/doc/source/vol-cli.rst index 0699068730..9db29c8186 100644 --- a/doc/source/vol-cli.rst +++ b/doc/source/vol-cli.rst @@ -116,7 +116,7 @@ Options **** The name of the plugin to execute (these are usually categorized by - the operating system, such as `windows.pslist.PsList`). Any subtring + the operating system, such as `windows.pslist.PsList`). Any substring that uniquely matches the desired plugin name can be used. As such `hivescan` would match `windows.registry.hivescan.HiveScan`, but `pslist` is ambiguous because it could match `windows.pslist` or diff --git a/doc/source/vol2to3.rst b/doc/source/vol2to3.rst index bc1733dcf8..eb33b6618f 100644 --- a/doc/source/vol2to3.rst +++ b/doc/source/vol2to3.rst @@ -24,7 +24,7 @@ Object Model changes -------------------- The object model has changed as well, objects now inherit directly from their Python counterparts, meaning an integer -object is actually a Python integer (and has all the associated methods, and can be used whereever a normal int could). +object is actually a Python integer (and has all the associated methods, and can be used wherever a normal int could). In Volatility 2, a complex proxy object was constructed which tried to emulate all the methods of the host object, but ultimately it was a different type and could not be used in the same places (critically, it could make the ordering of operations important, since a + b might not work, but b + a might work fine). diff --git a/doc/source/volshell.rst b/doc/source/volshell.rst new file mode 100644 index 0000000000..1b6846e6ba --- /dev/null +++ b/doc/source/volshell.rst @@ -0,0 +1,191 @@ +Volshell - A CLI tool for working with memory +============================================= + +Volshell is a utility to access the volatility framework interactively with a specific memory image. It allows for +direct introspection and access to all features of the volatility library from within a command line environment. + +Starting volshell +----------------- + +Volshell is started in much the same way as volatility. Rather than providing a plugin, you just specify the file. +If the operating system of the memory image is known, a flag can be provided allowing additional methods for the +specific operating system. + +:: + + $ volshell.py -f [-w|-m|-l] + +The flags to specify a known operating system are -w for windows, -m for mac and -l for linux. Volshell will run +through the usual automagic, trying to load the memory image. If no operating system is specified, all automagic will +be run. + +When volshell starts, it will show the version of volshell, a brief message indicating how to get more help, the current +operating system mode for volshell, and the current layer available for use. + +.. code-block:: python + + Volshell (Volatility 3 Framework) 1.0.1 + Readline imported successfully PDB scanning finished + + Call help() to see available functions + + Volshell mode: Generic + Current Layer: primary + + (primary) >>> + +Volshell itself in essentially a plugin, but an interactive one. As such, most values are accessed through `self` +although there is also a `context` object whenever a context must be provided. + +The prompt for the tool will indicate the name of the current layer (which can be accessed as `self.current_layer` +from within the tool). + +The generic mode is quite limited, won't have any symbols loaded and therefore won't be able to display much +information. When an operating system is chosen, the appropriate symbols should be loaded and additional functions +become available. The mode cannot easily be changed once the tool has started. + +Accessing objects +----------------- +All operating systems come with their equivalent of a process list, aliased to the function `ps()`. Running this +will provide a list of volatility objects, based on the operating system in question. We will use these objects to +run our examples against. + +We'll start by creating a process variable, and putting the first result from `ps()` in it. Since the shell is a +python environment, we can do the following: + +.. code-block:: python + + (primary) >>> proc = ps()[0] + (primary) >>> proc + + +When printing a volatility structure, various information is output, in this case the `type_name`, the `layer` and +`offset` that it's been constructed on, and the size of the structure. + +We can directly access the volatility information about a structure, using the `.vol` attribute, which contains +basic information such as structure size, type_name, and the list of members amongst others. However, volshell has a +built-in mechanism for providing more information about a structure, called `display_type` or `dt`. This can be given +either a type name (which if not prefixed with symbol table name, will use the kernel symbol table identified by the +automagic). + +.. code-block:: python + + (primary) >>> dt('_EPROCESS') + nt_symbols1!_EPROCESS (2624 bytes) + 0x0 : Pcb nt_symbols1!_KPROCESS + 0x438 : ProcessLock nt_symbols1!_EX_PUSH_LOCK + 0x440 : UniqueProcessId nt_symbols1!pointer + 0x448 : ActiveProcessLinks nt_symbols1!_LIST_ENTRY + ... + +It can also be provided with an object and will interpret the data for each in the process: + +.. code-block:: python + + (primary) >>> dt(proc) + nt_symbols1!_EPROCESS (2624 bytes) + 0x0 : Pcb nt_symbols1!_KPROCESS 0x8c0bccf8d040 + 0x438 : ProcessLock nt_symbols1!_EX_PUSH_LOCK 0x8c0bccf8d478 + 0x440 : UniqueProcessId nt_symbols1!pointer 356 + 0x448 : ActiveProcessLinks nt_symbols1!_LIST_ENTRY 0x8c0bccf8d488 + ... + +These values can be accessed directory as attributes + +.. code-block:: python + + (primary) >>> proc.UniqueProcessId + 356 + +Pointer structures contain the value they point to, but attributes accessed are forwarded to the object they point to. +This means that pointers do not need to be explicitly dereferenced to access underling objects. + +.. code-block:: python + + (primary) >>> proc.Pcb.DirectoryTableBase + 4355817472 + +Running plugins +--------------- + +It's possible to run any plugin by importing it appropriately and passing it to the `display_plugin_ouptut` or `dpo` +method. In the following example we'll provide no additional parameters. Volatility will show us which parameters +were required: + +.. code-block:: python + + (primary) >>> from volatility3.plugins.windows import pslist + (primary) >>> display_plugin_output(pslist.PsList) + Unable to validate the plugin requirements: ['plugins.Volshell.9QZLXJKFWESI0BAP3M1U7Y5VCT468GRN.PsList.primary', 'plugins.Volshell.9QZLXJKFWESI0BAP3M1U7Y5VCT468GRN.PsList.nt_symbols'] + +We can see that it's made a temporary configuration path for the plugin, and that neither `primary` nor `nt_symbols` +was fulfilled. + +We can see all the options that the plugin can accept by access the `get_requirements()` method of the plugin. +This is a classmethod, so can be called on an uninstantiated copy of the plugin. + +.. code-block:: python + + (primary) >>> pslist.PsList.get_requirements() + [, , , , ] + +We can provide arguments via the `dpo` method call: + +.. code-block:: python + + (primary) >>> display_plugin_output(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) + + PID PPID ImageFileName Offset(V) Threads Handles SessionId Wow64 CreateTime ExitTime File output + + 4 0 System 0x8c0bcac87040 143 - N/A False 2021-03-13 17:25:33.000000 N/A Disabled + 92 4 Registry 0x8c0bcac5d080 4 - N/A False 2021-03-13 17:25:28.000000 N/A Disabled + 356 4 smss.exe 0x8c0bccf8d040 3 - N/A False 2021-03-13 17:25:33.000000 N/A Disabled + ... + +Here's we've provided the current layer as the TranslationLayerRequirement, and used the symbol tables requirement +requested by the volshell plugin itself. A different table could be loaded and provided instead. The context used +by the `dpo` method is always `context`. + +Instead of print the results directly to screen, they can be gathered into a TreeGrid objects for direct access by +using the `generate_treegrid` or `gt` command. + +.. code-block:: python + + (primary) >>> treegrid = gt(pslist.PsList, primary = self.current_layer, nt_symbols = self.config['nt_symbols']) + (primary) >>> treegrid.populate() + +Treegrids must be populated before the data in them can be accessed. This is where the plugin actually runs and +produces data. + + +Running scripts +--------------- + +It might be beneficial to code up a small snippet of code, and execute that on a memory image, rather than writing +a full plugin. + +The snippet should be lines that will be executed within the volshell context (as such they can immediately access +`self` and `context`, for example). These can be executed using the `run_script` or `rs` command, or by providing the +file on the command line with `--script`. + +For example, to load a layer and extract bytes from a particular offset into a new file, the following snippet could be +used: + +.. code-block:: python + + import volatility3.framework.layers.mynewlayer as mynewlayer + + layer = cc(mynewlayer.MyNewLayer, on_top_of = 'primary', other_parameter = 'important') + with open('output.dmp', 'wb') as fp: + for i in range(0, 1073741824, 0x1000): + data = layer.read(i, 0x1000, pad = True) + fp.write(data) + +As this demonstrates, all of the python is accessible, as are the volshell built in functions (such as `cc` which +creates a constructable, like a layer or a symbol table). + +Loading files +------------- + +Files can be loaded as physical layers using the `load_file` or `lf` command, which takes a filename or a URI. This will be added +to `context.layers` and can be accessed by the name returned by `lf`. diff --git a/requirements-minimal.txt b/requirements-minimal.txt new file mode 100644 index 0000000000..31ac028148 --- /dev/null +++ b/requirements-minimal.txt @@ -0,0 +1,2 @@ +# These packages are required for core functionality. +pefile>=2017.8.1 #foo \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000000..290d9ca977 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,25 @@ +# The following packages are required for core functionality. +pefile>=2017.8.1 + +# The following packages are optional. +# If certain packages are not necessary, place a comment (#) at the start of the line. + +# This is required for the yara plugins +yara-python>=3.8.0 + +# This is required for several plugins that perform malware analysis and disassemble code. +# It can also improve accuracy of Windows 8 and later memory samples. +capstone>=3.0.5 + +# This is required by plugins that decrypt passwords, password hashes, etc. +pycryptodome + +# This can improve error messages regarding improperly configured ISF files. +jsonschema>=2.3.0 + +# This is required for memory acquisition via leechcore/pcileech. +leechcorepyc>=2.4.0 + +# This is required for analyzing Linux samples compressed using AVMLs native +# compression format. It is not required for AVML's standard LiME compression. +python-snappy==0.6.0 diff --git a/setup.py b/setup.py index ea0ddb8ddf..f6bb687f24 100644 --- a/setup.py +++ b/setup.py @@ -6,9 +6,19 @@ from volatility3.framework import constants -with open("README.md", "r", encoding="utf-8") as fh: +with open("README.md", "r", encoding = "utf-8") as fh: long_description = fh.read() +def get_install_requires(): + requirements = [] + with open("requirements-minimal.txt", "r", encoding="utf-8") as fh: + for line in fh.readlines(): + stripped_line = line.strip() + if stripped_line == "" or stripped_line.startswith("#"): + continue + requirements.append(stripped_line) + return requirements + setuptools.setup(name = "volatility3", description = "Memory forensics framework", version = constants.PACKAGE_VERSION, @@ -24,7 +34,7 @@ "Documentation": "https://volatility3.readthedocs.io/", "Source Code": "https://github.com/volatilityfoundation/volatility3", }, - python_requires = '>=3.5.3', + python_requires = '>=3.6.0', include_package_data = True, exclude_package_data = { '': ['development', 'development.*'], @@ -37,11 +47,4 @@ 'volshell = volatility3.cli.volshell:main', ], }, - install_requires = ["pefile"], - extras_require = { - 'jsonschema': ["jsonschema>=2.3.0"], - 'yara': ["yara-python>=3.8.0"], - 'crypto': ["pycryptodome>=3"], - 'disasm': ["capstone;platform_system=='Linux'", "capstone-windows;platform_system=='Windows'"], - 'doc': ["sphinx>=1.8.2", "sphinx_autodoc_typehints>=1.4.0", "sphinx-rtd-theme>=0.4.3"], - }) + install_requires = get_install_requires()) diff --git a/volatility3/__init__.py b/volatility3/__init__.py index 5d3c34e436..db52aa9b0e 100644 --- a/volatility3/__init__.py +++ b/volatility3/__init__.py @@ -43,7 +43,7 @@ def find_spec(fullname: str, path: Optional[List[str]], target: None = None, **k raise Warning(warning) -warning_find_spec = [WarningFindSpec()] # type: List[abc.MetaPathFinder] +warning_find_spec: List[abc.MetaPathFinder] = [WarningFindSpec()] sys.meta_path = warning_find_spec + sys.meta_path # We point the volatility3.plugins __path__ variable at BOTH diff --git a/volatility3/cli/__init__.py b/volatility3/cli/__init__.py index 27d85a22b2..608fdf79cf 100644 --- a/volatility3/cli/__init__.py +++ b/volatility3/cli/__init__.py @@ -32,7 +32,8 @@ # Make sure we log everything -vollog = logging.getLogger() +rootlog = logging.getLogger() +vollog = logging.getLogger(__name__) console = logging.StreamHandler() console.setLevel(logging.WARNING) formatter = logging.Formatter('%(levelname)-8s %(name)-12s: %(message)s') @@ -55,7 +56,7 @@ def __call__(self, progress: Union[int, float], description: str = None): Args: progress: Percentage of progress of the current procedure """ - message = "\rProgress: {0: 7.2f}\t\t{1:}".format(round(progress, 2), description or '') + message = f"\rProgress: {round(progress, 2): 7.2f}\t\t{description or ''}" message_len = len(message) self._max_message_len = max([self._max_message_len, message_len]) sys.stderr.write(message + (' ' * (self._max_message_len - message_len)) + '\r') @@ -80,14 +81,14 @@ def __init__(self): @classmethod def setup_logging(cls): # Delay the setting of vollog for those that want to import volatility3.cli (issue #241) - vollog.setLevel(1) - vollog.addHandler(console) + rootlog.setLevel(1) + rootlog.addHandler(console) def run(self): """Executes the command line module, taking the system arguments, determining the plugin to run and then running it.""" - volatility3.framework.require_interface_version(1, 0, 0) + volatility3.framework.require_interface_version(2, 0, 0) renderers = dict([(x.name.lower(), x) for x in framework.class_subclasses(text_renderer.CLIRenderer)]) @@ -143,7 +144,7 @@ def run(self): parser.add_argument("-r", "--renderer", metavar = 'RENDERER', - help = "Determines how to render the output ({})".format(", ".join(list(renderers))), + help = f"Determines how to render the output ({', '.join(list(renderers))})", default = "quick", choices = list(renderers)) parser.add_argument("-f", @@ -160,6 +161,14 @@ def run(self): help = "Clears out all short-term cached items", default = False, action = 'store_true') + parser.add_argument("--cache-path", + help = f"Change the default path ({constants.CACHE_PATH}) used to store the cache", + default = constants.CACHE_PATH, + type = str) + parser.add_argument("--offline", + help = "Do not search online for additional JSON files", + default = False, + action = 'store_true') # We have to filter out help, otherwise parse_known_args will trigger the help message before having # processed the plugin choice or had the plugin subparser added. @@ -169,7 +178,7 @@ def run(self): banner_output = sys.stdout if renderers[partial_args.renderer].structured_output: banner_output = sys.stderr - banner_output.write("Volatility 3 Framework {}\n".format(constants.PACKAGE_VERSION)) + banner_output.write(f"Volatility 3 Framework {constants.PACKAGE_VERSION}\n") if partial_args.plugin_dirs: volatility3.plugins.__path__ = [os.path.abspath(p) @@ -179,21 +188,26 @@ def run(self): volatility3.symbols.__path__ = [os.path.abspath(p) for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS + if partial_args.cache_path: + constants.CACHE_PATH = partial_args.cache_path + if partial_args.log: file_logger = logging.FileHandler(partial_args.log) file_logger.setLevel(1) file_formatter = logging.Formatter(datefmt = '%y-%m-%d %H:%M:%S', fmt = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s') file_logger.setFormatter(file_formatter) - vollog.addHandler(file_logger) + rootlog.addHandler(file_logger) vollog.info("Logging started") if partial_args.verbosity < 3: + if partial_args.verbosity < 1: + sys.tracebacklimit = None console.setLevel(30 - (partial_args.verbosity * 10)) else: console.setLevel(10 - (partial_args.verbosity - 2)) - vollog.info("Volatility plugins path: {}".format(volatility3.plugins.__path__)) - vollog.info("Volatility symbols path: {}".format(volatility3.symbols.__path__)) + vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") + vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") # Set the PARALLELISM if partial_args.parallelism == 'processes': @@ -206,6 +220,9 @@ def run(self): if partial_args.clear_cache: framework.clear_cache() + if partial_args.offline: + constants.OFFLINE = partial_args.offline + # Do the initialization ctx = contexts.Context() # Construct a blank context failures = framework.import_files(volatility3.plugins, @@ -246,7 +263,7 @@ def run(self): if args.plugin is None: parser.error("Please select a plugin to run") - vollog.log(constants.LOGLEVEL_VVV, "Cache directory used: {}".format(constants.CACHE_PATH)) + vollog.log(constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}") plugin = plugin_list[args.plugin] chosen_configurables_list[args.plugin] = plugin @@ -258,12 +275,11 @@ def run(self): # NOTE: This will *BREAK* if LayerStacker, or the automagic configuration system, changes at all ### if args.file: - file_name = os.path.abspath(args.file) - if not os.path.exists(file_name): - vollog.log(logging.INFO, "File does not exist: {}".format(file_name)) - else: - single_location = "file:" + request.pathname2url(file_name) + try: + single_location = self.location_from_file(args.file) ctx.config['automagic.LayerStacker.single_location'] = single_location + except ValueError as excp: + parser.error(str(excp)) # UI fills in the config, here we load it from the config file and do it before we process the CL parameters if args.config: @@ -280,7 +296,7 @@ def run(self): ctx.config['automagic.LayerStacker.stackers'] = stacker.choose_os_stackers(plugin) self.output_dir = args.output_dir if not os.path.exists(self.output_dir): - parser.error("The output directory specified does not exist: {}".format(self.output_dir)) + parser.error(f"The output directory specified does not exist: {self.output_dir}") self.populate_config(ctx, chosen_configurables_list, args, plugin_config_path) @@ -309,7 +325,7 @@ def run(self): json.dump(dict(constructed.build_configuration()), f, sort_keys = True, indent = 2) except exceptions.UnsatisfiedException as excp: self.process_unsatisfied_exceptions(excp) - parser.exit(1, "Unable to validate the plugin requirements: {}\n".format([x for x in excp.unsatisfied])) + parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") try: # Construct and run the plugin @@ -318,6 +334,28 @@ def run(self): except (exceptions.VolatilityException) as excp: self.process_exceptions(excp) + @classmethod + def location_from_file(cls, filename: str) -> str: + """Returns the URL location from a file parameter (which may be a URL) + + Args: + filename: The path to the file (either an absolute, relative, or URL path) + + Returns: + The URL for the location of the file + """ + # We want to work in URLs, but we need to accept absolute and relative files (including on windows) + single_location = parse.urlparse(filename, '') + if single_location.scheme == '' or len(single_location.scheme) == 1: + single_location = parse.urlparse(parse.urljoin('file:', request.pathname2url(os.path.abspath(filename)))) + if single_location.scheme == 'file': + if not os.path.exists(request.url2pathname(single_location.path)): + filename = request.url2pathname(single_location.path) + if not filename: + raise ValueError("File URL looks incorrect (potentially missing /)") + raise ValueError(f"File does not exist: {filename}") + return parse.urlunparse(single_location) + def process_exceptions(self, excp): """Provide useful feedback if an exception occurs during a run of a plugin.""" # Ensure there's nothing in the cache @@ -332,20 +370,20 @@ def process_exceptions(self, excp): if isinstance(excp, exceptions.InvalidAddressException): general = "Volatility was unable to read a requested page:" if isinstance(excp, exceptions.SwappedInvalidAddressException): - detail = "Swap error {} in layer {} ({})".format(hex(excp.invalid_address), excp.layer_name, excp) + detail = f"Swap error {hex(excp.invalid_address)} in layer {excp.layer_name} ({excp})" caused_by = [ "No suitable swap file having been provided (locate and provide the correct swap file)", "An intentionally invalid page (operating system protection)" ] elif isinstance(excp, exceptions.PagedInvalidAddressException): - detail = "Page error {} in layer {} ({})".format(hex(excp.invalid_address), excp.layer_name, excp) + detail = f"Page error {hex(excp.invalid_address)} in layer {excp.layer_name} ({excp})" caused_by = [ "Memory smear during acquisition (try re-acquiring if possible)", "An intentionally invalid page lookup (operating system protection)", "A bug in the plugin/volatility3 (re-run with -vvv and file a bug)" ] else: - detail = "{} in layer {} ({})".format(hex(excp.invalid_address), excp.layer_name, excp) + detail = f"{hex(excp.invalid_address)} in layer {excp.layer_name} ({excp})" caused_by = [ "The base memory file being incomplete (try re-acquiring if possible)", "Memory smear during acquisition (try re-acquiring if possible)", @@ -354,7 +392,7 @@ def process_exceptions(self, excp): ] elif isinstance(excp, exceptions.SymbolError): general = "Volatility experienced a symbol-related issue:" - detail = "{}{}{}: {}".format(excp.table_name, constants.BANG, excp.symbol_name, excp) + detail = f"{excp.table_name}{constants.BANG}{excp.symbol_name}: {excp}" caused_by = [ "An invalid symbol table", "A plugin requesting a bad symbol", @@ -362,32 +400,32 @@ def process_exceptions(self, excp): ] elif isinstance(excp, exceptions.SymbolSpaceError): general = "Volatility experienced an issue related to a symbol table:" - detail = "{}".format(excp) + detail = f"{excp}" caused_by = [ "An invalid symbol table", "A plugin requesting a bad symbol", "A plugin requesting a symbol from the wrong table" ] elif isinstance(excp, exceptions.LayerException): - general = "Volatility experienced a layer-related issue: {}".format(excp.layer_name) - detail = "{}".format(excp) + general = f"Volatility experienced a layer-related issue: {excp.layer_name}" + detail = f"{excp}" caused_by = ["A faulty layer implementation (re-run with -vvv and file a bug)"] elif isinstance(excp, exceptions.MissingModuleException): - general = "Volatility could not import a necessary module: {}".format(excp.module) - detail = "{}".format(excp) + general = f"Volatility could not import a necessary module: {excp.module}" + detail = f"{excp}" caused_by = ["A required python module is not installed (install the module and re-run)"] else: general = "Volatilty encountered an unexpected situation." detail = "" caused_by = [ - "Please re-run using with -vvv and file a bug with the output", "at {}".format(constants.BUG_URL) + "Please re-run using with -vvv and file a bug with the output", f"at {constants.BUG_URL}" ] # Code that actually renders the exception output = sys.stderr - output.write(general + "\n") - output.write(detail + "\n\n") + output.write(f"{general}\n") + output.write(f"{detail}\n\n") for cause in caused_by: - output.write("\t* " + cause + "\n") + output.write(f" * {cause}\n") output.write("\nNo further results will be produced\n") sys.exit(1) @@ -403,7 +441,7 @@ def process_unsatisfied_exceptions(self, excp): symbols_failed = symbols_failed or isinstance(excp.unsatisfied[config_path], configuration.requirements.SymbolTableRequirement) - print("Unsatisfied requirement {}: {}".format(config_path, excp.unsatisfied[config_path].description)) + print(f"Unsatisfied requirement {config_path}: {excp.unsatisfied[config_path].description}") if symbols_failed: print("\nA symbol table requirement was not fulfilled. Please verify that:\n" @@ -440,8 +478,8 @@ def populate_config(self, context: interfaces.context.ContextInterface, if not scheme or len(scheme) <= 1: if not os.path.exists(value): raise FileNotFoundError( - "Non-existant file {} passed to URIRequirement".format(value)) - value = "file://" + request.pathname2url(os.path.abspath(value)) + f"Non-existent file {value} passed to URIRequirement") + value = f"file://{request.pathname2url(os.path.abspath(value))}" if isinstance(requirement, requirements.ListRequirement): if not isinstance(value, list): raise TypeError("Configuration for ListRequirement was not a list: {}".format( @@ -468,11 +506,11 @@ def _get_final_filename(self): pref_name_array = self.preferred_filename.split('.') filename, extension = os.path.join(output_dir, '.'.join(pref_name_array[:-1])), pref_name_array[-1] - output_filename = "{}.{}".format(filename, extension) + output_filename = f"{filename}.{extension}" counter = 1 while os.path.exists(output_filename): - output_filename = "{}-{}.{}".format(filename, counter, extension) + output_filename = f"{filename}-{counter}.{extension}" counter += 1 return output_filename @@ -494,7 +532,7 @@ def close(self): with open(output_filename, "wb") as current_file: current_file.write(self.read()) self._committed = True - vollog.log(logging.INFO, "Saved stored plugin file: {}".format(output_filename)) + vollog.log(logging.INFO, f"Saved stored plugin file: {output_filename}") super().close() @@ -547,12 +585,12 @@ def populate_requirements_argparse(self, parser: Union[argparse.ArgumentParser, configurable: The plugin object to pull the requirements from """ if not issubclass(configurable, interfaces.configuration.ConfigurableInterface): - raise TypeError("Expected ConfigurableInterface type, not: {}".format(type(configurable))) + raise TypeError(f"Expected ConfigurableInterface type, not: {type(configurable)}") # Construct an argparse group for requirement in configurable.get_requirements(): - additional = {} # type: Dict[str, Any] + additional: Dict[str, Any] = {} if not isinstance(requirement, interfaces.configuration.RequirementInterface): raise TypeError("Plugin contains requirements that are not RequirementInterfaces: {}".format( configurable.__name__)) diff --git a/volatility3/cli/text_renderer.py b/volatility3/cli/text_renderer.py index f40a5d3239..35b2468e85 100644 --- a/volatility3/cli/text_renderer.py +++ b/volatility3/cli/text_renderer.py @@ -33,13 +33,13 @@ def hex_bytes_as_text(value: bytes) -> str: A text representation of the hexadecimal bytes plus their ascii equivalents, separated by newline characters """ if not isinstance(value, bytes): - raise TypeError("hex_bytes_as_text takes bytes not: {}".format(type(value))) + raise TypeError(f"hex_bytes_as_text takes bytes not: {type(value)}") ascii = [] hex = [] count = 0 output = "" for byte in value: - hex.append("{:02x}".format(byte)) + hex.append(f"{byte:02x}") ascii.append(chr(byte) if 0x20 < byte <= 0x7E else ".") if (count % 8) == 7: output += "\n" @@ -87,10 +87,10 @@ def wrapped(x: Any) -> str: if result == "-" or result == "N/A": return "" if isinstance(x, format_hints.MultiTypeData) and x.converted_int: - return "{}".format(result) + return f"{result}" if isinstance(x, int) and not isinstance(x, (format_hints.Hex, format_hints.Bin)): - return "{}".format(result) - return "\"{}\"".format(result) + return f"{result}" + return f"\"{result}\"" return wrapped @@ -115,7 +115,7 @@ def display_disassembly(disasm: interfaces.renderers.Disassembly) -> str: output = "" if disasm.architecture is not None: for i in disasm_types[disasm.architecture].disasm(disasm.data, disasm.offset): - output += "\n0x%x:\t%s\t%s" % (i.address, i.mnemonic, i.op_str) + output += f"\n0x{i.address:x}:\t{i.mnemonic}\t{i.op_str}" return output return QuickTextRenderer._type_renderers[bytes](disasm.data) @@ -128,14 +128,14 @@ class CLIRenderer(interfaces.renderers.Renderer): class QuickTextRenderer(CLIRenderer): _type_renderers = { - format_hints.Bin: optional(lambda x: "0b{:b}".format(x)), - format_hints.Hex: optional(lambda x: "0x{:x}".format(x)), + format_hints.Bin: optional(lambda x: f"0b{x:b}"), + format_hints.Hex: optional(lambda x: f"0x{x:x}"), format_hints.HexBytes: optional(hex_bytes_as_text), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), interfaces.renderers.Disassembly: optional(display_disassembly), - bytes: optional(lambda x: " ".join(["{0:02x}".format(b) for b in x])), + bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), datetime.datetime: optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), - 'default': optional(lambda x: "{}".format(x)) + 'default': optional(lambda x: f"{x}") } name = "quick" @@ -158,7 +158,7 @@ def render(self, grid: interfaces.renderers.TreeGrid) -> None: line = [] for column in grid.columns: # Ignore the type because namedtuples don't realize they have accessible attributes - line.append("{}".format(column.name)) + line.append(f"{column.name}") outfd.write("\n{}\n".format("\t".join(line))) def visitor(node: interfaces.renderers.TreeNode, accumulator): @@ -184,14 +184,14 @@ def visitor(node: interfaces.renderers.TreeNode, accumulator): class CSVRenderer(CLIRenderer): _type_renderers = { - format_hints.Bin: quoted_optional(lambda x: "0b{:b}".format(x)), - format_hints.Hex: quoted_optional(lambda x: "0x{:x}".format(x)), + format_hints.Bin: quoted_optional(lambda x: f"0b{x:b}"), + format_hints.Hex: quoted_optional(lambda x: f"0x{x:x}"), format_hints.HexBytes: quoted_optional(hex_bytes_as_text), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), interfaces.renderers.Disassembly: quoted_optional(display_disassembly), - bytes: quoted_optional(lambda x: " ".join(["{0:02x}".format(b) for b in x])), + bytes: quoted_optional(lambda x: " ".join([f"{b:02x}" for b in x])), datetime.datetime: quoted_optional(lambda x: x.strftime("%Y-%m-%d %H:%M:%S.%f %Z")), - 'default': quoted_optional(lambda x: "{}".format(x)) + 'default': quoted_optional(lambda x: f"{x}") } name = "csv" @@ -212,7 +212,7 @@ def render(self, grid: interfaces.renderers.TreeGrid) -> None: for column in grid.columns: # Ignore the type because namedtuples don't realize they have accessible attributes line.append("{}".format('"' + column.name + '"')) - outfd.write("{}".format(",".join(line))) + outfd.write(f"{','.join(line)}") def visitor(node: interfaces.renderers.TreeNode, accumulator): accumulator.write("\n") @@ -223,7 +223,7 @@ def visitor(node: interfaces.renderers.TreeNode, accumulator): column = grid.columns[column_index] renderer = self._type_renderers.get(column.type, self._type_renderers['default']) line.append(renderer(node.values[column_index])) - accumulator.write("{}".format(",".join(line))) + accumulator.write(f"{','.join(line)}") return accumulator if not grid.populated: @@ -273,12 +273,12 @@ def visitor( renderer = self._type_renderers.get(column.type, self._type_renderers['default']) data = renderer(node.values[column_index]) max_column_widths[column.name] = max(max_column_widths.get(column.name, len(column.name)), - len("{}".format(data))) + len(f"{data}")) line[column] = data accumulator.append((node.path_depth, line)) return accumulator - final_output = [] # type: List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]] + final_output: List[Tuple[int, Dict[interfaces.renderers.Column, bytes]]] = [] if not grid.populated: grid.populate(visitor, final_output) else: @@ -304,7 +304,7 @@ class JsonRenderer(CLIRenderer): format_hints.HexBytes: quoted_optional(hex_bytes_as_text), interfaces.renderers.Disassembly: quoted_optional(display_disassembly), format_hints.MultiTypeData: quoted_optional(multitypedata_as_text), - bytes: optional(lambda x: " ".join(["{0:02x}".format(b) for b in x])), + bytes: optional(lambda x: " ".join([f"{b:02x}" for b in x])), datetime.datetime: lambda x: x.isoformat() if not isinstance(x, interfaces.renderers.BaseAbsentValue) else None, 'default': lambda x: x } @@ -323,15 +323,15 @@ def render(self, grid: interfaces.renderers.TreeGrid): outfd = sys.stdout outfd.write("\n") - final_output = ( - {}, []) # type: Tuple[Dict[str, List[interfaces.renderers.TreeNode]], List[interfaces.renderers.TreeNode]] + final_output: Tuple[Dict[str, List[interfaces.renderers.TreeNode]], List[interfaces.renderers.TreeNode]] = ( + {}, []) def visitor( node: interfaces.renderers.TreeNode, accumulator: Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]] ) -> Tuple[Dict[str, Dict[str, Any]], List[Dict[str, Any]]]: # Nodes always have a path value, giving them a path_depth of at least 1, we use max just in case acc_map, final_tree = accumulator - node_dict = {'__children': []} # type: Dict[str, Any] + node_dict: Dict[str, Any] = {'__children': []} for column_index in range(len(grid.columns)): column = grid.columns[column_index] renderer = self._type_renderers.get(column.type, self._type_renderers['default']) diff --git a/volatility3/cli/volargparse.py b/volatility3/cli/volargparse.py index 3935d912f0..8ba807fee1 100644 --- a/volatility3/cli/volargparse.py +++ b/volatility3/cli/volargparse.py @@ -5,13 +5,14 @@ import argparse import gettext import re -from typing import List +from typing import List, Optional, Sequence, Any, Union + # This effectively overrides/monkeypatches the core argparse module to provide more helpful output around choices # We shouldn't really steal a private member from argparse, but otherwise we're just duplicating code # HelpfulSubparserAction gives more information about the possible choices from a subparsed choice -# HelpfulArgParser gives the list of choices when no arguments are provided to a choice option whilst still using a METAVAR +# HelpfulArgParser gives the list of choices when no arguments are provided to a choice option whilst still using a class HelpfulSubparserAction(argparse._SubParsersAction): @@ -24,12 +25,19 @@ def __init__(self, *args, **kwargs) -> None: self.choices = None def __call__(self, - parser: 'HelpfulArgParser', + parser: argparse.ArgumentParser, namespace: argparse.Namespace, - values: List[str], - option_string: None = None) -> None: - parser_name = values[0] - arg_strings = values[1:] + values: Union[str, Sequence[Any], None], + option_string: Optional[str] = None) -> None: + + parser_name = '' + arg_strings = [] # type: List[str] + if values is not None: + for value in values: + if not parser_name: + parser_name = value + else: + arg_strings += [value] # set the parser name if requested if self.dest != argparse.SUPPRESS: @@ -38,10 +46,10 @@ def __call__(self, matched_parsers = [name for name in self._name_parser_map if parser_name in name] if len(matched_parsers) < 1: - msg = 'invalid choice {} (choose from {})'.format(parser_name, ', '.join(self._name_parser_map)) + msg = f"invalid choice {parser_name} (choose from {', '.join(self._name_parser_map)})" raise argparse.ArgumentError(self, msg) if len(matched_parsers) > 1: - msg = 'plugin {} matches multiple plugins ({})'.format(parser_name, ', '.join(matched_parsers)) + msg = f"plugin {parser_name} matches multiple plugins ({', '.join(matched_parsers)})" raise argparse.ArgumentError(self, msg) parser = self._name_parser_map[matched_parsers[0]] setattr(namespace, 'plugin', matched_parsers[0]) @@ -80,7 +88,7 @@ def _match_argument(self, action, arg_strings_pattern) -> int: if msg is None: msg = gettext.ngettext('expected %s argument', 'expected %s arguments', action.nargs) % action.nargs if action.choices: - msg = "{} (from: {})".format(msg, ", ".join(action.choices)) + msg = f"{msg} (from: {', '.join(action.choices)})" raise argparse.ArgumentError(action, msg) # return the number of arguments matched diff --git a/volatility3/cli/volshell/__init__.py b/volatility3/cli/volshell/__init__.py index 6dde839057..f2375c7740 100644 --- a/volatility3/cli/volshell/__init__.py +++ b/volatility3/cli/volshell/__init__.py @@ -7,7 +7,6 @@ import logging import os import sys -from urllib import request import volatility3.plugins import volatility3.symbols @@ -42,9 +41,9 @@ def __init__(self): def run(self): """Executes the command line module, taking the system arguments, determining the plugin to run and then running it.""" - sys.stdout.write("Volshell (Volatility 3 Framework) {}\n".format(constants.PACKAGE_VERSION)) + sys.stdout.write(f"Volshell (Volatility 3 Framework) {constants.PACKAGE_VERSION}\n") - framework.require_interface_version(1, 0, 0) + framework.require_interface_version(2, 0, 0) parser = argparse.ArgumentParser(prog = self.CLI_NAME, description = "A tool for interactivate forensic analysis of memory images") @@ -90,6 +89,10 @@ def run(self): help = "Clears out all short-term cached items", default = False, action = 'store_true') + parser.add_argument("--cache-path", + help = f"Change the default path ({constants.CACHE_PATH}) used to store the cache", + default = constants.CACHE_PATH, + type = str) # Volshell specific flags os_specific = parser.add_mutually_exclusive_group(required = False) @@ -113,8 +116,11 @@ def run(self): volatility3.symbols.__path__ = [os.path.abspath(p) for p in partial_args.symbol_dirs.split(";")] + constants.SYMBOL_BASEPATHS - vollog.info("Volatility plugins path: {}".format(volatility3.plugins.__path__)) - vollog.info("Volatility symbols path: {}".format(volatility3.symbols.__path__)) + if partial_args.cache_path: + constants.CACHE_PATH = partial_args.cache_path + + vollog.info(f"Volatility plugins path: {volatility3.plugins.__path__}") + vollog.info(f"Volatility symbols path: {volatility3.symbols.__path__}") if partial_args.log: file_logger = logging.FileHandler(partial_args.log) @@ -174,7 +180,7 @@ def run(self): # Run the argparser args = parser.parse_args() - vollog.log(constants.LOGLEVEL_VVV, "Cache directory used: {}".format(constants.CACHE_PATH)) + vollog.log(constants.LOGLEVEL_VVV, f"Cache directory used: {constants.CACHE_PATH}") plugin = generic.Volshell if args.windows: @@ -192,12 +198,11 @@ def run(self): # NOTE: This will *BREAK* if LayerStacker, or the automagic configuration system, changes at all ### if args.file: - file_name = os.path.abspath(args.file) - if not os.path.exists(file_name): - vollog.log(logging.INFO, "File does not exist: {}".format(file_name)) - else: - single_location = "file:" + request.pathname2url(file_name) + try: + single_location = self.location_from_file(args.file) ctx.config['automagic.LayerStacker.single_location'] = single_location + except ValueError as excp: + parser.error(str(excp)) # UI fills in the config, here we load it from the config file and do it before we process the CL parameters if args.config: @@ -238,7 +243,7 @@ def run(self): constructed.run() except exceptions.VolatilityException as excp: self.process_exceptions(excp) - parser.exit(1, "Unable to validate the plugin requirements: {}\n".format([x for x in excp.unsatisfied])) + parser.exit(1, f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") def main(): diff --git a/volatility3/cli/volshell/generic.py b/volatility3/cli/volshell/generic.py index 19d398ba92..8f81a04207 100644 --- a/volatility3/cli/volshell/generic.py +++ b/volatility3/cli/volshell/generic.py @@ -8,13 +8,13 @@ import string import struct import sys -from typing import Any, Dict, List, Optional, Tuple, Union, Type -from urllib import request +from typing import Any, Dict, List, Optional, Tuple, Union, Type, Iterable +from urllib import request, parse -from volatility3.cli import text_renderer +from volatility3.cli import text_renderer, volshell from volatility3.framework import renderers, interfaces, objects, plugins, exceptions from volatility3.framework.configuration import requirements -from volatility3.framework.layers import intel, physical +from volatility3.framework.layers import intel, physical, resources try: import capstone @@ -26,18 +26,29 @@ class Volshell(interfaces.plugins.PluginInterface): """Shell environment to directly interact with a memory image.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.__current_layer = None # type: Optional[str] + self.__current_layer: Optional[str] = None + self.__console = None def random_string(self, length: int = 32) -> str: return ''.join(random.sample(string.ascii_uppercase + string.digits, length)) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: - return [requirements.TranslationLayerRequirement(name = 'primary', description = 'Memory layer for the kernel')] + reqs: List[interfaces.configuration.RequirementInterface] = [] + if cls == Volshell: + reqs = [ + requirements.URIRequirement(name = 'script', + description = 'File to load and execute at start', + default = None, + optional = True) + ] + return reqs + [ + requirements.TranslationLayerRequirement(name = 'primary', description = 'Memory layer for the kernel'), + ] def run(self, additional_locals: Dict[str, Any] = None) -> interfaces.renderers.TreeGrid: """Runs the interactive volshell plugin. @@ -46,7 +57,7 @@ def run(self, additional_locals: Dict[str, Any] = None) -> interfaces.renderers. Return a TreeGrid but this is always empty since the point of this plugin is to run interactively """ - self._current_layer = self.config['primary'] + self.__current_layer = self.config['primary'] # Try to enable tab completion try: @@ -65,15 +76,21 @@ def run(self, additional_locals: Dict[str, Any] = None) -> interfaces.renderers. mode = self.__module__.split('.')[-1] mode = mode[0].upper() + mode[1:] - banner = """ + banner = f""" Call help() to see available functions - Volshell mode: {} - Current Layer: {} - """.format(mode, self.current_layer) + Volshell mode: {mode} + Current Layer: {self.current_layer} + """ + + sys.ps1 = f"({self.current_layer}) >>> " + self.__console = code.InteractiveConsole(locals = self._construct_locals_dict()) + # Since we have to do work to add the option only once for all different modes of volshell, we can't + # rely on the default having been set + if self.config.get('script', None) is not None: + self.run_script(location = self.config['script']) - sys.ps1 = "({}) >>> ".format(self.current_layer) - code.interact(banner = banner, local = self._construct_locals_dict()) + self.__console.interact(banner = banner) return renderers.TreeGrid([("Terminating", str)], None) @@ -88,14 +105,14 @@ def help(self, *args): for aliases, item in self.construct_locals(): name = ", ".join(aliases) if item.__doc__ and callable(item): - print("* {}".format(name)) - print(" {}".format(item.__doc__)) + print(f"* {name}") + print(f" {item.__doc__}") else: variables.append(name) print("\nVariables:") for var in variables: - print(" {}".format(var)) + print(f" {var}") def construct_locals(self) -> List[Tuple[List[str], Any]]: """Returns a dictionary listing the functions to be added to the @@ -109,7 +126,8 @@ def construct_locals(self) -> List[Tuple[List[str], Any]]: (['gt', 'generate_treegrid'], self.generate_treegrid), (['rt', 'render_treegrid'], self.render_treegrid), (['ds', 'display_symbols'], self.display_symbols), (['hh', 'help'], self.help), - (['cc', 'create_configurable'], self.create_configurable), (['lf', 'load_file'], self.load_file)] + (['cc', 'create_configurable'], self.create_configurable), (['lf', 'load_file'], self.load_file), + (['rs', 'run_script'], self.run_script)] def _construct_locals_dict(self) -> Dict[str, Any]: """Returns a dictionary of the locals """ @@ -156,14 +174,14 @@ def _ascii_bytes(bytes): @property def current_layer(self): - return self._current_layer + return self.__current_layer def change_layer(self, layer_name = None): """Changes the current default layer""" if not layer_name: layer_name = self.config['primary'] - self._current_layer = layer_name - sys.ps1 = "({}) >>> ".format(self.current_layer) + self.__current_layer = layer_name + sys.ps1 = f"({self.current_layer}) >>> " def display_bytes(self, offset, count = 128, layer_name = None): """Displays byte values and ASCII characters""" @@ -203,7 +221,7 @@ def disassemble(self, offset, count = 128, layer_name = None, architecture = Non } if architecture is not None: for i in disasm_types[architecture].disasm(remaining_data, offset): - print("0x%x:\t%s\t%s" % (i.address, i.mnemonic, i.op_str)) + print(f"0x{i.address:x}:\t{i.mnemonic}\t{i.op_str}") def display_type(self, object: Union[str, interfaces.objects.ObjectInterface, interfaces.objects.Template], @@ -227,7 +245,7 @@ def display_type(self, volobject = self.context.object(volobject.vol.type_name, layer_name = self.current_layer, offset = offset) if hasattr(volobject.vol, 'size'): - print("{} ({} bytes)".format(volobject.vol.type_name, volobject.vol.size)) + print(f"{volobject.vol.type_name} ({volobject.vol.size} bytes)") elif hasattr(volobject.vol, 'data_format'): data_format = volobject.vol.data_format print("{} ({} bytes, {} endian, {})".format(volobject.vol.type_name, data_format.length, @@ -283,7 +301,7 @@ def generate_treegrid(self, plugin: Type[interfaces.plugins.PluginInterface], constructed = plugins.construct_plugin(self.context, [], plugin, plugin_path, None, NullFileHandler) return constructed.run() except exceptions.UnsatisfiedException as excp: - print("Unable to validate the plugin requirements: {}\n".format([x for x in excp.unsatisfied])) + print(f"Unable to validate the plugin requirements: {[x for x in excp.unsatisfied]}\n") return None def render_treegrid(self, @@ -318,11 +336,20 @@ def display_symbols(self, symbol_table: str = None): len_offset = len(hex(symbol.address)) print(" " * (longest_offset - len_offset), hex(symbol.address), " ", symbol.name) - def load_file(self, location: str = None, filename: str = ''): + def run_script(self, location: str): + """Runs a python script within the context of volshell""" + if not parse.urlparse(location).scheme: + location = "file:" + request.pathname2url(location) + print(f"Running code from {location}\n") + accessor = resources.ResourceAccessor() + with io.TextIOWrapper(accessor.open(url = location), encoding = 'utf-8') as fp: + self.__console.runsource(fp.read(), symbol = 'exec') + print("\nCode complete") + + def load_file(self, location: str): """Loads a file into a Filelayer and returns the name of the layer""" layer_name = self.context.layers.free_layer_name() - if location is None: - location = "file:" + request.pathname2url(filename) + location = volshell.VolShell.location_from_file(location) current_config_path = 'volshell.layers.' + layer_name self.context.config[interfaces.configuration.path_join(current_config_path, "location")] = location layer = physical.FileLayer(self.context, current_config_path, layer_name) @@ -371,10 +398,10 @@ def __init__(self, preferred_name: str): interfaces.plugins.FileHandlerInterface.__init__(self, preferred_name) super().__init__() - def writelines(self, lines): + def writelines(self, lines: Iterable[bytes]): """Dummy method""" pass - def write(self, data): + def write(self, b: bytes): """Dummy method""" - return len(data) + return len(b) diff --git a/volatility3/cli/volshell/linux.py b/volatility3/cli/volshell/linux.py index 73d5481aa1..4338ae06f0 100644 --- a/volatility3/cli/volshell/linux.py +++ b/volatility3/cli/volshell/linux.py @@ -17,7 +17,7 @@ class Volshell(generic.Volshell): def get_requirements(cls): return (super().get_requirements() + [ requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID", optional = True) ]) @@ -30,9 +30,9 @@ def change_task(self, pid = None): if process_layer is not None: self.change_layer(process_layer) return - print("Layer for task ID {} could not be constructed".format(pid)) + print(f"Layer for task ID {pid} could not be constructed") return - print("No task with task ID {} found".format(pid)) + print(f"No task with task ID {pid} found") def list_tasks(self): """Returns a list of task objects from the primary layer""" diff --git a/volatility3/cli/volshell/mac.py b/volatility3/cli/volshell/mac.py index 662f8dcb47..8218848baf 100644 --- a/volatility3/cli/volshell/mac.py +++ b/volatility3/cli/volshell/mac.py @@ -30,9 +30,9 @@ def change_task(self, pid = None): if process_layer is not None: self.change_layer(process_layer) return - print("Layer for task ID {} could not be constructed".format(pid)) + print(f"Layer for task ID {pid} could not be constructed") return - print("No task with task ID {} found".format(pid)) + print(f"No task with task ID {pid} found") def list_tasks(self): """Returns a list of task objects from the primary layer""" diff --git a/volatility3/cli/volshell/windows.py b/volatility3/cli/volshell/windows.py index d9de5a92ff..6c191ad28a 100644 --- a/volatility3/cli/volshell/windows.py +++ b/volatility3/cli/volshell/windows.py @@ -29,7 +29,7 @@ def change_process(self, pid = None): process_layer = process.add_process_layer() self.change_layer(process_layer) return - print("No process with process ID {} found".format(pid)) + print(f"No process with process ID {pid} found") def list_processes(self): """Returns a list of EPROCESS objects from the primary layer""" diff --git a/volatility3/framework/__init__.py b/volatility3/framework/__init__.py index e8c67588cb..2c94441302 100644 --- a/volatility3/framework/__init__.py +++ b/volatility3/framework/__init__.py @@ -3,13 +3,13 @@ # """Volatility 3 framework.""" # Check the python version to ensure it's suitable -# We currently require 3.5.3 since 3.5.1 has no typing.Type and 3.5.2 is broken for ''/delayed encapsulated types import glob import sys +import zipfile -required_python_version = (3, 5, 3) +required_python_version = (3, 6, 0) if (sys.version_info.major != required_python_version[0] or sys.version_info.minor < required_python_version[1] or - (sys.version_info.minor == required_python_version[1] and sys.version_info.micro < required_python_version[2])): + (sys.version_info.minor == required_python_version[1] and sys.version_info.micro < required_python_version[2])): raise RuntimeError( "Volatility framework requires python version {}.{}.{} or greater".format(*required_python_version)) @@ -21,6 +21,7 @@ from volatility3.framework import constants, interfaces + # ## # # SemVer version scheme @@ -53,22 +54,22 @@ def require_interface_version(*args) -> None: ".".join([str(x) for x in interface_version()[0:1]]), ".".join([str(x) for x in args[0:2]]))) -class noninheritable(object): +class NonInheritable(object): def __init__(self, value: Any, cls: Type) -> None: self.default_value = value self.cls = cls - def __get__(self, obj: Any, type: Type = None) -> Any: + def __get__(self, obj: Any, get_type: Type = None) -> Any: if type == self.cls: if hasattr(self.default_value, '__get__'): - return self.default_value.__get__(obj, type) + return self.default_value.__get__(obj, get_type) return self.default_value raise AttributeError def hide_from_subclasses(cls: Type) -> Type: - cls.hidden = noninheritable(True, cls) + cls.hidden = NonInheritable(True, cls) return cls @@ -78,7 +79,7 @@ def hide_from_subclasses(cls: Type) -> Type: def class_subclasses(cls: Type[T]) -> Generator[Type[T], None, None]: """Returns all the (recursive) subclasses of a given class.""" if not inspect.isclass(cls): - raise TypeError("class_subclasses parameter not a valid class: {}".format(cls)) + raise TypeError(f"class_subclasses parameter not a valid class: {cls}") for clazz in cls.__subclasses__(): # The typing system is not clever enough to realize that clazz has a hidden attr after the hasattr check if not hasattr(clazz, 'hidden') or not clazz.hidden: # type: ignore @@ -87,35 +88,90 @@ def class_subclasses(cls: Type[T]) -> Generator[Type[T], None, None]: yield return_value -def import_files(base_module, ignore_errors = False) -> List[str]: +def import_files(base_module, ignore_errors: bool = False) -> List[str]: """Imports all plugins present under plugins module namespace.""" failures = [] if not isinstance(base_module.__path__, list): raise TypeError("[base_module].__path__ must be a list of paths") vollog.log(constants.LOGLEVEL_VVVV, - "Importing from the following paths: {}".format(", ".join(base_module.__path__))) + f"Importing from the following paths: {', '.join(base_module.__path__)}") for path in base_module.__path__: for root, _, files in os.walk(path, followlinks = True): # TODO: Figure out how to import pycache files if root.endswith("__pycache__"): continue - for f in files: - if (f.endswith(".py") or f.endswith(".pyc") or f.endswith(".pyo")) and not f.startswith("__"): - modpath = os.path.join(root[len(path) + len(os.path.sep):], f[:f.rfind(".")]) - module = modpath.replace(os.path.sep, ".") - if base_module.__name__ + "." + module not in sys.modules: - try: - importlib.import_module(base_module.__name__ + "." + module) - except ImportError as e: - vollog.debug(str(e)) - vollog.debug("Failed to import module {} based on file: {}".format( - base_module.__name__ + "." + module, modpath)) - failures.append(base_module.__name__ + "." + module) - if not ignore_errors: - raise + for filename in files: + if zipfile.is_zipfile(os.path.join(root, filename)): + # Use the root to add this to the module path, and sub-traverse the files + new_module = base_module + premodules = root[len(path) + len(os.path.sep):].replace(os.path.sep, '.') + for component in premodules.split('.'): + if component: + try: + new_module = getattr(new_module, component) + except AttributeError: + failures += [new_module + '.' + component] + new_module.__path__ = [os.path.join(root, filename)] + new_module.__path__ + for ziproot, zipfiles in _zipwalk(os.path.join(root, filename)): + for zfile in zipfiles: + if _filter_files(zfile): + submodule = zfile[:zfile.rfind('.')].replace(os.path.sep, '.') + failures += import_file(new_module.__name__ + '.' + submodule, + os.path.join(path, ziproot, zfile)) + else: + if _filter_files(filename): + modpath = os.path.join(root[len(path) + len(os.path.sep):], filename[:filename.rfind(".")]) + submodule = modpath.replace(os.path.sep, ".") + failures += import_file(base_module.__name__ + '.' + submodule, + os.path.join(root, filename), + ignore_errors) + return failures +def _filter_files(filename: str): + """Ensures that a filename traversed is an importable python file""" + return (filename.endswith(".py") or filename.endswith(".pyc") or filename.endswith( + ".pyo")) and not filename.startswith("__") + + +def import_file(module: str, path: str, ignore_errors: bool = False) -> List[str]: + """Imports a python file based on an existing module, a submodule and a filepath for error messages + + Args + module: Module name to be imported + path: File to be imported from (used for error messages) + + Returns + List of modules that may have failed to import + + """ + failures = [] + if module not in sys.modules: + try: + importlib.import_module(module) + except ImportError as e: + vollog.debug(str(e)) + vollog.debug("Failed to import module {} based on file: {}".format(module, path)) + failures.append(module) + if not ignore_errors: + raise + return failures + + +def _zipwalk(path: str): + """Walks the contents of a zipfile just like os.walk""" + zip_results = {} + with zipfile.ZipFile(path) as archive: + for file in archive.filelist: + if not file.is_dir(): + dirlist = zip_results.get(os.path.dirname(file.filename), []) + dirlist.append(os.path.basename(file.filename)) + zip_results[os.path.join(path, os.path.dirname(file.filename))] = dirlist + for value in zip_results: + yield value, zip_results[value] + + def list_plugins() -> Dict[str, Type[interfaces.plugins.PluginInterface]]: plugin_list = {} for plugin in class_subclasses(interfaces.plugins.PluginInterface): diff --git a/volatility3/framework/automagic/__init__.py b/volatility3/framework/automagic/__init__.py index be1badb6b5..e4d422c995 100644 --- a/volatility3/framework/automagic/__init__.py +++ b/volatility3/framework/automagic/__init__.py @@ -21,11 +21,13 @@ vollog = logging.getLogger(__name__) -windows_automagic = ['ConstructionMagic', 'LayerStacker', 'WintelHelper', 'KernelPDBScanner', 'WinSwapLayers'] +windows_automagic = [ + 'ConstructionMagic', 'LayerStacker', 'KernelPDBScanner', 'WinSwapLayers', 'KernelModule' +] -linux_automagic = ['ConstructionMagic', 'LayerStacker', 'LinuxBannerCache', 'LinuxSymbolFinder'] +linux_automagic = ['ConstructionMagic', 'LayerStacker', 'LinuxBannerCache', 'LinuxSymbolFinder', 'KernelModule'] -mac_automagic = ['ConstructionMagic', 'LayerStacker', 'MacBannerCache', 'MacSymbolFinder'] +mac_automagic = ['ConstructionMagic', 'LayerStacker', 'MacBannerCache', 'MacSymbolFinder', 'KernelModule'] def available(context: interfaces.context.ContextInterface) -> List[interfaces.automagic.AutomagicInterface]: @@ -44,11 +46,11 @@ def available(context: interfaces.context.ContextInterface) -> List[interfaces.a clazz(context, interfaces.configuration.path_join(config_path, clazz.__name__)) for clazz in class_subclasses(interfaces.automagic.AutomagicInterface) ], - key = lambda x: x.priority) + key = lambda x: x.priority) def choose_automagic( - automagics: List[interfaces.automagic.AutomagicInterface], + automagics: List[Type[interfaces.automagic.AutomagicInterface]], plugin: Type[interfaces.plugins.PluginInterface]) -> List[Type[interfaces.automagic.AutomagicInterface]]: """Chooses which automagics to run, maintaining the order they were handed in.""" @@ -72,7 +74,7 @@ def choose_automagic( vollog.info("No plugin category detected") return automagics - vollog.info("Detected a {} category plugin".format(plugin_category)) + vollog.info(f"Detected a {plugin_category} category plugin") output = [] for amagic in automagics: if amagic.__class__.__name__ in automagic_categories[plugin_category]: @@ -127,7 +129,7 @@ def run(automagics: List[interfaces.automagic.AutomagicInterface], for automagic in automagics: try: - vollog.info("Running automagic: {}".format(automagic.__class__.__name__)) + vollog.info(f"Running automagic: {automagic.__class__.__name__}") automagic(context, config_path, requirement, progress_callback) except Exception as excp: exceptions.append(traceback.TracebackException.from_exception(excp)) diff --git a/volatility3/framework/automagic/construct_layers.py b/volatility3/framework/automagic/construct_layers.py index 15e6af4bde..40a17419f8 100644 --- a/volatility3/framework/automagic/construct_layers.py +++ b/volatility3/framework/automagic/construct_layers.py @@ -37,7 +37,7 @@ def __call__(self, # Make sure we import the layers, so they can reconstructed framework.import_files(sys.modules['volatility3.framework.layers']) - result = [] # type: List[str] + result: List[str] = [] if requirement.unsatisfied(context, config_path): # Having called validate at the top level tells us both that we need to dig deeper # but also ensures that TranslationLayerRequirements have got the correct subrequirements if their class is populated @@ -48,12 +48,12 @@ def __call__(self, self(context, subreq_config_path, subreq, optional = optional or subreq.optional) except Exception as e: # We don't really care if this fails, it tends to mean the configuration isn't complete for that item - vollog.log(constants.LOGLEVEL_VVVV, "Construction Exception occurred: {}".format(e)) + vollog.log(constants.LOGLEVEL_VVVV, f"Construction Exception occurred: {e}") invalid = subreq.unsatisfied(context, subreq_config_path) # We want to traverse optional paths, so don't check until we've tried to validate # We also don't want to emit a debug message when a parent is optional, hence the optional parameter if invalid and not (optional or subreq.optional): - vollog.log(constants.LOGLEVEL_V, "Failed on requirement: {}".format(subreq_config_path)) + vollog.log(constants.LOGLEVEL_V, f"Failed on requirement: {subreq_config_path}") result.append(interfaces.configuration.path_join(subreq_config_path, subreq.name)) if result: return result diff --git a/volatility3/framework/automagic/linux.py b/volatility3/framework/automagic/linux.py index 34d6e79365..f9fa22c071 100644 --- a/volatility3/framework/automagic/linux.py +++ b/volatility3/framework/automagic/linux.py @@ -14,7 +14,7 @@ class LinuxIntelStacker(interfaces.automagic.StackerLayerInterface): - stack_order = 45 + stack_order = 35 exclusion_list = ['mac', 'windows'] @classmethod @@ -41,7 +41,7 @@ def stack(cls, mss = scanners.MultiStringScanner([x for x in linux_banners if x is not None]) for _, banner in layer.scan(context = context, scanner = mss, progress_callback = progress_callback): dtb = None - vollog.debug("Identified banner: {}".format(repr(banner))) + vollog.debug(f"Identified banner: {repr(banner)}") symbol_files = linux_banners.get(banner, None) if symbol_files: @@ -57,7 +57,7 @@ def stack(cls, layer_name, progress_callback = progress_callback) - layer_class = intel.Intel # type: Type + layer_class: Type = intel.Intel if 'init_top_pgt' in table.symbols: layer_class = intel.Intel32e dtb_symbol_name = 'init_top_pgt' @@ -79,10 +79,11 @@ def stack(cls, layer = layer_class(context, config_path = config_path, name = new_layer_name, - metadata = {'kaslr_value': aslr_shift}) + metadata = {'os': 'Linux'}) + layer.config['kernel_virtual_offset'] = aslr_shift if layer and dtb: - vollog.debug("DTB was found at: 0x{:0x}".format(dtb)) + vollog.debug(f"DTB was found at: 0x{dtb:0x}") return layer vollog.debug("No suitable linux banner could be matched") return None diff --git a/volatility3/framework/automagic/mac.py b/volatility3/framework/automagic/mac.py index 3ba1790688..fb725a2346 100644 --- a/volatility3/framework/automagic/mac.py +++ b/volatility3/framework/automagic/mac.py @@ -6,7 +6,7 @@ import struct from typing import Optional -from volatility3.framework import interfaces, constants, layers +from volatility3.framework import interfaces, constants, layers, exceptions from volatility3.framework.automagic import symbol_cache, symbol_finder from volatility3.framework.layers import intel, scanners from volatility3.framework.symbols import mac @@ -15,7 +15,7 @@ class MacIntelStacker(interfaces.automagic.StackerLayerInterface): - stack_order = 45 + stack_order = 35 exclusion_list = ['windows', 'linux'] @classmethod @@ -44,7 +44,7 @@ def stack(cls, for banner_offset, banner in layer.scan(context = context, scanner = mss, progress_callback = progress_callback): dtb = None - vollog.debug("Identified banner: {}".format(repr(banner))) + vollog.debug(f"Identified banner: {repr(banner)}") symbol_files = mac_banners.get(banner, None) if symbol_files: @@ -63,7 +63,7 @@ def stack(cls, progress_callback = progress_callback) if kaslr_shift == 0: - vollog.log(constants.LOGLEVEL_VVV, "Invalid kalsr_shift found at offset: {}".format(banner_offset)) + vollog.log(constants.LOGLEVEL_VVV, f"Invalid kalsr_shift found at offset: {banner_offset}") continue bootpml4_addr = cls.virtual_to_physical_address(table.get_symbol("BootPML4").address + kaslr_shift) @@ -79,13 +79,18 @@ def stack(cls, metadata = {'os': 'Mac'}) idlepml4_ptr = table.get_symbol("IdlePML4").address + kaslr_shift - idlepml4_str = layer.read(idlepml4_ptr, 4) + try: + idlepml4_str = layer.read(idlepml4_ptr, 4) + except exceptions.InvalidAddressException: + vollog.log(constants.LOGLEVEL_VVVV, f"Skipping invalid idlepml4_ptr: 0x{idlepml4_ptr:0x}") + continue + idlepml4_addr = struct.unpack(" None: + new_config_path = interfaces.configuration.path_join(config_path, requirement.name) + if not isinstance(requirement, configuration.requirements.ModuleRequirement): + # Check subrequirements + for req in requirement.requirements: + self(context, new_config_path, requirement.requirements[req], progress_callback) + return + if not requirement.unsatisfied(context, config_path): + return + # The requirement is unfulfilled and is a ModuleRequirement + + context.config[interfaces.configuration.path_join( + new_config_path, 'class')] = 'volatility3.framework.contexts.Module' + + for req in requirement.requirements: + if requirement.requirements[req].unsatisfied(context, new_config_path) and req != 'offset': + return + + # We now just have the offset requirement, but the layer requirement has been fulfilled. + # Unfortunately we don't know the layer name requirement's exact name + + for req in requirement.requirements: + if isinstance(requirement.requirements[req], configuration.requirements.TranslationLayerRequirement): + layer_kvo_config_path = interfaces.configuration.path_join(new_config_path, req, + 'kernel_virtual_offset') + offset_config_path = interfaces.configuration.path_join(new_config_path, 'offset') + offset = context.config[layer_kvo_config_path] + context.config[offset_config_path] = offset + + # Now construct the module based on the sub-requirements + requirement.construct(context, config_path) diff --git a/volatility3/framework/automagic/pdbscan.py b/volatility3/framework/automagic/pdbscan.py index 59b78c3cac..93d4337da3 100644 --- a/volatility3/framework/automagic/pdbscan.py +++ b/volatility3/framework/automagic/pdbscan.py @@ -10,7 +10,7 @@ import logging import math import os -from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union +from typing import Any, Dict, Iterable, List, Optional, Set, Tuple, Union, Callable from volatility3.framework import constants, exceptions, interfaces, layers from volatility3.framework.configuration import requirements @@ -56,13 +56,12 @@ def find_virtual_layers_from_req(self, context: interfaces.context.ContextInterf context: The context in which the `requirement` lives config_path: The path within the `context` for the `requirement`'s configuration variables requirement: The root of the requirement tree to search for :class:~`volatility3.framework.interfaces.layers.TranslationLayerRequirement` objects to scan - progress_callback: Means of providing the user with feedback during long processes Returns: A list of (layer_name, scan_results) """ sub_config_path = interfaces.configuration.path_join(config_path, requirement.name) - results = [] # type: List[str] + results: List[str] = [] if isinstance(requirement, requirements.TranslationLayerRequirement): # Check for symbols in this layer # FIXME: optionally allow a full (slow) scan @@ -90,6 +89,7 @@ def recurse_symbol_fulfiller(self, Args: context: Context on which to operate valid_kernel: A list of offsets where valid kernels have been found + progress_callback: Means of providing the user with feedback during long processes """ for sub_config_path, requirement in self._symbol_requirements: # TODO: Potentially think about multiple symbol requirements in both the same and different levels of the requirement tree @@ -124,35 +124,40 @@ def set_kernel_virtual_offset(self, context: interfaces.context.ContextInterface if valid_kernel: # Set the virtual offset under the TranslationLayer it applies to virtual_layer, kvo, kernel = valid_kernel - kvo_path = interfaces.configuration.path_join(context.layers[virtual_layer].config_path, - 'kernel_virtual_offset') - context.config[kvo_path] = kvo - vollog.debug("Setting kernel_virtual_offset to {}".format(hex(kvo))) + if kvo is not None: + kvo_path = interfaces.configuration.path_join(context.layers[virtual_layer].config_path, + 'kernel_virtual_offset') + context.config[kvo_path] = kvo + vollog.debug(f"Setting kernel_virtual_offset to {hex(kvo)}") def get_physical_layer_name(self, context, vlayer): return context.config.get(interfaces.configuration.path_join(vlayer.config_path, 'memory_layer'), None) + def method_slow_scan(self, + context: interfaces.context.ContextInterface, + vlayer: layers.intel.Intel, + progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: + + def test_virtual_kernel(physical_layer_name, virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ValidKernelType]: + # It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet) + if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int): + # Rule out kernels that couldn't find a suitable MZ header + return None + return (virtual_layer_name, kernel['mz_offset'], kernel) + + vollog.debug("Kernel base determination - slow scan virtual layer") + return self._method_layer_pdb_scan(context, vlayer, test_virtual_kernel, False, progress_callback) + def method_fixed_mapping(self, context: interfaces.context.ContextInterface, vlayer: layers.intel.Intel, progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: - # TODO: Verify this is a windows image - vollog.debug("Kernel base determination - testing fixed base address") - valid_kernel = None - virtual_layer_name = vlayer.name - physical_layer_name = self.get_physical_layer_name(context, vlayer) - kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES] - kernels = PDBUtility.pdbname_scan(ctx = context, - layer_name = physical_layer_name, - page_size = vlayer.page_size, - pdb_names = kernel_pdb_names, - progress_callback = progress_callback) - for kernel in kernels: + def test_physical_kernel(physical_layer_name:str , virtual_layer_name: str, kernel: Dict[str, Any]) -> Optional[ValidKernelType]: # It seems the kernel is loaded at a fixed mapping (presumably because the memory manager hasn't started yet) if kernel['mz_offset'] is None or not isinstance(kernel['mz_offset'], int): # Rule out kernels that couldn't find a suitable MZ header - continue + return None if vlayer.bits_per_register == 64: kvo = kernel['mz_offset'] + (31 << int(math.ceil(math.log2(vlayer.maximum_address + 1)) - 5)) else: @@ -161,13 +166,41 @@ def method_fixed_mapping(self, kvp = vlayer.mapping(kvo, 0) if (any([(p == kernel['mz_offset'] and layer_name == physical_layer_name) for (_, _, p, _, layer_name) in kvp])): - valid_kernel = (virtual_layer_name, kvo, kernel) - break + return (virtual_layer_name, kvo, kernel) else: vollog.debug("Potential kernel_virtual_offset did not map to expected location: {}".format( hex(kvo))) except exceptions.InvalidAddressException: - vollog.debug("Potential kernel_virtual_offset caused a page fault: {}".format(hex(kvo))) + vollog.debug(f"Potential kernel_virtual_offset caused a page fault: {hex(kvo)}") + + vollog.debug("Kernel base determination - testing fixed base address") + return self._method_layer_pdb_scan(context, vlayer, test_physical_kernel, True, progress_callback) + + def _method_layer_pdb_scan(self, + context: interfaces.context.ContextInterface, + vlayer: layers.intel.Intel, + test_kernel: Callable, + physical: bool = True, + progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: + # TODO: Verify this is a windows image + valid_kernel = None + virtual_layer_name = vlayer.name + physical_layer_name = self.get_physical_layer_name(context, vlayer) + + layer_to_scan = physical_layer_name + if not physical: + layer_to_scan = virtual_layer_name + + kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES] + kernels = PDBUtility.pdbname_scan(ctx = context, + layer_name = layer_to_scan, + page_size = vlayer.page_size, + pdb_names = kernel_pdb_names, + progress_callback = progress_callback) + for kernel in kernels: + valid_kernel = test_kernel(physical_layer_name, virtual_layer_name, kernel) + if valid_kernel is not None: + break return valid_kernel def _method_offset(self, @@ -179,13 +212,13 @@ def _method_offset(self, """Method for finding a suitable kernel offset based on a module table.""" vollog.debug("Kernel base determination - searching layer module list structure") - valid_kernel = None # type: Optional[ValidKernelType] + valid_kernel: Optional[ValidKernelType] = None # If we're here, chances are high we're in a Win10 x64 image with kernel base randomization physical_layer_name = self.get_physical_layer_name(context, vlayer) physical_layer = context.layers[physical_layer_name] # TODO: On older windows, this might be \WINDOWS\system32\nt rather than \SystemRoot\system32\nt results = physical_layer.scan(context, scanners.BytesScanner(pattern), progress_callback = progress_callback) - seen = set() # type: Set[int] + seen: Set[int] = set() # Because this will launch a scan of the virtual layer, we want to be careful for result in results: # TODO: Identify the specific structure we're finding and document this a bit better @@ -206,14 +239,14 @@ def _method_offset(self, def method_module_offset(self, context: interfaces.context.ContextInterface, vlayer: layers.intel.Intel, - progress_callback: constants.ProgressCallback = None) -> ValidKernelType: + progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: return self._method_offset(context, vlayer, b"\\SystemRoot\\system32\\nt", -16 - int(vlayer.bits_per_register / 8), progress_callback) def method_kdbg_offset(self, context: interfaces.context.ContextInterface, vlayer: layers.intel.Intel, - progress_callback: constants.ProgressCallback = None) -> ValidKernelType: + progress_callback: constants.ProgressCallback = None) -> Optional[ValidKernelType]: return self._method_offset(context, vlayer, b"KDBG", 8, progress_callback) def check_kernel_offset(self, @@ -224,7 +257,7 @@ def check_kernel_offset(self, """Scans a virtual address.""" # Scan a few megs of the virtual space at the location to see if they're potential kernels - valid_kernel = None # type: Optional[ValidKernelType] + valid_kernel: Optional[ValidKernelType] = None kernel_pdb_names = [bytes(name + ".pdb", "utf-8") for name in constants.windows.KERNEL_MODULE_NAMES] virtual_layer_name = vlayer.name @@ -245,7 +278,7 @@ def check_kernel_offset(self, return valid_kernel # List of methods to be run, in order, to determine the valid kernels - methods = [method_kdbg_offset, method_module_offset, method_fixed_mapping] + methods = [method_kdbg_offset, method_module_offset, method_fixed_mapping, method_slow_scan] def determine_valid_kernel(self, context: interfaces.context.ContextInterface, @@ -261,13 +294,13 @@ def determine_valid_kernel(self, Args: context: Context on which to operate - potential_kernels: Dictionary containing `GUID`, `age`, `pdb_name` and `mz_offset` keys + potential_layers: List of layer names that the kernel might live at progress_callback: Function taking a percentage and optional description to be called during expensive computations to indicate progress Returns: A dictionary of valid kernels """ - valid_kernel = None # type: Optional[ValidKernelType] + valid_kernel: Optional[ValidKernelType] = None for virtual_layer_name in potential_layers: vlayer = context.layers.get(virtual_layer_name, None) if isinstance(vlayer, layers.intel.Intel): @@ -298,8 +331,8 @@ def __call__(self, if symbol_req.unsatisfied(context, parent_path): valid_kernel = self.determine_valid_kernel(context, potential_layers, progress_callback) if valid_kernel: - self.recurse_symbol_fulfiller(context, valid_kernel, progress_callback) self.set_kernel_virtual_offset(context, valid_kernel) + self.recurse_symbol_fulfiller(context, valid_kernel, progress_callback) if progress_callback is not None: progress_callback(100, "PDB scanning finished") diff --git a/volatility3/framework/automagic/stacker.py b/volatility3/framework/automagic/stacker.py index 78fbab76c2..928e3d0684 100644 --- a/volatility3/framework/automagic/stacker.py +++ b/volatility3/framework/automagic/stacker.py @@ -13,7 +13,7 @@ import logging import sys import traceback -from typing import Any, List, Optional, Tuple, Type +from typing import List, Optional, Tuple, Type from volatility3 import framework from volatility3.framework import interfaces, constants @@ -58,7 +58,7 @@ def __call__(self, # Bow out quickly if the UI hasn't provided a single_location unsatisfied = self.unsatisfied(self.context, self.config_path) if unsatisfied: - vollog.info("Unable to run LayerStacker, unsatisfied requirement: {}".format(unsatisfied)) + vollog.info(f"Unable to run LayerStacker, unsatisfied requirement: {unsatisfied}") return list(unsatisfied) if not self.config or not self.config.get('single_location', None): raise ValueError("Unable to run LayerStacker, single_location parameter not provided") @@ -123,7 +123,7 @@ def stack(self, context: interfaces.context.ContextInterface, config_path: str, # Stash the changed config items self._cached = context.config.get(path, None), context.config.branch(path) - vollog.debug("Stacked layers: {}".format(stacked_layers)) + vollog.debug(f"Stacked layers: {stacked_layers}") @classmethod def stack_layer(cls, @@ -158,7 +158,7 @@ def stack_layer(cls, for stacker_item in stack_set: if not issubclass(stacker_item, interfaces.automagic.StackerLayerInterface): - raise TypeError("Stacker {} is not a descendent of StackerLayerInterface".format(stacker_item.__name__)) + raise TypeError(f"Stacker {stacker_item.__name__} is not a descendent of StackerLayerInterface") while stacked: stacked = False @@ -167,17 +167,17 @@ def stack_layer(cls, for stacker_cls in stack_set: stacker = stacker_cls() try: - vollog.log(constants.LOGLEVEL_VV, "Attempting to stack using {}".format(stacker_cls.__name__)) + vollog.log(constants.LOGLEVEL_VV, f"Attempting to stack using {stacker_cls.__name__}") new_layer = stacker.stack(context, initial_layer, progress_callback) if new_layer: context.layers.add_layer(new_layer) vollog.log(constants.LOGLEVEL_VV, - "Stacked {} using {}".format(new_layer.name, stacker_cls.__name__)) + f"Stacked {new_layer.name} using {stacker_cls.__name__}") break except Exception as excp: # Stacking exceptions are likely only of interest to developers, so the lowest level of logging fulltrace = traceback.TracebackException.from_exception(excp).format(chain = True) - vollog.log(constants.LOGLEVEL_VVV, "Exception during stacking: {}".format(str(excp))) + vollog.log(constants.LOGLEVEL_VVV, f"Exception during stacking: {str(excp)}") vollog.log(constants.LOGLEVEL_VVVV, "\n".join(fulltrace)) else: stacked = False diff --git a/volatility3/framework/automagic/symbol_cache.py b/volatility3/framework/automagic/symbol_cache.py index b2407f8e9b..7b6adf9b47 100644 --- a/volatility3/framework/automagic/symbol_cache.py +++ b/volatility3/framework/automagic/symbol_cache.py @@ -1,6 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import base64 import gc import json import logging @@ -9,9 +10,11 @@ import urllib import urllib.parse import urllib.request +import zipfile from typing import Dict, List, Optional from volatility3.framework import constants, exceptions, interfaces +from volatility3.framework.layers import resources from volatility3.framework.symbols import intermed vollog = logging.getLogger(__name__) @@ -26,15 +29,15 @@ class SymbolBannerCache(interfaces.automagic.AutomagicInterface): # The user would run it eventually either way, but running it first means it can be used that run priority = 0 - os = None # type: Optional[str] - symbol_name = "banner_name" # type: str - banner_path = None # type: Optional[str] + os: Optional[str] = None + symbol_name: str = "banner_name" + banner_path: Optional[str] = None @classmethod def load_banners(cls) -> BannersType: if not cls.banner_path: raise ValueError("Banner_path not appropriately set") - banners = {} # type: BannersType + banners: BannersType = {} if os.path.exists(cls.banner_path): with open(cls.banner_path, "rb") as f: # We use pickle over JSON because we're dealing with bytes objects @@ -51,13 +54,13 @@ def load_banners(cls) -> BannersType: path, str(banner or b'', 'latin-1'))) banners[banner].remove(path) # This is probably excessive, but it's here if we need it - # if url.scheme == 'jar': - # zip_file, zip_path = url.path.split("!") - # zip_file = urllib.parse.urlparse(zip_file).path - # if ((not os.path.exists(zip_file)) or (zip_path not in zipfile.ZipFile(zip_file).namelist())): - # vollog.log(constants.LOGLEVEL_VV, - # "Removing cached path {} for banner {}: file does not exist".format(path, banner)) - # banners[banner].remove(path) + if url.scheme == 'jar': + zip_file, zip_path = url.path.split("!") + zip_file = urllib.parse.urlparse(zip_file).path + if ((not os.path.exists(zip_file)) or (zip_path not in zipfile.ZipFile(zip_file).namelist())): + vollog.log(constants.LOGLEVEL_VV, + "Removing cached path {} for banner {}: file does not exist".format(path, banner)) + banners[banner].remove(path) if not banners[banner]: remove_banners.append(banner) @@ -81,20 +84,42 @@ def __call__(self, context, config_path, configurable, progress_callback = None) # We only need to be called once, so no recursion necessary banners = self.load_banners() - cacheables = list(intermed.IntermediateSymbolTable.file_symbol_url(self.os)) + cacheables = self.find_new_banner_files(banners, self.os) - for banner in banners: - for json_file in banners[banner]: - if json_file in cacheables: - cacheables.remove(json_file) + new_banners = self.read_new_banners(context, config_path, cacheables, self.symbol_name, self.os, + progress_callback) + + # Add in any new banners to the existing list + for new_banner in new_banners: + banner_list = banners.get(new_banner, []) + banners[new_banner] = list(set(banner_list + new_banners[new_banner])) + + # Do remote banners *after* the JSON loading, so that it doesn't pull down all the remote JSON + self.remote_banners(banners, self.os) + + # Rewrite the cached banners each run, since writing is faster than the banner_cache validation portion + self.save_banners(banners) + + if progress_callback is not None: + progress_callback(100, f"Built {self.os} caches") + + @classmethod + def read_new_banners(cls, context: interfaces.context.ContextInterface, config_path: str, new_urls: List[str], + symbol_name: str, operating_system: str = None, + progress_callback = None) -> Optional[Dict[bytes, List[str]]]: + """Reads the any new banners for the OS in question""" + if operating_system is None: + return None + + banners = {} - total = len(cacheables) + total = len(new_urls) if total > 0: - vollog.info("Building {} caches...".format(self.os)) + vollog.info(f"Building {operating_system} caches...") for current in range(total): if progress_callback is not None: - progress_callback(current * 100 / total, "Building {} caches".format(self.os)) - isf_url = cacheables[current] + progress_callback(current * 100 / total, f"Building {operating_system} caches") + isf_url = new_urls[current] isf = None try: @@ -104,8 +129,8 @@ def __call__(self, context, config_path, configurable, progress_callback = None) # We should store the banner against the filename # We don't bother with the hash (it'll likely take too long to validate) # but we should check at least that the banner matches on load. - banner = isf.get_symbol(self.symbol_name).constant_data - vollog.log(constants.LOGLEVEL_VV, "Caching banner {} for file {}".format(banner, isf_url)) + banner = isf.get_symbol(symbol_name).constant_data + vollog.log(constants.LOGLEVEL_VV, f"Caching banner {banner} for file {isf_url}") bannerlist = banners.get(banner, []) bannerlist.append(isf_url) @@ -113,15 +138,73 @@ def __call__(self, context, config_path, configurable, progress_callback = None) except exceptions.SymbolError: pass except json.JSONDecodeError: - vollog.log(constants.LOGLEVEL_VV, "Caching file {} failed due to JSON error".format(isf_url)) + vollog.log(constants.LOGLEVEL_VV, f"Caching file {isf_url} failed due to JSON error") finally: # Get rid of the loaded file, in case it sits in memory if isf: del isf gc.collect() + return banners - # Rewrite the cached banners each run, since writing is faster than the banner_cache validation portion - self.save_banners(banners) + @classmethod + def find_new_banner_files(cls, banners: Dict[bytes, List[str]], operating_system: str) -> List[str]: + """Gathers all files and remove existing banners""" + cacheables = list(intermed.IntermediateSymbolTable.file_symbol_url(operating_system)) + for banner in banners: + for json_file in banners[banner]: + if json_file in cacheables: + cacheables.remove(json_file) + return cacheables - if progress_callback is not None: - progress_callback(100, "Built {} caches".format(self.os)) + @classmethod + def remote_banners(cls, banners: Dict[bytes, List[str]], operating_system = None, banner_location = None): + """Adds remote URLs to the banner list""" + if operating_system is None: + return None + + if banner_location is None: + banner_location = constants.REMOTE_ISF_URL + + if not constants.OFFLINE and banner_location is not None: + try: + rbf = RemoteBannerFormat(banner_location) + rbf.process(banners, operating_system) + except urllib.error.URLError: + vollog.debug(f"Unable to download remote banner list from {banner_location}") + + +class RemoteBannerFormat: + def __init__(self, location: str): + self._location = location + with resources.ResourceAccessor().open(url = location) as fp: + self._data = json.load(fp) + if not self._verify(): + raise ValueError("Unsupported version for remote banner list format") + + def _verify(self) -> bool: + version = self._data.get('version', 0) + if version in [1]: + setattr(self, 'process', getattr(self, f'process_v{version}')) + return True + return False + + def process(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]): + raise ValueError("Banner List version not verified") + + def process_v1(self, banners: Dict[bytes, List[str]], operating_system: Optional[str]): + if operating_system in self._data: + for banner in self._data[operating_system]: + binary_banner = base64.b64decode(banner) + file_list = banners.get(binary_banner, []) + for value in self._data[operating_system][banner]: + if value not in file_list: + file_list = file_list + [value] + banners[binary_banner] = file_list + if 'additional' in self._data: + for location in self._data['additional']: + try: + subrbf = RemoteBannerFormat(location) + subrbf.process(banners, operating_system) + except IOError: + vollog.debug(f"Remote file not found: {location}") + return banners diff --git a/volatility3/framework/automagic/symbol_finder.py b/volatility3/framework/automagic/symbol_finder.py index f299767699..72dc071a5f 100644 --- a/volatility3/framework/automagic/symbol_finder.py +++ b/volatility3/framework/automagic/symbol_finder.py @@ -5,7 +5,7 @@ import logging from typing import Any, Iterable, List, Tuple, Type, Optional, Callable -from volatility3.framework import interfaces, constants, layers, exceptions +from volatility3.framework import interfaces, constants from volatility3.framework.automagic import symbol_cache from volatility3.framework.configuration import requirements from volatility3.framework.layers import scanners @@ -17,15 +17,15 @@ class SymbolFinder(interfaces.automagic.AutomagicInterface): """Symbol loader based on signature strings.""" priority = 40 - banner_config_key = "banner" # type: str - banner_cache = None # type: Optional[Type[symbol_cache.SymbolBannerCache]] - symbol_class = None # type: Optional[str] - find_aslr = None # type: Optional[Callable] + banner_config_key: str = "banner" + banner_cache: Optional[Type[symbol_cache.SymbolBannerCache]] = None + symbol_class: Optional[str] = None + find_aslr: Optional[Callable] = None def __init__(self, context: interfaces.context.ContextInterface, config_path: str) -> None: super().__init__(context, config_path) - self._requirements = [] # type: List[Tuple[str, interfaces.configuration.RequirementInterface]] - self._banners = {} # type: symbol_cache.BannersType + self._requirements: List[Tuple[str, interfaces.configuration.RequirementInterface]] = [] + self._banners: symbol_cache.BannersType = {} @property def banners(self) -> symbol_cache.BannersType: @@ -33,7 +33,7 @@ def banners(self) -> symbol_cache.BannersType: requested.""" if not self._banners: if not self.banner_cache: - raise RuntimeError("Cache has not been properly defined for {}".format(self.__class__.__name__)) + raise RuntimeError(f"Cache has not been properly defined for {self.__class__.__name__}") self._banners = self.banner_cache.load_banners() return self._banners @@ -93,16 +93,18 @@ def _banner_scan(self, 'raw_unicode_escape'))] # type: Iterable[Any] else: # Swap to the physical layer for scanning + # Only traverse down a layer if it's an intel layer # TODO: Fix this so it works for layers other than just Intel - layer = context.layers[layer.config['memory_layer']] + if isinstance(layer, layers.intel.Intel): + layer = context.layers[layer.config['memory_layer']] banner_list = layer.scan(context = context, scanner = mss, progress_callback = progress_callback) for _, banner in banner_list: - vollog.debug("Identified banner: {}".format(repr(banner))) + vollog.debug(f"Identified banner: {repr(banner)}") symbol_files = self.banners.get(banner, None) if symbol_files: isf_path = symbol_files[0] - vollog.debug("Using symbol library: {}".format(symbol_files[0])) + vollog.debug(f"Using symbol library: {symbol_files[0]}") clazz = self.symbol_class # Set the discovered options path_join = interfaces.configuration.path_join @@ -110,31 +112,12 @@ def _banner_scan(self, context.config[path_join(config_path, requirement.name, "isf_url")] = isf_path context.config[path_join(config_path, requirement.name, "symbol_mask")] = layer.address_mask - # Set a default symbol_shift when attempt to determine it, - # so we can create the symbols which are used in finding the aslr_shift anyway - if not context.config.get(path_join(config_path, requirement.name, "symbol_shift"), None): - # Don't overwrite it if it's already been set, it will be manually refound if not present - prefound_kaslr_value = context.layers[layer_name].metadata.get('kaslr_value', 0) - context.config[path_join(config_path, requirement.name, "symbol_shift")] = prefound_kaslr_value # Construct the appropriate symbol table requirement.construct(context, config_path) - - # Apply the ASLR masking (only if we're not already shifted) - if self.find_aslr and not context.config.get(path_join(config_path, requirement.name, "symbol_shift"), - None): - unmasked_symbol_table_name = context.config.get(path_join(config_path, requirement.name), None) - if not unmasked_symbol_table_name: - raise exceptions.SymbolSpaceError("Symbol table could not be constructed") - if not isinstance(layer, layers.intel.Intel): - raise TypeError("Layer name {} is not an intel space") - aslr_shift = self.find_aslr(context, unmasked_symbol_table_name, layer.config['memory_layer']) - context.config[path_join(config_path, requirement.name, "symbol_shift")] = aslr_shift - context.symbol_space.clear_symbol_cache(unmasked_symbol_table_name) - break else: if symbol_files: - vollog.debug("Symbol library path not found: {}".format(symbol_files[0])) + vollog.debug(f"Symbol library path not found: {symbol_files[0]}") # print("Kernel", banner, hex(banner_offset)) else: vollog.debug("No existing banners found") diff --git a/volatility3/framework/automagic/windows.py b/volatility3/framework/automagic/windows.py index 198e64e993..eb63a75e59 100644 --- a/volatility3/framework/automagic/windows.py +++ b/volatility3/framework/automagic/windows.py @@ -28,7 +28,7 @@ """ import logging import struct -from typing import Any, Generator, List, Optional, Tuple, Type +from typing import Generator, List, Optional, Tuple, Type, Iterable from volatility3.framework import interfaces, layers, constants from volatility3.framework.configuration import requirements @@ -37,156 +37,43 @@ vollog = logging.getLogger(__name__) -class DtbTest: - """This class generically contains the tests for a page based on a set of - class parameters. - - When constructed it contains all the information necessary to - extract a specific index from a page and determine whether it points - back to that page's offset. - """ +class DtbSelfReferential: + """A generic DTB test which looks for a self-referential pointer at *any* + index within the page.""" - def __init__(self, layer_type: Type[layers.intel.Intel], ptr_struct: str, ptr_reference: int, mask: int) -> None: + def __init__(self, layer_type: Type[layers.intel.Intel], ptr_struct: str, mask: int, + valid_range: Iterable[int], reserved_bits: int) -> None: self.layer_type = layer_type self.ptr_struct = ptr_struct self.ptr_size = struct.calcsize(ptr_struct) - self.ptr_reference = ptr_reference self.mask = mask - self.page_size = layer_type.page_size # type: int - - def _unpack(self, value: bytes) -> int: - return struct.unpack("<" + self.ptr_struct, value)[0] - - def __call__(self, data: bytes, data_offset: int, page_offset: int) -> Optional[Tuple[int, Any]]: - """Tests a specific page in a chunk of data to see if it contains a - self-referential pointer. - - Args: - data: The chunk of data that contains the page to be scanned - data_offset: Where, within the layer, the chunk of data lives - page_offset: Where, within the data, the page to be scanned starts - - Returns: - A valid DTB within this page (and an additional parameter for data) - """ - value = data[page_offset + (self.ptr_reference * self.ptr_size):page_offset + - ((self.ptr_reference + 1) * self.ptr_size)] - try: - ptr = self._unpack(value) - except struct.error: - return None - # The value *must* be present (bit 0) since it's a mapped page - # It's almost always writable (bit 1) - # It's occasionally Super, but not reliably so, haven't checked when/why not - # The top 3-bits are usually ignore (which in practice means 0 - # Need to find out why the middle 3-bits are usually 6 (0110) - if ptr != 0 and (ptr & self.mask == data_offset + page_offset) & (ptr & 0xFF1 == 0x61): - dtb = (ptr & self.mask) - return self.second_pass(dtb, data, data_offset) - return None - - def second_pass(self, dtb: int, data: bytes, data_offset: int) -> Optional[Tuple[int, Any]]: - """Re-reads over the whole page to validate other records based on the - number of pages marked user vs super. - - Args: - dtb: The identified dtb that needs validating - data: The chunk of data that contains the dtb to be validated - data_offset: Where, within the layer, the chunk of data lives - - Returns: - A valid DTB within this page - """ - page = data[dtb - data_offset:dtb - data_offset + self.page_size] - usr_count, sup_count = 0, 0 - for i in range(0, self.page_size, self.ptr_size): - val = self._unpack(page[i:i + self.ptr_size]) - if val & 0x1: - sup_count += 0 if (val & 0x4) else 1 - usr_count += 1 if (val & 0x4) else 0 - # print(hex(dtb), usr_count, sup_count, usr_count + sup_count) - # We sometimes find bogus DTBs at 0x16000 with a very low sup_count and 0 usr_count - # I have a winxpsp2-x64 image with identical usr/sup counts at 0x16000 and 0x24c00 as well as the actual 0x3c3000 - if usr_count or sup_count > 5: - return dtb, None - return None - - -class DtbTest32bit(DtbTest): - - def __init__(self) -> None: - super().__init__(layer_type = layers.intel.WindowsIntel, - ptr_struct = "I", - ptr_reference = 0x300, - mask = 0xFFFFF000) - - -class DtbTest64bit(DtbTest): - - def __init__(self) -> None: - super().__init__(layer_type = layers.intel.WindowsIntel32e, - ptr_struct = "Q", - ptr_reference = 0x1ED, - mask = 0x3FFFFFFFFFF000) - - -class DtbTestPae(DtbTest): - - def __init__(self) -> None: - super().__init__(layer_type = layers.intel.WindowsIntelPAE, - ptr_struct = "Q", - ptr_reference = 0x3, - mask = 0x3FFFFFFFFFF000) - - def second_pass(self, dtb: int, data: bytes, data_offset: int) -> Optional[Tuple[int, Any]]: - """PAE top level directory tables contains four entries and the self- - referential pointer occurs in the second level of tables (so as not to - use up a full quarter of the space). This is very high in the space, - and occurs in the fourht (last quarter) second-level table. The - second-level tables appear always to come sequentially directly after - the real dtb. The value for the real DTB is therefore four page - earlier (and the fourth entry should point back to the `dtb` parameter - this function was originally passed. - - Args: - dtb: The identified self-referential pointer that needs validating - data: The chunk of data that contains the dtb to be validated - data_offset: Where, within the layer, the chunk of data lives - - Returns: - Returns the actual DTB of the PAE space - """ - dtb -= 0x4000 - # If we're not in something that the overlap would pick up - if dtb - data_offset >= 0: - pointers = data[dtb - data_offset + (3 * self.ptr_size):dtb - data_offset + (4 * self.ptr_size)] - val = self._unpack(pointers) - if (val & self.mask == dtb + 0x4000) and (val & 0xFFF == 0x001): - return dtb, None - return None - - -class DtbSelfReferential(DtbTest): - """A generic DTB test which looks for a self-referential pointer at *any* - index within the page.""" - - def __init__(self, layer_type: Type[layers.intel.Intel], ptr_struct: str, ptr_reference: int, mask: int) -> None: - super().__init__(layer_type = layer_type, ptr_struct = ptr_struct, ptr_reference = ptr_reference, mask = mask) + self.page_size: int = layer_type.page_size + self.valid_range = valid_range + self.reserved_bits = reserved_bits def __call__(self, data: bytes, data_offset: int, page_offset: int) -> Optional[Tuple[int, int]]: page = data[page_offset:page_offset + self.page_size] if not page: return None ref_pages = set() + for ref in range(0, self.page_size, self.ptr_size): ptr_data = page[ref:ref + self.ptr_size] - if len(ptr_data) == self.ptr_size: - ptr, = struct.unpack(self.ptr_struct, ptr_data) - if ((ptr & self.mask) == (data_offset + page_offset)) and (data_offset + page_offset > 0): + ptr, = struct.unpack(self.ptr_struct, ptr_data) + # For both Intel-32e, bit 7 is reserved (more are reserved in PAE), so if that's ever set, + # we can move on + if (ptr & self.reserved_bits) and (ptr & 0x01): + return None + if ((ptr & self.mask) == (data_offset + page_offset)) and (data_offset + page_offset > 0): + # Pointer must be valid + if (ptr & 0x01): ref_pages.add(ref) + # The DTB is extremely unlikely to refer back to itself. so the number of reference should always be exactly 1 if len(ref_pages) == 1: - return (data_offset + page_offset), ref_pages.pop() + ref_page = ref_pages.pop() + if (ref_page // self.ptr_size) in self.valid_range: + return (data_offset + page_offset), ref_page return None @@ -195,8 +82,9 @@ class DtbSelfRef32bit(DtbSelfReferential): def __init__(self): super().__init__(layer_type = layers.intel.WindowsIntel, ptr_struct = "I", - ptr_reference = 0x300, - mask = 0xFFFFF000) + mask = 0xFFFFF000, + valid_range = [0x300], + reserved_bits = 0x0) class DtbSelfRef64bit(DtbSelfReferential): @@ -204,8 +92,35 @@ class DtbSelfRef64bit(DtbSelfReferential): def __init__(self) -> None: super().__init__(layer_type = layers.intel.WindowsIntel32e, ptr_struct = "Q", - ptr_reference = 0x1ED, - mask = 0x3FFFFFFFFFF000) + mask = 0x3FFFFFFFFFF000, + valid_range = range(0x100, 0x1ff), + reserved_bits = 0x80) + + +class DtbSelfRef64bitOldWindows(DtbSelfReferential): + + def __init__(self) -> None: + super().__init__(layer_type = layers.intel.WindowsIntel32e, + ptr_struct = "Q", + mask = 0x3FFFFFFFFFF000, + valid_range = [0x1ed], + reserved_bits = 0x80) + + +class DtbSelfRefPae(DtbSelfReferential): + + def __init__(self) -> None: + super().__init__(layer_type = layers.intel.WindowsIntelPAE, + ptr_struct = "Q", + valid_range = [0x3], + mask = 0x3FFFFFFFFFF000, + reserved_bits = 0x0) + + def __call__(self, *args, **kwargs): + dtb = super().__call__(*args, **kwargs) + if dtb: + return dtb[0] - 0x4000, dtb[1] + return dtb class PageMapScanner(interfaces.layers.ScannerInterface): @@ -213,82 +128,33 @@ class PageMapScanner(interfaces.layers.ScannerInterface): architecture.""" overlap = 0x4000 thread_safe = True - tests = [DtbTest64bit(), DtbTest32bit(), DtbTestPae()] + tests = [DtbSelfRef64bit(), DtbSelfRefPae(), DtbSelfRef32bit()] """The default tests to run when searching for DTBs""" - def __init__(self, tests: List[DtbTest]) -> None: + def __init__(self, tests: Optional[List[DtbSelfReferential]]) -> None: super().__init__() - self.tests = tests + if tests: + self.tests = tests - def __call__(self, data: bytes, data_offset: int) -> Generator[Tuple[DtbTest, int], None, None]: - for test in self.tests: - for page_offset in range(0, len(data), 0x1000): + def __call__(self, data: bytes, data_offset: int) -> Generator[Tuple[DtbSelfReferential, int], None, None]: + for page_offset in range(0, len(data), 0x1000): + for test in self.tests: result = test(data, data_offset, page_offset) if result is not None: yield (test, result[0]) -class WintelHelper(interfaces.automagic.AutomagicInterface): - """Windows DTB finder based on self-referential pointers. - - This class adheres to the :class:`~volatility3.framework.interfaces.automagic.AutomagicInterface` interface - and both determines the directory table base of an intel layer if one hasn't been specified, and constructs - the intel layer if necessary (for example when reconstructing a pre-existing configuration). - - It will scan for existing TranslationLayers that do not have a DTB using the :class:`PageMapScanner` - """ - priority = 20 - tests = [DtbTest64bit(), DtbTest32bit(), DtbTestPae()] - - def __call__(self, - context: interfaces.context.ContextInterface, - config_path: str, - requirement: interfaces.configuration.RequirementInterface, - progress_callback: constants.ProgressCallback = None) -> None: - useful = [] - sub_config_path = interfaces.configuration.path_join(config_path, requirement.name) - if (isinstance(requirement, requirements.TranslationLayerRequirement) - and requirement.requirements.get("class", False) and requirement.unsatisfied(context, config_path)): - class_req = requirement.requirements["class"] - - for test in self.tests: - if (test.layer_type.__module__ + "." + test.layer_type.__name__ == class_req.config_value( - context, sub_config_path)): - useful.append(test) - - # Determine if a class has been chosen - # Once an appropriate class has been chosen, attempt to determine the page_map_offset value - if ("memory_layer" in requirement.requirements - and not requirement.requirements["memory_layer"].unsatisfied(context, sub_config_path)): - # Only bother getting the DTB if we don't already have one - page_map_offset_path = interfaces.configuration.path_join(sub_config_path, "page_map_offset") - if not context.config.get(page_map_offset_path, None): - physical_layer_name = requirement.requirements["memory_layer"].config_value( - context, sub_config_path) - if not isinstance(physical_layer_name, str): - raise TypeError("Physical layer name is not a string: {}".format(sub_config_path)) - physical_layer = context.layers[physical_layer_name] - # Check lower layer metadata first - if physical_layer.metadata.get('page_map_offset', None): - context.config[page_map_offset_path] = physical_layer.metadata['page_map_offset'] - else: - hits = physical_layer.scan(context, PageMapScanner(useful), progress_callback) - for test, dtb in hits: - context.config[page_map_offset_path] = dtb - break - else: - return None - if isinstance(requirement, interfaces.configuration.ConstructableRequirementInterface): - requirement.construct(context, config_path) - else: - for subreq in requirement.requirements.values(): - self(context, sub_config_path, subreq) - - class WindowsIntelStacker(interfaces.automagic.StackerLayerInterface): stack_order = 40 exclusion_list = ['mac', 'linux'] + # Group these by region so we only run over the data once + test_sets = [("Detecting Self-referential pointer for recent windows", + [DtbSelfRef64bit()], [(0x150000, 0x150000), (0x650000, 0xa0000)]), + ("Older windows fixed location self-referential pointers", + [DtbSelfRefPae(), DtbSelfRef32bit(), DtbSelfRef64bitOldWindows()], [(0x30000, 0x1000000)]) + ] + @classmethod def stack(cls, context: interfaces.context.ContextInterface, @@ -319,7 +185,7 @@ def stack(cls, if arch not in ['Intel32', 'Intel64']: return None # Set the layer type - layer_type = intel.WindowsIntel # type: Type + layer_type: Type = intel.WindowsIntel if arch == 'Intel64': layer_type = intel.WindowsIntel32e elif base_layer.metadata.get('pae', False): @@ -332,44 +198,36 @@ def stack(cls, config_path, "page_map_offset")] = base_layer.metadata['page_map_offset'] layer = layer_type(context, config_path = config_path, name = new_layer_name, metadata = {'os': 'Windows'}) - # Check for the self-referential pointer - if layer is None: - hits = base_layer.scan(context, PageMapScanner(WintelHelper.tests)) - layer = None - config_path = None - for test, dtb in hits: - new_layer_name = context.layers.free_layer_name("IntelLayer") - config_path = interfaces.configuration.path_join("IntelHelper", new_layer_name) - context.config[interfaces.configuration.path_join(config_path, "memory_layer")] = layer_name - context.config[interfaces.configuration.path_join(config_path, "page_map_offset")] = dtb - layer = test.layer_type(context, - config_path = config_path, - name = new_layer_name, - metadata = {'os': 'Windows'}) - break - - # Fall back to a heuristic for finding the Windows DTB - if layer is None: - vollog.debug("Self-referential pointer not in well-known location, moving to recent windows heuristic") - # There is a very high chance that the DTB will live in this narrow segment, assuming we couldn't find it previously + # Self Referential finder + for description, tests, sections in cls.test_sets: + vollog.debug(description) + # There is a very high chance that the DTB will live in these very narrow segments, assuming we couldn't find them previously hits = context.layers[layer_name].scan(context, - PageMapScanner([DtbSelfRef64bit()]), - sections = [(0x1a0000, 0x50000)], + PageMapScanner(tests = tests), + sections = sections, progress_callback = progress_callback) + # Flatten the generator - hits = list(hits) + def sort_by_tests(x): + return tests.index(x[0]), x[1] + + hits = sorted(list(hits), key = sort_by_tests) + if hits: # TODO: Decide which to use if there are multiple options test, page_map_offset = hits[0] + vollog.debug(f"{test.__class__.__name__} test succeeded at {hex(page_map_offset)}") new_layer_name = context.layers.free_layer_name("IntelLayer") config_path = interfaces.configuration.path_join("IntelHelper", new_layer_name) context.config[interfaces.configuration.path_join(config_path, "memory_layer")] = layer_name context.config[interfaces.configuration.path_join(config_path, "page_map_offset")] = page_map_offset # TODO: Need to determine the layer type (chances are high it's x64, hence this default) - layer = layers.intel.WindowsIntel32e(context, - config_path = config_path, - name = new_layer_name, - metadata = {'os': 'Windows'}) + layer = test.layer_type(context, + config_path = config_path, + name = new_layer_name, + metadata = {'os': 'Windows'}) + break + if layer is not None and config_path: vollog.debug("DTB was found at: 0x{:0x}".format(context.config[interfaces.configuration.path_join( config_path, "page_map_offset")])) diff --git a/volatility3/framework/configuration/requirements.py b/volatility3/framework/configuration/requirements.py index 2a8881ace0..a0fb186aec 100644 --- a/volatility3/framework/configuration/requirements.py +++ b/volatility3/framework/configuration/requirements.py @@ -36,13 +36,13 @@ class BooleanRequirement(interfaces.configuration.SimpleTypeRequirement): class IntRequirement(interfaces.configuration.SimpleTypeRequirement): """A requirement type that contains a single integer.""" - instance_type = int # type: ClassVar[Type] + instance_type: ClassVar[Type] = int class StringRequirement(interfaces.configuration.SimpleTypeRequirement): """A requirement type that contains a single unicode string.""" # TODO: Maybe add string length limits? - instance_type = str # type: ClassVar[Type] + instance_type: ClassVar[Type] = str class URIRequirement(StringRequirement): @@ -53,7 +53,7 @@ class URIRequirement(StringRequirement): class BytesRequirement(interfaces.configuration.SimpleTypeRequirement): """A requirement type that contains a byte string.""" - instance_type = bytes # type: ClassVar[Type] + instance_type: ClassVar[Type] = bytes class ListRequirement(interfaces.configuration.RequirementInterface): @@ -83,9 +83,9 @@ def __init__(self, super().__init__(*args, **kwargs) if not issubclass(element_type, interfaces.configuration.BasicTypes): raise TypeError("ListRequirements can only be populated with simple InstanceRequirements") - self.element_type = element_type # type: Type - self.min_elements = min_elements or 0 # type: int - self.max_elements = max_elements # type: Optional[int] + self.element_type: Type = element_type + self.min_elements: int = min_elements or 0 + self.max_elements: Optional[int] = max_elements def unsatisfied(self, context: interfaces.context.ContextInterface, config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: @@ -105,7 +105,7 @@ def unsatisfied(self, context: interfaces.context.ContextInterface, context.config[config_path] = [] if not isinstance(value, list): # TODO: Check this is the correct response for an error - raise TypeError("Unexpected config value found: {}".format(repr(value))) + raise TypeError(f"Unexpected config value found: {repr(value)}") if not (self.min_elements <= len(value)): vollog.log(constants.LOGLEVEL_V, "TypeError - Too few values provided to list option.") return {config_path: self} @@ -208,6 +208,9 @@ def construct(self, context: interfaces.context.ContextInterface, config_path: s num_layers_path = interfaces.configuration.path_join(new_config_path, "number_of_elements") number_of_layers = context.config[num_layers_path] + if not isinstance(number_of_layers, int): + raise TypeError("Number of layers must be an integer") + # Build all the layers that can be built for i in range(number_of_layers): layer_req = self.requirements.get(self.name + str(i), None) @@ -261,20 +264,20 @@ def unsatisfied(self, context: interfaces.context.ContextInterface, value = self.config_value(context, config_path, None) if isinstance(value, str): if value not in context.layers: - vollog.log(constants.LOGLEVEL_V, "IndexError - Layer not found in memory space: {}".format(value)) + vollog.log(constants.LOGLEVEL_V, f"IndexError - Layer not found in memory space: {value}") return {config_path: self} if self.oses and context.layers[value].metadata.get('os', None) not in self.oses: - vollog.log(constants.LOGLEVEL_V, "TypeError - Layer is not the required OS: {}".format(value)) + vollog.log(constants.LOGLEVEL_V, f"TypeError - Layer is not the required OS: {value}") return {config_path: self} if (self.architectures and context.layers[value].metadata.get('architecture', None) not in self.architectures): - vollog.log(constants.LOGLEVEL_V, "TypeError - Layer is not the required Architecture: {}".format(value)) + vollog.log(constants.LOGLEVEL_V, f"TypeError - Layer is not the required Architecture: {value}") return {config_path: self} return {} if value is not None: vollog.log(constants.LOGLEVEL_V, - "TypeError - Translation Layer Requirement only accepts string labels: {}".format(repr(value))) + f"TypeError - Translation Layer Requirement only accepts string labels: {repr(value)}") return {config_path: self} # TODO: check that the space in the context lives up to the requirements for arch/os etc @@ -282,7 +285,7 @@ def unsatisfied(self, context: interfaces.context.ContextInterface, ### NOTE: This validate method has side effects (the dependencies can change)!!! self._validate_class(context, interfaces.configuration.parent_path(config_path)) - vollog.log(constants.LOGLEVEL_V, "IndexError - No configuration provided: {}".format(config_path)) + vollog.log(constants.LOGLEVEL_V, f"IndexError - No configuration provided: {config_path}") return {config_path: self} def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None: @@ -330,7 +333,7 @@ def unsatisfied(self, context: interfaces.context.ContextInterface, value = self.config_value(context, config_path, None) if not isinstance(value, str) and value is not None: vollog.log(constants.LOGLEVEL_V, - "TypeError - SymbolTableRequirement only accepts string labels: {}".format(repr(value))) + f"TypeError - SymbolTableRequirement only accepts string labels: {repr(value)}") return {config_path: self} if value and value in context.symbol_space: # This is an expected situation, so return rather than raise @@ -342,7 +345,7 @@ def unsatisfied(self, context: interfaces.context.ContextInterface, ### NOTE: This validate method has side effects (the dependencies can change)!!! self._validate_class(context, interfaces.configuration.parent_path(config_path)) - vollog.log(constants.LOGLEVEL_V, "Symbol table requirement not yet fulfilled: {}".format(config_path)) + vollog.log(constants.LOGLEVEL_V, f"Symbol table requirement not yet fulfilled: {config_path}") return {config_path: self} def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None: @@ -363,6 +366,8 @@ def construct(self, context: interfaces.context.ContextInterface, config_path: s raise TypeError("Class requirement is not of type ClassRequirement: {}".format( repr(self.requirements["class"]))) cls = self.requirements["class"].cls + if cls is None: + return None node_config = context.config.branch(config_path) for req in cls.get_requirements(): if req.name in node_config.data and req.name != "class": @@ -392,7 +397,7 @@ def __init__(self, super().__init__(name = name, description = description, default = default, optional = optional) if component is None: raise TypeError("Component cannot be None") - self._component = component + self._component: Type[interfaces.configuration.VersionableInterface] = component if version is None: raise TypeError("Version cannot be None") self._version = version @@ -424,3 +429,71 @@ def __init__(self, optional = optional, component = plugin, version = version) + + +class ModuleRequirement(interfaces.configuration.ConstructableRequirementInterface, + interfaces.configuration.ConfigurableRequirementInterface): + + def __init__(self, name: str, description: str = None, default: bool = False, + architectures: Optional[List[str]] = None, optional: bool = False): + super().__init__(name = name, description = description, default = default, optional = optional) + self.add_requirement(TranslationLayerRequirement(name = 'layer_name', architectures = architectures)) + self.add_requirement(SymbolTableRequirement(name = 'symbol_table_name')) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + IntRequirement(name = 'offset'), + ] + + def unsatisfied(self, context: 'interfaces.context.ContextInterface', + config_path: str) -> Dict[str, interfaces.configuration.RequirementInterface]: + """Validate that the value is a valid module""" + config_path = interfaces.configuration.path_join(config_path, self.name) + value = self.config_value(context, config_path, None) + if isinstance(value, str): + if value not in context.modules: + vollog.log(constants.LOGLEVEL_V, f"IndexError - Module not found in context: {value}") + return {config_path: self} + return {} + + if value is not None: + vollog.log(constants.LOGLEVEL_V, + "TypeError - Module Requirement only accepts string labels: {}".format(repr(value))) + return {config_path: self} + + ### NOTE: This validate method has side effects (the dependencies can change)!!! + + self._validate_class(context, interfaces.configuration.parent_path(config_path)) + vollog.log(constants.LOGLEVEL_V, f"IndexError - No configuration provided: {config_path}") + return {config_path: self} + + def construct(self, context: interfaces.context.ContextInterface, config_path: str) -> None: + """Constructs the appropriate layer and adds it based on the class parameter.""" + config_path = interfaces.configuration.path_join(config_path, self.name) + + # Determine the layer name + name = self.name + counter = 2 + while name in context.modules: + name = self.name + str(counter) + counter += 1 + + args = {"context": context, "config_path": config_path, "name": name} + + if any( + [subreq.unsatisfied(context, config_path) for subreq in self.requirements.values() if not subreq.optional]): + return None + + obj = self._construct_class(context, config_path, args) + if obj is not None and isinstance(obj, interfaces.context.ModuleInterface): + context.add_module(obj) + # This should already be done by the _construct_class method + # context.config[config_path] = obj.name + return None + + def build_configuration(self, context: 'interfaces.context.ContextInterface', _: str, + value: Any) -> interfaces.configuration.HierarchicalDict: + """Builds the appropriate configuration for the specified + requirement.""" + return context.modules[value].build_configuration() diff --git a/volatility3/framework/constants/__init__.py b/volatility3/framework/constants/__init__.py index c04fbdb460..23598837b1 100644 --- a/volatility3/framework/constants/__init__.py +++ b/volatility3/framework/constants/__init__.py @@ -38,11 +38,13 @@ """Constant used to delimit table names from type names when referring to a symbol""" # We use the SemVer 2.0.0 versioning scheme -VERSION_MAJOR = 1 # Number of releases of the library with a breaking change +VERSION_MAJOR = 2 # Number of releases of the library with a breaking change VERSION_MINOR = 0 # Number of changes that only add to the interface -VERSION_PATCH = 1 # Number of changes that do not change the interface +VERSION_PATCH = 0 # Number of changes that do not change the interface VERSION_SUFFIX = "" +# TODO: At version 2.0.0, remove the symbol_shift feature + PACKAGE_VERSION = ".".join([str(x) for x in [VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH]]) + VERSION_SUFFIX """The canonical version of the volatility3 package""" @@ -92,3 +94,8 @@ class Parallelism(enum.IntEnum): """The minimum supported version of the Intermediate Symbol Format""" ISF_MINIMUM_DEPRECATED = (3, 9, 9) """The highest version of the ISF that's deprecated (usually higher than supported)""" +OFFLINE = False +"""Whether to go online to retrieve missing/necessary JSON files""" + +REMOTE_ISF_URL = None # 'http://localhost:8000/banners.json' +"""Remote URL to query for a list of ISF addresses""" diff --git a/volatility3/framework/contexts/__init__.py b/volatility3/framework/contexts/__init__.py index c082a29dd2..518215ab4e 100644 --- a/volatility3/framework/contexts/__init__.py +++ b/volatility3/framework/contexts/__init__.py @@ -6,15 +6,18 @@ This has been made an object to allow quick swapping and changing of contexts, to allow a plugin to act on multiple different contexts -without them interfering eith each other. +without them interfering with each other. """ import functools import hashlib -from typing import Callable, Dict, Iterable, List, Optional, Set, Tuple, Union +import logging +from typing import Callable, Iterable, List, Optional, Set, Tuple, Union from volatility3.framework import constants, interfaces, symbols, exceptions from volatility3.framework.objects import templates +vollog = logging.getLogger(__name__) + class Context(interfaces.context.ContextInterface): """Maintains the context within which to construct objects. @@ -33,6 +36,7 @@ def __init__(self) -> None: """Initializes the context.""" super().__init__() self._symbol_space = symbols.SymbolSpace() + self._module_space = ModuleCollection() self._memory = interfaces.layers.LayerContainer() self._config = interfaces.configuration.HierarchicalDict() @@ -50,6 +54,11 @@ def config(self, value: interfaces.configuration.HierarchicalDict) -> None: raise TypeError("Config must be of type HierarchicalDict") self._config = value + @property + def modules(self) -> interfaces.context.ModuleContainer: + """A container for modules loaded in this context""" + return self._module_space + @property def symbol_space(self) -> interfaces.symbols.SymbolSpaceInterface: """The space of all symbols that can be accessed within this @@ -135,17 +144,17 @@ def module(self, size: The size, in bytes, that the module occupys from offset location within the layer named layer_name """ if size: - return SizedModule(self, - module_name = module_name, - layer_name = layer_name, - offset = offset, - size = size, - native_layer_name = native_layer_name) - return Module(self, - module_name = module_name, - layer_name = layer_name, - offset = offset, - native_layer_name = native_layer_name) + return SizedModule.create(self, + module_name = module_name, + layer_name = layer_name, + offset = offset, + size = size, + native_layer_name = native_layer_name) + return Module.create(self, + module_name = module_name, + layer_name = layer_name, + offset = offset, + native_layer_name = native_layer_name) def get_module_wrapper(method: str) -> Callable: @@ -153,9 +162,11 @@ def get_module_wrapper(method: str) -> Callable: def wrapper(self, name: str) -> Callable: if constants.BANG not in name: - name = self._module_name + constants.BANG + name + name = self.symbol_table_name + constants.BANG + name + elif name.startswith(self.symbol_table_name + constants.BANG): + pass else: - raise ValueError("Cannot reference another module when calling {}".format(method)) + raise ValueError(f"Cannot reference another module when calling {method}") return getattr(self._context.symbol_space, method)(name) for entry in ['__annotations__', '__doc__', '__module__', '__name__', '__qualname__']: @@ -168,6 +179,34 @@ def wrapper(self, name: str) -> Callable: class Module(interfaces.context.ModuleInterface): + @classmethod + def create(cls, + context: interfaces.context.ContextInterface, + module_name: str, + layer_name: str, + offset: int, + **kwargs) -> 'Module': + pathjoin = interfaces.configuration.path_join + # Check if config_path is None + free_module_name = context.modules.free_module_name(module_name) + config_path = kwargs.get('config_path', None) + if config_path is None: + config_path = pathjoin('temporary', 'modules', free_module_name) + # Populate the configuration + context.config[pathjoin(config_path, 'layer_name')] = layer_name + context.config[pathjoin(config_path, 'offset')] = offset + # This is important, since the module_name may be changed in case it is already in use + if 'symbol_table_name' not in kwargs: + kwargs['symbol_table_name'] = module_name + for arg in kwargs: + context.config[pathjoin(config_path, arg)] = kwargs.get(arg, None) + # Construct the object + return_val = cls(context, config_path, free_module_name) + context.add_module(return_val) + context.config[config_path] = return_val.name + # Add the module to the context modules collection + return return_val + def object(self, object_type: str, offset: int = None, @@ -232,7 +271,7 @@ def object_from_symbol(self, offset += self._offset if symbol_val.type is None: - raise TypeError("Symbol {} has no associated type".format(symbol_val.name)) + raise TypeError(f"Symbol {symbol_val.name} has no associated type") # Ensure we don't use a layer_name other than the module's, why would anyone do that? if 'layer_name' in kwargs: @@ -245,6 +284,20 @@ def object_from_symbol(self, native_layer_name = native_layer_name or self._native_layer_name, **kwargs) + def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: + """Returns the symbols within this module that live at the specified + absolute offset provided.""" + if size < 0: + raise ValueError("Size must be strictly non-negative") + return list( + self._context.symbol_space.get_symbols_by_location(offset = offset - self._offset, + size = size, + table_name = self.symbol_table_name)) + + @property + def symbols(self): + return self.context.symbol_space[self.symbol_table_name].symbols + get_symbol = get_module_wrapper('get_symbol') get_type = get_module_wrapper('get_type') get_enumeration = get_module_wrapper('get_enumeration') @@ -255,26 +308,11 @@ def object_from_symbol(self, class SizedModule(Module): - def __init__(self, - context: interfaces.context.ContextInterface, - module_name: str, - layer_name: str, - offset: int, - size: int, - symbol_table_name: Optional[str] = None, - native_layer_name: Optional[str] = None) -> None: - super().__init__(context, - module_name = module_name, - layer_name = layer_name, - offset = offset, - native_layer_name = native_layer_name, - symbol_table_name = symbol_table_name) - self._size = size - @property def size(self) -> int: """Returns the size of the module (0 for unknown size)""" - return self._size + size = self.config.get('size', 0) + return size or 0 @property # type: ignore # FIXME: mypy #5107 @functools.lru_cache() @@ -294,22 +332,17 @@ def hash(self) -> str: def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: """Returns the symbols within this module that live at the specified absolute offset provided.""" - if size < 0: - raise ValueError("Size must be strictly non-negative") if offset > self._offset + self.size: return [] - return list( - self._context.symbol_space.get_symbols_by_location(offset = offset - self._offset, - size = size, - table_name = self.symbol_table_name)) + return super().get_symbols_by_absolute_location(offset, size) -class ModuleCollection: +class ModuleCollection(interfaces.context.ModuleContainer): """Class to contain a collection of SizedModules and reason about their contents.""" - def __init__(self, modules: List[SizedModule]) -> None: - self._modules = modules + def __init__(self, modules: Optional[List[interfaces.context.ModuleInterface]] = None) -> None: + super().__init__(modules) def deduplicate(self) -> 'ModuleCollection': """Returns a new deduplicated ModuleCollection featuring no repeated @@ -319,27 +352,27 @@ def deduplicate(self) -> 'ModuleCollection': included in the deduplicated version """ new_modules = [] - seen = set() # type: Set[str] + seen: Set[str] = set() for mod in self._modules: if mod.hash not in seen or mod.size == 0: new_modules.append(mod) seen.add(mod.hash) # type: ignore # FIXME: mypy #5107 return ModuleCollection(new_modules) + def free_module_name(self, prefix: str = "module") -> str: + """Returns an unused module name""" + count = 1 + while prefix + str(count) in self: + count += 1 + return prefix + str(count) + @property - def modules(self) -> Dict[str, List[SizedModule]]: + def modules(self) -> 'ModuleCollection': """A name indexed dictionary of modules using that name in this collection.""" - return self._generate_module_dict(self._modules) - - @classmethod - def _generate_module_dict(cls, modules: List[SizedModule]) -> Dict[str, List[SizedModule]]: - result = {} # type: Dict[str, List[SizedModule]] - for module in modules: - modlist = result.get(module.name, []) - modlist.append(module) - result[module.name] = modlist - return result + vollog.warning( + "This method has been deprecated in favour of the ModuleCollection acting as a dictionary itself") + return self def get_module_symbols_by_absolute_location(self, offset: int, size: int = 0) -> Iterable[Tuple[str, List[str]]]: """Returns a tuple of (module_name, list_of_symbol_names) for each @@ -347,6 +380,19 @@ def get_module_symbols_by_absolute_location(self, offset: int, size: int = 0) -> provided.""" if size < 0: raise ValueError("Size must be strictly non-negative") - for module in self._modules: - if (offset <= module.offset + module.size) and (offset + size >= module.offset): - yield (module.name, module.get_symbols_by_absolute_location(offset, size)) + for module_name in self._modules: + module = self._modules[module_name] + if isinstance(module, SizedModule): + if (offset <= module.offset + module.size) and (offset + size >= module.offset): + yield (module.name, module.get_symbols_by_absolute_location(offset, size)) + + +class ConfigurableModule(Module, interfaces.configuration.ConfigurableInterface): + + def __init__(self, context: interfaces.context.ContextInterface, config_path: str, name: str) -> None: + interfaces.configuration.ConfigurableInterface.__init__(self, context, config_path) + layer_name = self.config['layer_name'] + offset = self.config['offset'] + symbol_table_name = self.config['symbol_table_name'] + interfaces.configuration.ConfigurableInterface.__init__(self, context, config_path) + Module.__init__(self, context, name, layer_name, offset, symbol_table_name, layer_name) diff --git a/volatility3/framework/exceptions.py b/volatility3/framework/exceptions.py index 117933bbd5..a234a353af 100644 --- a/volatility3/framework/exceptions.py +++ b/volatility3/framework/exceptions.py @@ -96,6 +96,17 @@ def __init__(self, unsatisfied: Dict[str, interfaces.configuration.RequirementIn class MissingModuleException(VolatilityException): - def __init__(self, module: str, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) + def __init__(self, module: str, *args) -> None: + super().__init__(*args) self.module = module + + +class OfflineException(VolatilityException): + """Throw when a remote resource is requested but Volatility is in offline mode""" + + def __init__(self, url: str, *args) -> None: + super().__init__(*args) + self._url = url + + def __str__(self): + return f'Volatility 3 is offline: unable to access {self._url}' diff --git a/volatility3/framework/interfaces/automagic.py b/volatility3/framework/interfaces/automagic.py index c6eb2e5cea..c310f5b4a7 100644 --- a/volatility3/framework/interfaces/automagic.py +++ b/volatility3/framework/interfaces/automagic.py @@ -26,7 +26,7 @@ class AutomagicInterface(interfaces.configuration.ConfigurableInterface, metacla Args: context: The context in which to store configuration data that the automagic might populate config_path: Configuration path where the configurable's data under the context's config lives - configurable: The top level configurable whose requirements may need statisfying + configurable: The top level configurable whose requirements may need satisfying progress_callback: An optional function accepting a percentage and optional description to indicate progress during long calculations @@ -82,7 +82,7 @@ def find_requirements(self, A list of tuples containing the config_path, sub_config_path and requirement identifying the unsatisfied `Requirements` """ sub_config_path = interfaces.configuration.path_join(config_path, requirement_root.name) - results = [] # type: List[Tuple[str, interfaces.configuration.RequirementInterface]] + results: List[Tuple[str, interfaces.configuration.RequirementInterface]] = [] recurse = not shortcut if isinstance(requirement_root, requirement_type): if recurse or requirement_root.unsatisfied(context, config_path): @@ -105,7 +105,7 @@ class StackerLayerInterface(metaclass = ABCMeta): stack_order = 0 """The order in which to attempt stacking, the lower the earlier""" - exclusion_list = [] # type: List[str] + exclusion_list: List[str] = [] """The list operating systems/first-level plugin hierarchy that should exclude this stacker""" @classmethod diff --git a/volatility3/framework/interfaces/configuration.py b/volatility3/framework/interfaces/configuration.py index 418763d6d3..7dc046a3e8 100644 --- a/volatility3/framework/interfaces/configuration.py +++ b/volatility3/framework/interfaces/configuration.py @@ -23,9 +23,9 @@ import string import sys from abc import ABCMeta, abstractmethod -from typing import Any, ClassVar, Dict, Generator, Iterator, List, Optional, Type, Union, Tuple +from typing import Any, ClassVar, Dict, Generator, Iterator, List, Optional, Type, Union, Tuple, Set -from volatility3 import classproperty +from volatility3 import classproperty, framework from volatility3.framework import constants, interfaces CONFIG_SEPARATOR = "." @@ -77,10 +77,10 @@ def __init__(self, separator: A custom hierarchy separator (defaults to CONFIG_SEPARATOR) """ if not (isinstance(separator, str) and len(separator) == 1): - raise TypeError("Separator must be a one character string: {}".format(separator)) + raise TypeError(f"Separator must be a one character string: {separator}") self._separator = separator - self._data = {} # type: Dict[str, ConfigSimpleType] - self._subdict = {} # type: Dict[str, 'HierarchicalDict'] + self._data: Dict[str, ConfigSimpleType] = {} + self._subdict: Dict[str, 'HierarchicalDict'] = {} if isinstance(initial_dict, str): initial_dict = json.loads(initial_dict) if isinstance(initial_dict, dict): @@ -88,7 +88,7 @@ def __init__(self, self[k] = v elif initial_dict is not None: raise TypeError( - "Initial_dict must be a dictionary or JSON string containing a dictionary: {}".format(initial_dict)) + f"Initial_dict must be a dictionary or JSON string containing a dictionary: {initial_dict}") def __eq__(self, other): """Define equality between HierarchicalDicts""" @@ -186,12 +186,13 @@ def _sanitize_value(self, value: Any) -> ConfigSimpleType: element_value = self._sanitize_value(element) if isinstance(element_value, list): raise TypeError("Configuration list types cannot contain list types") - new_list.append(element_value) + if element_value is not None: + new_list.append(element_value) return new_list elif value is None: return None else: - raise TypeError("Invalid type stored in configuration") + raise TypeError(f"Invalid type stored in configuration: {type(value)}") def __delitem__(self, key: str) -> None: """Deletes an item from the hierarchical dict.""" @@ -314,16 +315,24 @@ def __init__(self, """ super().__init__() if CONFIG_SEPARATOR in name: - raise ValueError("Name cannot contain the config-hierarchy divider ({})".format(CONFIG_SEPARATOR)) + raise ValueError(f"Name cannot contain the config-hierarchy divider ({CONFIG_SEPARATOR})") self._name = name self._description = description or "" self._default = default self._optional = optional - self._requirements = {} # type: Dict[str, RequirementInterface] + self._requirements: Dict[str, RequirementInterface] = {} def __repr__(self) -> str: return "<" + self.__class__.__name__ + ": " + self.name + ">" + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + for name in self.__dict__: + if other.__dict__.get(name, None) != self.__dict__[name]: + return False + return True + @property def name(self) -> str: """The name of the Requirement. @@ -429,7 +438,7 @@ def unsatisfied(self, context: 'interfaces.context.ContextInterface', class SimpleTypeRequirement(RequirementInterface): """Class to represent a single simple type (such as a boolean, a string, an integer or a series of bytes)""" - instance_type = bool # type: ClassVar[Type] + instance_type: ClassVar[Type] = bool def add_requirement(self, requirement: RequirementInterface): """Always raises a TypeError as instance requirements cannot have @@ -469,8 +478,13 @@ def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self._cls = None + def __eq__(self, other): + # We can just use super because it checks all member of `__dict__` + # This appeases LGTM and does the right thing + return super().__eq__(other) + @property - def cls(self) -> Type: + def cls(self) -> Optional[Type]: """Contains the actual chosen class based on the configuration value's class name.""" return self._cls @@ -515,7 +529,12 @@ class ConstructableRequirementInterface(RequirementInterface): def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.add_requirement(ClassRequirement("class", "Class of the constructable requirement")) - self._current_class_requirements = set() + self._current_class_requirements: Set[Any] = set() + + def __eq__(self, other): + # We can just use super because it checks all member of `__dict__` + # This appeases LGTM and does the right thing + return super().__eq__(other) @abstractmethod def construct(self, context: 'interfaces.context.ContextInterface', config_path: str) -> None: @@ -563,6 +582,9 @@ def _construct_class(self, return None cls = self.requirements["class"].cls + if cls is None: + return None + # These classes all have a name property # We could subclass this out as a NameableInterface, but it seems a little excessive # FIXME: We can't test this, because importing the other interfaces causes all kinds of import loops @@ -598,7 +620,7 @@ def __init__(self, context: 'interfaces.context.ContextInterface', config_path: super().__init__() self._context = context self._config_path = config_path - self._config_cache = None # type: Optional[HierarchicalDict] + self._config_cache: Optional[HierarchicalDict] = None @property def context(self) -> 'interfaces.context.ContextInterface': @@ -707,7 +729,12 @@ class VersionableInterface: All version number should use semantic versioning """ - _version = (0, 0, 0) # type: Tuple[int, int, int] + _version: Tuple[int, int, int] = (0, 0, 0) + _required_framework_version: Tuple[int, int, int] = (0, 0, 0) + + def __init__(self, *args, **kwargs): + framework.require_interface_version(*self._required_framework_version) + super().__init__(*args, **kwargs) @classproperty def version(cls) -> Tuple[int, int, int]: diff --git a/volatility3/framework/interfaces/context.py b/volatility3/framework/interfaces/context.py index 6fede426a8..c52e1aaa59 100644 --- a/volatility3/framework/interfaces/context.py +++ b/volatility3/framework/interfaces/context.py @@ -11,11 +11,12 @@ `object`, which will construct a symbol on a layer at a particular offset. """ +import collections import copy from abc import ABCMeta, abstractmethod -from typing import Optional, Union +from typing import Optional, Union, Dict, List, Iterable -from volatility3.framework import interfaces +from volatility3.framework import interfaces, exceptions class ContextInterface(metaclass = ABCMeta): @@ -44,6 +45,24 @@ def symbol_space(self) -> 'interfaces.symbols.SymbolSpaceInterface': # ## Memory Functions + @property + @abstractmethod + def modules(self) -> 'ModuleContainer': + """Returns the memory object for the context.""" + raise NotImplementedError("ModuleContainer has not been implemented.") + + def add_module(self, module: 'interfaces.context.ModuleInterface'): + """Adds a named module to the context. + + Args: + module: The module to be added to the module object collection + + Raises: + volatility3.framework.exceptions.VolatilityException: if the module is already present, or has + unmet dependencies + """ + self.modules.add_module(module) + @property @abstractmethod def layers(self) -> 'interfaces.layers.LayerContainer': @@ -117,7 +136,7 @@ def module(self, """ -class ModuleInterface(metaclass = ABCMeta): +class ModuleInterface(interfaces.configuration.ConfigurableInterface): """Maintains state concerning a particular loaded module in memory. This object is OS-independent. @@ -125,30 +144,51 @@ class ModuleInterface(metaclass = ABCMeta): def __init__(self, context: ContextInterface, - module_name: str, - layer_name: str, - offset: int, - symbol_table_name: Optional[str] = None, - native_layer_name: Optional[str] = None) -> None: + config_path: str, + name: str) -> None: """Constructs a new os-independent module. Args: context: The context within which this module will exist - module_name: The name of the module - layer_name: The layer within the context in which the module exists - offset: The offset at which the module exists in the layer - symbol_table_name: The name of an associated symbol table - native_layer_name: The default native layer for objects constructed by the module + config_path: The path within the context's configuration tree + name: The name of the module """ - self._context = context - self._module_name = module_name - self._layer_name = layer_name - self._offset = offset - self._native_layer_name = None - if native_layer_name: - self._native_layer_name = native_layer_name - self.symbol_table_name = symbol_table_name or self._module_name - super().__init__() + super().__init__(context, config_path) + self._module_name = name + + @property + def _layer_name(self) -> str: + return self.config['layer_name'] + + @property + def _offset(self) -> int: + return self.config['offset'] + + @property + def _native_layer_name(self) -> str: + return self.config.get('native_layer_name', self._layer_name) + + @property + def _symbol_table_name(self) -> str: + return self.config.get('symbol_table_name', self._module_name) + + def build_configuration(self) -> 'interfaces.configuration.HierarchicalDict': + """Builds the configuration dictionary for this specific Module""" + + config = super().build_configuration() + + config['offset'] = self.config['offset'] + subconfigs = {'symbol_table_name': self.context.symbol_space[self.symbol_table_name].build_configuration(), + 'layer_name': self.context.layers[self.layer_name].build_configuration()} + + if self.layer_name != self._native_layer_name: + subconfigs['native_layer_name'] = self.context.layers[self._native_layer_name].build_configuration() + + for subconfig in subconfigs: + for req in subconfigs[subconfig]: + config[interfaces.configuration.path_join(subconfig, req)] = subconfigs[subconfig][req] + + return config @property def name(self) -> str: @@ -171,6 +211,11 @@ def context(self) -> ContextInterface: """Context that the module uses.""" return self._context + @property + def symbol_table_name(self) -> str: + """The name of the symbol table associated with this module""" + return self._symbol_table_name + @abstractmethod def object(self, object_type: str, @@ -209,20 +254,82 @@ def object_from_symbol(self, The constructed object """ + def get_absolute_symbol_address(self, name: str) -> int: + """Returns the absolute address of the symbol within this module""" + symbol = self.get_symbol(name) + return self.offset + symbol.address + def get_type(self, name: str) -> 'interfaces.objects.Template': - """Returns a type from the module.""" + """Returns a type from the module's symbol table.""" def get_symbol(self, name: str) -> 'interfaces.symbols.SymbolInterface': - """Returns a symbol from the module.""" + """Returns a symbol object from the module's symbol table.""" def get_enumeration(self, name: str) -> 'interfaces.objects.Template': - """Returns an enumeration from the module.""" + """Returns an enumeration from the module's symbol table.""" def has_type(self, name: str) -> bool: - """Determines whether a type is present in the module.""" + """Determines whether a type is present in the module's symbol table.""" def has_symbol(self, name: str) -> bool: - """Determines whether a symbol is present in the module.""" + """Determines whether a symbol is present in the module's symbol table.""" def has_enumeration(self, name: str) -> bool: - """Determines whether an enumeration is present in the module.""" + """Determines whether an enumeration is present in the module's symbol table.""" + + def symbols(self) -> List: + """Lists the symbols contained in the symbol table for this module""" + + def get_symbols_by_absolute_location(self, offset: int, size: int = 0) -> List[str]: + """Returns the symbols within table_name (or this module if not specified) that live at the specified + absolute offset provided.""" + + +class ModuleContainer(collections.abc.Mapping): + """Container for multiple layers of data.""" + + def __init__(self, modules: Optional[List[ModuleInterface]] = None) -> None: + self._modules: Dict[str, ModuleInterface] = {} + if modules is not None: + for module in modules: + self.add_module(module) + + def __eq__(self, other): + return dict(self) == dict(other) + + def add_module(self, module: ModuleInterface) -> None: + """Adds a module to the module collection + + This will throw an exception if the required dependencies are not met + + Args: + module: the module to add to the list of modules (based on module.name) + """ + if module.name in self._modules: + raise exceptions.VolatilityException(f"Module already exists: {module.name}") + self._modules[module.name] = module + + def __delitem__(self, name: str) -> None: + """Removes a module from the module list""" + del self._modules[name] + + def __getitem__(self, name: str) -> ModuleInterface: + """Returns the layer of specified name.""" + return self._modules[name] + + def __len__(self) -> int: + return len(self._modules) + + def __iter__(self): + return iter(self._modules) + + def free_module_name(self, prefix: str = "module") -> str: + """Returns an unused table name to ensure no collision occurs when + inserting a symbol table.""" + + def get_modules_by_symbol_tables(self, symbol_table: str) -> Iterable[str]: + """Returns the modules which use the specified symbol table name""" + for module_name in self._modules: + module = self._modules[module_name] + if module.symbol_table_name == symbol_table: + yield module_name diff --git a/volatility3/framework/interfaces/layers.py b/volatility3/framework/interfaces/layers.py index 56426331e3..a42282c394 100644 --- a/volatility3/framework/interfaces/layers.py +++ b/volatility3/framework/interfaces/layers.py @@ -54,12 +54,14 @@ class ScannerInterface(interfaces.configuration.VersionableInterface, metaclass """ thread_safe = False + _required_framework_version = (2, 0, 0) + def __init__(self) -> None: super().__init__() self.chunk_size = 0x1000000 # Default to 16Mb chunks self.overlap = 0x1000 # A page of overlap by default - self._context = None # type: Optional[interfaces.context.ContextInterface] - self._layer_name = None # type: Optional[str] + self._context: Optional[interfaces.context.ContextInterface] = None + self._layer_name: Optional[str] = None @property def context(self) -> Optional['interfaces.context.ContextInterface']: @@ -99,10 +101,7 @@ class DataLayerInterface(interfaces.configuration.ConfigurableInterface, metacla accesses a data source and exposes it within volatility. """ - _direct_metadata = collections.ChainMap({}, { - 'architecture': 'Unknown', - 'os': 'Unknown' - }) # type: collections.ChainMap + _direct_metadata: Mapping = {'architecture': 'Unknown', 'os': 'Unknown'} def __init__(self, context: 'interfaces.context.ContextInterface', @@ -111,8 +110,7 @@ def __init__(self, metadata: Optional[Dict[str, Any]] = None) -> None: super().__init__(context, config_path) self._name = name - if metadata: - self._direct_metadata.update(metadata) + self._metadata = metadata or {} # Standard attributes @@ -231,7 +229,7 @@ def scan(self, sections = list(self._coalesce_sections(sections)) try: - progress = DummyProgress() # type: ProgressValue + progress: ProgressValue = DummyProgress() scan_iterator = functools.partial(self._scan_iterator, scanner, sections) scan_metric = self._scan_metric(scanner, sections) if not scanner.thread_safe or constants.PARALLELISM == constants.Parallelism.Off: @@ -240,11 +238,11 @@ def scan(self, for value in scan_iterator(): if progress_callback: progress_callback(scan_metric(progress.value), - "Scanning {} using {}".format(self.name, scanner.__class__.__name__)) + f"Scanning {self.name} using {scanner.__class__.__name__}") yield from scan_chunk(value) else: progress = multiprocessing.Manager().Value("Q", 0) - parallel_module = multiprocessing # type: types.ModuleType + parallel_module: types.ModuleType = multiprocessing if constants.PARALLELISM == constants.Parallelism.Threading: progress = DummyProgress() parallel_module = threading @@ -255,7 +253,7 @@ def scan(self, if progress_callback: # Run the progress_callback progress_callback(scan_metric(progress.value), - "Scanning {} using {}".format(self.name, scanner.__class__.__name__)) + f"Scanning {self.name} using {scanner.__class__.__name__}") # Ensures we don't burn CPU cycles going round in a ready waiting loop # without delaying the user too long between progress updates/results result.wait(0.1) @@ -263,14 +261,14 @@ def scan(self, yield from result_value except Exception as e: # We don't care the kind of exception, so catch and report on everything, yielding nothing further - vollog.debug("Scan Failure: {}".format(str(e))) + vollog.debug(f"Scan Failure: {str(e)}") vollog.log(constants.LOGLEVEL_VVV, "\n".join(traceback.TracebackException.from_exception(e).format(chain = True))) def _coalesce_sections(self, sections: Iterable[Tuple[int, int]]) -> Iterable[Tuple[int, int]]: """Take a list of (start, length) sections and coalesce any adjacent sections.""" - result = [] # type: List[Tuple[int, int]] + result: List[Tuple[int, int]] = [] position = 0 for (start, length) in sorted(sections): if result and start <= position: @@ -328,6 +326,9 @@ def _scan_chunk(self, scanner: 'ScannerInterface', progress: 'ProgressValue', vollog.debug("Invalid address in layer {} found scanning {} at address {:x}".format( layer_name, self.name, address)) + if len(data) > scanner.chunk_size + scanner.overlap: + vollog.debug(f"Scan chunk too large: {hex(len(data))}") + progress.value = chunk_end return list(scanner(data, chunk_end - len(data))) @@ -357,7 +358,7 @@ def build_configuration(self) -> interfaces.configuration.HierarchicalDict: def metadata(self) -> Mapping: """Returns a ReadOnly copy of the metadata published by this layer.""" maps = [self.context.layers[layer_name].metadata for layer_name in self.dependencies] - return interfaces.objects.ReadOnlyMapping(collections.ChainMap({}, self._direct_metadata, *maps)) + return interfaces.objects.ReadOnlyMapping(collections.ChainMap(self._metadata, self._direct_metadata, *maps)) class TranslationLayerInterface(DataLayerInterface, metaclass = ABCMeta): @@ -424,13 +425,13 @@ def read(self, offset: int, length: int, pad: bool = False) -> bytes: """Reads an offset for length bytes and returns 'bytes' (not 'str') of length size.""" current_offset = offset - output = b'' # type: bytes + output: bytes = b'' for (layer_offset, sublength, mapped_offset, mapped_length, layer) in self.mapping(offset, length, ignore_errors = pad): if not pad and layer_offset > current_offset: raise exceptions.InvalidAddressException( - self.name, current_offset, "Layer {} cannot map offset: {}".format(self.name, current_offset)) + self.name, current_offset, f"Layer {self.name} cannot map offset: {current_offset}") elif layer_offset > current_offset: output += b"\x00" * (layer_offset - current_offset) current_offset = layer_offset @@ -453,7 +454,7 @@ def write(self, offset: int, value: bytes) -> None: for (layer_offset, sublength, mapped_offset, mapped_length, layer) in self.mapping(offset, length): if layer_offset > current_offset: raise exceptions.InvalidAddressException( - self.name, current_offset, "Layer {} cannot map offset: {}".format(self.name, current_offset)) + self.name, current_offset, f"Layer {self.name} cannot map offset: {current_offset}") value_chunk = value[layer_offset - offset:layer_offset - offset + sublength] new_data = self._encode_data(layer, mapped_offset, layer_offset, value_chunk) @@ -474,46 +475,66 @@ def _scan_iterator(self, assumed to have no holes """ for (section_start, section_length) in sections: - # For each section, split it into scan size chunks - for chunk_start in range(section_start, section_start + section_length, scanner.chunk_size): - # Shorten it, if we're at the end of the section - chunk_length = min(section_start + section_length - chunk_start, scanner.chunk_size + scanner.overlap) - - # Prev offset keeps track of the end of the previous subchunk - prev_offset = chunk_start - output = [] # type: List[Tuple[str, int, int]] - - # We populate the response based on subchunks that may be mapped all over the place - for mapped in self.mapping(chunk_start, chunk_length, ignore_errors = True): - # We don't bother with the other data in case the data's been processed by a lower layer - offset, sublength, mapped_offset, mapped_length, layer_name = mapped - - # We need to check if the offset is next to the end of the last one (contiguous) - if offset != prev_offset: - # Only yield if we've accumulated output - if len(output): - # Yield all the (joined) items so far - # and the ending point of that subchunk (where we'd gotten to previously) - yield output, prev_offset + output: List[Tuple[str, int, int]] = [] + + # Hold the offsets of each chunk (including how much has been filled) + chunk_start = chunk_position = 0 + + # For each section, find out which bits of its exists and where they map to + # This is faster than cutting the entire space into scan_chunk sized blocks and then + # finding out what exists (particularly if most of the space isn't mapped) + for mapped in self.mapping(section_start, section_length, ignore_errors = True): + offset, sublength, mapped_offset, mapped_length, layer_name = mapped + + # Setup the variables for this block + block_start = offset + block_end = offset + sublength + + # Setup the necessary bits for non-linear mappings + # For linear we give one layer down and mapped offsets (therefore the conversion) + # This saves an tiny amount of time not have to redo lookups we've already done + # For non-linear layers, we give the layer name and the offset in the layer name + # so that the read/conversion occurs properly + conversion = mapped_offset - offset if linear else 0 + return_name = layer_name if linear else self.name + + # If this isn't contiguous, start a new chunk + if chunk_position < block_start: + yield output, chunk_position + output = [] + chunk_start = chunk_position = block_start + + # Halfway through a chunk, finish the chunk, then take more + if chunk_position != chunk_start: + chunk_size = min(chunk_position - chunk_start, scanner.chunk_size + scanner.overlap) + output += [(return_name, chunk_position + conversion, chunk_size)] + chunk_start = chunk_position + chunk_size + chunk_position = chunk_start + + # Pack chunks, if we're enter the loop (starting a new chunk) and there's already chunk there, ship it + for chunk_start in range(chunk_position, block_end, scanner.chunk_size): + if output: + yield output, chunk_position output = [] + chunk_position = chunk_start + # Take from chunk_position as far as far as the block can go, + # or as much left of a scanner chunk as we can + chunk_size = min(block_end - chunk_position, + scanner.chunk_size + scanner.overlap - (chunk_position - chunk_start)) + output += [(return_name, chunk_position + conversion, chunk_size)] + chunk_start = chunk_position + chunk_size + chunk_position = chunk_start - # Shift the marker up to the end of what we just received and add it to the output - prev_offset = offset + sublength - - if not linear: - output += [(self.name, offset, sublength)] - else: - output += [(layer_name, mapped_offset, mapped_length)] - # If there's still output left, output it - if len(output): - yield output, prev_offset + # Ship anything that might be left + if output: + yield output, chunk_position class LayerContainer(collections.abc.Mapping): """Container for multiple layers of data.""" def __init__(self) -> None: - self._layers = {} # type: Dict[str, DataLayerInterface] + self._layers: Dict[str, DataLayerInterface] = {} def read(self, layer: str, offset: int, length: int, pad: bool = False) -> bytes: """Reads from a particular layer at offset for length bytes. @@ -547,12 +568,12 @@ def add_layer(self, layer: DataLayerInterface) -> None: layer: the layer to add to the list of layers (based on layer.name) """ if layer.name in self._layers: - raise exceptions.LayerException(layer.name, "Layer already exists: {}".format(layer.name)) + raise exceptions.LayerException(layer.name, f"Layer already exists: {layer.name}") if isinstance(layer, TranslationLayerInterface): missing_list = [sublayer for sublayer in layer.dependencies if sublayer not in self._layers] if missing_list: raise exceptions.LayerException( - layer.name, "Layer {} has unmet dependencies: {}".format(layer.name, ", ".join(missing_list))) + layer.name, f"Layer {layer.name} has unmet dependencies: {', '.join(missing_list)}") self._layers[layer.name] = layer def del_layer(self, name: str) -> None: @@ -568,7 +589,7 @@ def del_layer(self, name: str) -> None: if depend_list: raise exceptions.LayerException( self._layers[layer].name, - "Layer {} is depended upon: {}".format(self._layers[layer].name, ", ".join(depend_list))) + f"Layer {self._layers[layer].name} is depended upon: {', '.join(depend_list)}") self._layers[name].destroy() del self._layers[name] @@ -585,9 +606,9 @@ def free_layer_name(self, prefix: str = "layer") -> str: if prefix not in self: return prefix count = 1 - while "{}_{}".format(prefix, count) in self: + while f"{prefix}_{count}" in self: count += 1 - return "{}_{}".format(prefix, count) + return f"{prefix}_{count}" def __getitem__(self, name: str) -> DataLayerInterface: """Returns the layer of specified name.""" diff --git a/volatility3/framework/interfaces/objects.py b/volatility3/framework/interfaces/objects.py index 2f794b5655..811327094a 100644 --- a/volatility3/framework/interfaces/objects.py +++ b/volatility3/framework/interfaces/objects.py @@ -31,7 +31,7 @@ def __getattr__(self, attr: str) -> Any: return super().__getattribute__(attr) if attr in self._dict: return self._dict[attr] - raise AttributeError("Object has no attribute: {}.{}".format(self.__class__.__name__, attr)) + raise AttributeError(f"Object has no attribute: {self.__class__.__name__}.{attr}") def __getitem__(self, name: str) -> Any: """Returns the item requested.""" @@ -141,10 +141,10 @@ def get_symbol_table_name(self) -> str: KeyError: If the table_name is not valid within the object's context """ if constants.BANG not in self.vol.type_name: - raise ValueError("Unable to determine table for symbol: {}".format(self.vol.type_name)) + raise ValueError(f"Unable to determine table for symbol: {self.vol.type_name}") table_name = self.vol.type_name[:self.vol.type_name.index(constants.BANG)] if table_name not in self._context.symbol_space: - raise KeyError("Symbol table not found in context's symbol_space for symbol: {}".format(self.vol.type_name)) + raise KeyError(f"Symbol table not found in context's symbol_space for symbol: {self.vol.type_name}") return table_name def cast(self, new_type_name: str, **additional) -> 'ObjectInterface': @@ -212,9 +212,9 @@ class VolTemplateProxy(metaclass = abc.ABCMeta): takes a template since the templates may contain the necessary data about the yet-to-be-constructed object. It allows objects to control how their templates respond without needing to write - new templates for each and every potental object type. + new templates for each and every potential object type. """ - _methods = [] # type: List[str] + _methods: List[str] = [] @classmethod @abc.abstractmethod @@ -231,14 +231,14 @@ def children(cls, template: 'Template') -> List['Template']: @abc.abstractmethod def replace_child(cls, template: 'Template', old_child: 'Template', new_child: 'Template') -> None: """Substitutes the old_child for the new_child.""" - raise KeyError("Template does not contain any children to replace: {}".format(template.vol.type_name)) + raise KeyError(f"Template does not contain any children to replace: {template.vol.type_name}") @classmethod @abc.abstractmethod def relative_child_offset(cls, template: 'Template', child: str) -> int: """Returns the relative offset from the head of the parent data to the child member.""" - raise KeyError("Template does not contain any children: {}".format(template.vol.type_name)) + raise KeyError(f"Template does not contain any children: {template.vol.type_name}") @classmethod @abc.abstractmethod @@ -275,7 +275,7 @@ def __init__(self, type_name: str, **arguments) -> None: """Stores the keyword arguments for later object creation.""" # Allow the updating of template arguments whilst still in template form super().__init__() - empty_dict = {} # type: Dict[str, Any] + empty_dict: Dict[str, Any] = {} self._vol = collections.ChainMap(empty_dict, arguments, {'type_name': type_name}) @property @@ -330,7 +330,7 @@ def __getattr__(self, attr: str) -> Any: if attr != '_vol': if attr in self._vol: return self._vol[attr] - raise AttributeError("{} object has no attribute {}".format(self.__class__.__name__, attr)) + raise AttributeError(f"{self.__class__.__name__} object has no attribute {attr}") def __call__(self, context: 'interfaces.context.ContextInterface', object_info: ObjectInformation) -> ObjectInterface: diff --git a/volatility3/framework/interfaces/plugins.py b/volatility3/framework/interfaces/plugins.py index 5649629ca1..983232cf0f 100644 --- a/volatility3/framework/interfaces/plugins.py +++ b/volatility3/framework/interfaces/plugins.py @@ -14,7 +14,6 @@ from abc import ABCMeta, abstractmethod from typing import List, Tuple, Type -from volatility3 import framework from volatility3.framework import exceptions, constants, interfaces vollog = logging.getLogger(__name__) @@ -65,7 +64,7 @@ def __exit__(self, exc_type, exc_value, traceback): if exc_type is None and exc_value is None and traceback is None: self.close() else: - vollog.warning("File {} could not be written: {}".format(self._preferred_filename, str(exc_value))) + vollog.warning(f"File {self._preferred_filename} could not be written: {str(exc_value)}") self.close() @@ -95,7 +94,7 @@ class PluginInterface(interfaces.configuration.ConfigurableInterface, """ # Be careful with inheritance around this (We default to requiring a version which doesn't exist, so it must be set) - _required_framework_version = (0, 0, 0) # type: Tuple[int, int, int] + _required_framework_version: Tuple[int, int, int] = (0, 0, 0) """The _version variable is a quick way for plugins to define their current interface, it should follow SemVer rules""" def __init__(self, @@ -121,9 +120,7 @@ def __init__(self, if requirement.name not in self.config: self.config[requirement.name] = requirement.default - self._file_handler = FileHandlerInterface # type: Type[FileHandlerInterface] - - framework.require_interface_version(*self._required_framework_version) + self._file_handler: Type[FileHandlerInterface] = FileHandlerInterface @property def open(self): diff --git a/volatility3/framework/interfaces/renderers.py b/volatility3/framework/interfaces/renderers.py index 54b9f922fc..7f80425a4c 100644 --- a/volatility3/framework/interfaces/renderers.py +++ b/volatility3/framework/interfaces/renderers.py @@ -38,7 +38,7 @@ def render(self, grid: 'TreeGrid') -> None: class ColumnSortKey(metaclass = ABCMeta): - ascending = True # type: bool + ascending: bool = True @abstractmethod def __call__(self, values: List[Any]) -> Any: @@ -129,7 +129,7 @@ class TreeGrid(object, metaclass = ABCMeta): and to create cycles. """ - base_types = (int, str, float, bytes, datetime.datetime, Disassembly) # type: ClassVar[Tuple] + base_types: ClassVar[Tuple] = (int, str, float, bytes, datetime.datetime, Disassembly) def __init__(self, columns: ColumnsType, generator: Generator) -> None: """Constructs a TreeGrid object using a specific set of columns. @@ -215,6 +215,6 @@ def visit(self, Args: node: The initial node to be visited function: The visitor to apply to the nodes under the initial node - initial_accumulator: An accumulator that allows data to be transfered between one visitor call to the next + initial_accumulator: An accumulator that allows data to be transferred between one visitor call to the next sort_key: Information about the sort order of columns in order to determine the ordering of results """ diff --git a/volatility3/framework/interfaces/symbols.py b/volatility3/framework/interfaces/symbols.py index 8f427d2330..b271de412e 100644 --- a/volatility3/framework/interfaces/symbols.py +++ b/volatility3/framework/interfaces/symbols.py @@ -4,8 +4,8 @@ """Symbols provide structural information about a set of bytes.""" import bisect import collections.abc -from abc import abstractmethod, ABC -from typing import Any, Dict, Iterable, List, Optional, Tuple, Type, Mapping +from abc import ABC, abstractmethod +from typing import Any, Dict, Iterable, List, Mapping, Optional, Tuple, Type from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.configuration import requirements @@ -31,7 +31,7 @@ def __init__(self, """ self._name = name if constants.BANG in self._name: - raise ValueError("Symbol names cannot contain the symbol differentiator ({})".format(constants.BANG)) + raise ValueError(f"Symbol names cannot contain the symbol differentiator ({constants.BANG})") # Scope can be added at a later date self._location = None @@ -96,7 +96,7 @@ def __init__(self, table_mapping = {} self.table_mapping = table_mapping self._native_types = native_types - self._sort_symbols = [] # type: List[Tuple[int, str]] + self._sort_symbols: List[Tuple[int, str]] = [] # Set any provisioned class_types if class_types: @@ -303,10 +303,9 @@ def build_configuration(self) -> 'configuration.HierarchicalDict': @classmethod def get_requirements(cls) -> List[RequirementInterface]: return super().get_requirements() + [ - requirements.IntRequirement(name = 'symbol_shift', description = 'Symbol Shift', optional = False), - requirements.IntRequirement( - name = 'symbol_mask', description = 'Address mask for symbols', optional = True, default = 0), - ] + requirements.IntRequirement(name = 'symbol_mask', description = 'Address mask for symbols', optional = True, + default = 0), + ] class NativeTableInterface(BaseSymbolTableInterface): diff --git a/volatility3/framework/layers/avml.py b/volatility3/framework/layers/avml.py new file mode 100644 index 0000000000..acc4493f4d --- /dev/null +++ b/volatility3/framework/layers/avml.py @@ -0,0 +1,136 @@ +"""Functions that read AVML files. + +The user of the file doesn't have to worry about the compression, +but random access is not allowed.""" +import logging +import struct +from typing import Tuple, List, Optional + +from volatility3.framework import exceptions, interfaces, constants +from volatility3.framework.layers import segmented + +vollog = logging.getLogger(__name__) + +try: + import snappy + + HAS_SNAPPY = True +except ImportError: + HAS_SNAPPY = False + + +class AVMLLayer(segmented.NonLinearlySegmentedLayer): + """A Lime format TranslationLayer. + + Lime is generally used to store physical memory images where there + are large holes in the physical layer + """ + + def __init__(self, *args, **kwargs): + self._compressed = {} + super().__init__(*args, **kwargs) + + @classmethod + def _check_header(cls, layer: interfaces.layers.DataLayerInterface): + header_structure = " None: + base_layer = self.context.layers[self._base_layer] + offset = base_layer.minimum_address + while offset + 4 < base_layer.maximum_address: + avml_header_structure = " Tuple[ + List[Tuple[int, int, int, int, bool]], int]: + """ + Reads a framed-format snappy stream + + Args: + data: The stream to read + expected_length: How big the decompressed stream is expected to be (termination limit) + + Returns: + (offset, mapped_offset, length, mapped_length, compressed) relative to the data chunk (ie, not relative to the file start) + """ + segments = [] + decompressed_len = 0 + offset = 0 + crc_len = 4 + frame_header_struct = '> 8 + if frame_type == 0xff: + if data[offset + frame_header_len:offset + frame_header_len + frame_size] != b'sNaPpY': + raise ValueError(f"Snappy header missing at offset: {offset}") + elif frame_type in [0x00, 0x01]: + # CRC + (Un)compressed data + mapped_start = offset + frame_header_len + # frame_crc = data[mapped_start: mapped_start + crc_len] + frame_data = data[mapped_start + crc_len: mapped_start + frame_size] + if frame_type == 0x00: + # Compressed data + frame_data = snappy.decompress(frame_data) + # TODO: Verify CRC + segments.append((decompressed_len, mapped_start + crc_len, len(frame_data), frame_size - crc_len, + frame_type == 0x00)) + decompressed_len += len(frame_data) + elif frame_type in range(0x2, 0x80): + # Unskippable + raise exceptions.LayerException(f"Unskippable chunk of type {frame_type} found: {offset}") + offset += frame_header_len + frame_size + return segments, offset + + def _decode_data(self, data: bytes, mapped_offset: int, offset: int, output_length: int) -> bytes: + start_offset, _, _, _ = self._find_segment(offset) + if self._compressed[mapped_offset]: + decoded_data = snappy.decompress(data) + else: + decoded_data = data + decoded_data = decoded_data[offset - start_offset:] + decoded_data = decoded_data[:output_length] + return decoded_data + + +class AVMLStacker(interfaces.automagic.StackerLayerInterface): + stack_order = 10 + + @classmethod + def stack(cls, + context: interfaces.context.ContextInterface, + layer_name: str, + progress_callback: constants.ProgressCallback = None) -> Optional[interfaces.layers.DataLayerInterface]: + try: + AVMLLayer._check_header(context.layers[layer_name]) + except exceptions.LayerException: + return None + new_name = context.layers.free_layer_name("AVMLLayer") + context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name + return AVMLLayer(context, new_name, new_name) diff --git a/volatility3/framework/layers/crash.py b/volatility3/framework/layers/crash.py index 8909eceb86..c690c8d8fa 100644 --- a/volatility3/framework/layers/crash.py +++ b/volatility3/framework/layers/crash.py @@ -1,7 +1,6 @@ -# This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 +# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - import logging import struct from typing import Tuple, Optional @@ -9,6 +8,7 @@ from volatility3.framework import constants, exceptions, interfaces from volatility3.framework.layers import segmented from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows.extensions import crash vollog = logging.getLogger(__name__) @@ -19,7 +19,6 @@ class WindowsCrashDumpFormatException(exceptions.LayerException): class WindowsCrashDump32Layer(segmented.SegmentedLayer): """A Windows crash format TranslationLayer. - This TranslationLayer supports Microsoft complete memory dump files. It currently does not support kernel or small memory dump files. """ @@ -30,7 +29,7 @@ class WindowsCrashDump32Layer(segmented.SegmentedLayer): VALIDDUMP = 0x504d5544 crashdump_json = 'crash' - supported_dumptypes = [0x01] + supported_dumptypes = [0x01, 0x05] # we need 0x5 for 32-bit bitmaps dump_header_name = '_DUMP_HEADER' _magic_struct = struct.Struct(' interfaces.objects.ObjectInterface: + return self.context.object(self._crash_table_name + constants.BANG + self.dump_header_name, + offset = 0, + layer_name = self._base_layer) + + def get_summary_header(self) -> interfaces.objects.ObjectInterface: + return self.context.object(self._crash_common_table_name + constants.BANG + "_SUMMARY_DUMP", + offset = 0x1000 * self.headerpages, + layer_name = self._base_layer) + def _load_segments(self) -> None: """Loads up the segments from the meta_layer.""" - header = self.context.object(self._crash_table_name + constants.BANG + self.dump_header_name, - offset = 0, - layer_name = self._base_layer) segments = [] - offset = self.headerpages - header.PhysicalMemoryBlockBuffer.Run.count = header.PhysicalMemoryBlockBuffer.NumberOfRuns - for x in header.PhysicalMemoryBlockBuffer.Run: - segments.append((x.BasePage * 0x1000, offset * 0x1000, x.PageCount * 0x1000, x.PageCount * 0x1000)) - # print("Segments {:x} {:x} {:x}".format(x.BasePage * 0x1000, - # offset * 0x1000, - # x.PageCount * 0x1000)) - offset += x.PageCount + if self.dump_type == 0x1: + header = self.context.object(self._crash_table_name + constants.BANG + self.dump_header_name, + offset = 0, + layer_name = self._base_layer) + + offset = self.headerpages + header.PhysicalMemoryBlockBuffer.Run.count = header.PhysicalMemoryBlockBuffer.NumberOfRuns + for run in header.PhysicalMemoryBlockBuffer.Run: + segments.append( + (run.BasePage * 0x1000, offset * 0x1000, run.PageCount * 0x1000, run.PageCount * 0x1000)) + offset += run.PageCount + + elif self.dump_type == 0x05: + summary_header = self.get_summary_header() + first_bit = None # First bit in a run + first_offset = 0 # File offset of first bit + last_bit_seen = 0 # Most recent bit processed + offset = summary_header.HeaderSize # Size of file headers + buffer_char = summary_header.get_buffer_char() + buffer_long = summary_header.get_buffer_long() + + for outer_index in range(0, ((summary_header.BitmapSize + 31) // 32)): + if buffer_long[outer_index] == 0: + if first_bit is not None: + last_bit = ((outer_index - 1) * 32) + 31 + segment_length = (last_bit - first_bit + 1) * 0x1000 + segments.append((first_bit * 0x1000, first_offset, segment_length, segment_length)) + first_bit = None + elif buffer_long[outer_index] == 0xFFFFFFFF: + if first_bit is None: + first_offset = offset + first_bit = outer_index * 32 + offset = offset + (32 * 0x1000) + else: + for inner_index in range(0, 32): + bit_addr = outer_index * 32 + inner_index + if (buffer_char[bit_addr >> 3] >> (bit_addr & 0x7)) & 1: + if first_bit is None: + first_offset = offset + first_bit = bit_addr + offset = offset + 0x1000 + else: + if first_bit is not None: + segment_length = ((bit_addr - 1) - first_bit + 1) * 0x1000 + segments.append((first_bit * 0x1000, first_offset, segment_length, segment_length)) + first_bit = None + last_bit_seen = (outer_index * 32) + 31 + + if first_bit is not None: + segment_length = (last_bit_seen - first_bit + 1) * 0x1000 + segments.append((first_bit * 0x1000, first_offset, segment_length, segment_length)) + else: + vollog.log(constants.LOGLEVEL_VVVV, f"unsupported dump format 0x{self.dump_type:x}") + raise WindowsCrashDumpFormatException(self.name, f"unsupported dump format 0x{self.dump_type:x}") if len(segments) == 0: - raise WindowsCrashDumpFormatException(self.name, "No Crash segments defined in {}".format(self._base_layer)) + raise WindowsCrashDumpFormatException(self.name, f"No Crash segments defined in {self._base_layer}") + else: + # report the segments for debugging. this is valuable for dev/troubleshooting but + # not important enough for a dedicated plugin. + for idx, (start_position, mapped_offset, length, _) in enumerate(segments): + vollog.log( + constants.LOGLEVEL_VVVV, + "Segment {}: Position {:#x} Offset {:#x} Length {:#x}".format(idx, start_position, mapped_offset, + length)) self._segments = segments @@ -99,22 +167,21 @@ def check_header(cls, base_layer: interfaces.layers.DataLayerInterface, offset: header_data = base_layer.read(offset, cls._magic_struct.size) except exceptions.InvalidAddressException: raise WindowsCrashDumpFormatException(base_layer.name, - "Crashdump header not found at offset {}".format(offset)) + f"Crashdump header not found at offset {offset}") (signature, validdump) = cls._magic_struct.unpack(header_data) if signature != cls.SIGNATURE: raise WindowsCrashDumpFormatException( - base_layer.name, "Bad signature 0x{:x} at file offset 0x{:x}".format(signature, offset)) + base_layer.name, f"Bad signature 0x{signature:x} at file offset 0x{offset:x}") if validdump != cls.VALIDDUMP: raise WindowsCrashDumpFormatException(base_layer.name, - "Invalid dump 0x{:x} at file offset 0x{:x}".format(validdump, offset)) + f"Invalid dump 0x{validdump:x} at file offset 0x{offset:x}") return signature, validdump class WindowsCrashDump64Layer(WindowsCrashDump32Layer): """A Windows crash format TranslationLayer. - This TranslationLayer supports Microsoft complete memory dump files. It currently does not support kernel or small memory dump files. """ @@ -125,63 +192,6 @@ class WindowsCrashDump64Layer(WindowsCrashDump32Layer): supported_dumptypes = [0x1, 0x05] headerpages = 2 - def _load_segments(self) -> None: - """Loads up the segments from the meta_layer.""" - - segments = [] - - summary_header = self.context.object(self._crash_table_name + constants.BANG + "_SUMMARY_DUMP64", - offset = 0x2000, - layer_name = self._base_layer) - - if self.dump_type == 0x1: - header = self.context.object(self._crash_table_name + constants.BANG + self.dump_header_name, - offset = 0, - layer_name = self._base_layer) - - offset = self.headerpages - header.PhysicalMemoryBlockBuffer.Run.count = header.PhysicalMemoryBlockBuffer.NumberOfRuns - for x in header.PhysicalMemoryBlockBuffer.Run: - segments.append((x.BasePage * 0x1000, offset * 0x1000, x.PageCount * 0x1000, x.PageCount * 0x1000)) - offset += x.PageCount - - elif self.dump_type == 0x05: - summary_header.BufferLong.count = (summary_header.BitmapSize + 31) // 32 - previous_bit = 0 - start_position = 0 - # We cast as an int because we don't want to carry the context around with us for infinite loop reasons - mapped_offset = int(summary_header.HeaderSize) - current_word = None - for bit_position in range(len(summary_header.BufferLong) * 32): - if (bit_position % 32) == 0: - current_word = summary_header.BufferLong[bit_position // 32] - current_bit = (current_word >> (bit_position % 32)) & 1 - if current_bit != previous_bit: - if previous_bit == 0: - # Start - start_position = bit_position - else: - # Finish - length = (bit_position - start_position) * 0x1000 - segments.append((start_position * 0x1000, mapped_offset, length, length)) - mapped_offset += length - - # Finish it off - if bit_position == (len(summary_header.BufferLong) * 32) - 1 and current_bit == 1: - length = (bit_position - start_position) * 0x1000 - segments.append((start_position * 0x1000, mapped_offset, length, length)) - mapped_offset += length - - previous_bit = current_bit - else: - vollog.log(constants.LOGLEVEL_VVVV, "unsupported dump format 0x{:x}".format(self.dump_type)) - raise WindowsCrashDumpFormatException(self.name, "unsupported dump format 0x{:x}".format(self.dump_type)) - - if len(segments) == 0: - raise WindowsCrashDumpFormatException(self.name, "No Crash segments defined in {}".format(self._base_layer)) - - self._segments = segments - class WindowsCrashDumpStacker(interfaces.automagic.StackerLayerInterface): stack_order = 11 diff --git a/volatility3/framework/layers/elf.py b/volatility3/framework/layers/elf.py index 48876c2867..4eb93a1c88 100644 --- a/volatility3/framework/layers/elf.py +++ b/volatility3/framework/layers/elf.py @@ -46,7 +46,7 @@ def _load_segments(self) -> None: segments.append((int(phdr.p_paddr), int(phdr.p_offset), int(phdr.p_memsz), int(phdr.p_memsz))) if len(segments) == 0: - raise ElfFormatException(self.name, "No ELF segments defined in {}".format(self._base_layer)) + raise ElfFormatException(self.name, f"No ELF segments defined in {self._base_layer}") self._segments = segments @@ -56,12 +56,12 @@ def _check_header(cls, base_layer: interfaces.layers.DataLayerInterface, offset: header_data = base_layer.read(offset, cls._header_struct.size) except exceptions.InvalidAddressException: raise ElfFormatException(base_layer.name, - "Offset 0x{:0x} does not exist within the base layer".format(offset)) + f"Offset 0x{offset:0x} does not exist within the base layer") (magic, elf_class, elf_data_encoding, elf_version) = cls._header_struct.unpack(header_data) if magic != cls.MAGIC: - raise ElfFormatException(base_layer.name, "Bad magic 0x{:x} at file offset 0x{:x}".format(magic, offset)) + raise ElfFormatException(base_layer.name, f"Bad magic 0x{magic:x} at file offset 0x{offset:x}") if elf_class != cls.ELF_CLASS: - raise ElfFormatException(base_layer.name, "ELF class is not 64-bit (2): {:d}".format(elf_class)) + raise ElfFormatException(base_layer.name, f"ELF class is not 64-bit (2): {elf_class:d}") # Virtualbox uses an ELF version of 0, which isn't to specification, but is ok to deal with return True @@ -78,7 +78,7 @@ def stack(cls, if not Elf64Layer._check_header(context.layers[layer_name]): return None except ElfFormatException as excp: - vollog.log(constants.LOGLEVEL_VVVV, "Exception: {}".format(excp)) + vollog.log(constants.LOGLEVEL_VVVV, f"Exception: {excp}") return None new_name = context.layers.free_layer_name("Elf64Layer") context.config[interfaces.configuration.path_join(new_name, "base_layer")] = layer_name diff --git a/volatility3/framework/layers/intel.py b/volatility3/framework/layers/intel.py index cf78bbcd11..723e4143bc 100644 --- a/volatility3/framework/layers/intel.py +++ b/volatility3/framework/layers/intel.py @@ -39,7 +39,7 @@ def __init__(self, metadata: Optional[Dict[str, Any]] = None) -> None: super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) self._base_layer = self.config["memory_layer"] - self._swap_layers = [] # type: List[str] + self._swap_layers: List[str] = [] self._page_map_offset = self.config["page_map_offset"] # Assign constants @@ -52,6 +52,7 @@ def __init__(self, self._index_shift = int(math.ceil(math.log2(struct.calcsize(self._entry_format)))) @classproperty + @functools.lru_cache() def page_size(cls) -> int: """Page size for the intel memory layers. @@ -60,16 +61,19 @@ def page_size(cls) -> int: return 1 << cls._page_size_in_bits @classproperty + @functools.lru_cache() def bits_per_register(cls) -> int: """Returns the bits_per_register to determine the range of an IntelTranslationLayer.""" return cls._bits_per_register @classproperty + @functools.lru_cache() def minimum_address(cls) -> int: return 0 @classproperty + @functools.lru_cache() def maximum_address(cls) -> int: return (1 << cls._maxvirtaddr) - 1 @@ -103,7 +107,7 @@ def _translate(self, offset: int) -> Tuple[int, int, str]: # Now we're done if not self._page_is_valid(entry): raise exceptions.PagedInvalidAddressException(self.name, offset, position + 1, entry, - "Page Fault at entry {} in page entry".format(hex(entry))) + f"Page Fault at entry {hex(entry)} in page entry") page = self._mask(entry, self._maxphyaddr - 1, position + 1) | self._mask(offset, position, 0) return page, 1 << (position + 1), self._base_layer @@ -119,6 +123,10 @@ def _translate_entry(self, offset: int) -> Tuple[int, int]: position = self._initial_position entry = self._initial_entry + if self.minimum_address > offset > self.maximum_address: + raise exceptions.PagedInvalidAddressException(self.name, offset, position + 1, entry, + "Entry outside virtual address range: " + hex(entry)) + # Run through the offset in various chunks for (name, size, large_page) in self._structure: # Check we're valid @@ -127,6 +135,9 @@ def _translate_entry(self, offset: int) -> Tuple[int, int]: "Page Fault at entry " + hex(entry) + " in table " + name) # Check if we're a large page if large_page and (entry & (1 << 7)): + # Mask off the PAT bit + if entry & (1 << 12): + entry -= (1 << 12) # We're a large page, the rest is finished below # If we want to implement PSE-36, it would need to be done here break @@ -185,6 +196,38 @@ def mapping(self, """Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer) mappings. + This allows translation layers to provide maps of contiguous + regions in one layer + """ + stashed_offset = stashed_mapped_offset = stashed_size = stashed_mapped_size = stashed_map_layer = None + for offset, size, mapped_offset, mapped_size, map_layer in self._mapping(offset, length, ignore_errors): + if stashed_offset is None or (stashed_offset + stashed_size != offset) or ( + stashed_mapped_offset + stashed_mapped_size != mapped_offset) or (stashed_map_layer != map_layer): + # The block isn't contiguous + if stashed_offset is not None: + yield stashed_offset, stashed_size, stashed_mapped_offset, stashed_mapped_size, stashed_map_layer + # Update all the stashed values after output + stashed_offset = offset + stashed_mapped_offset = mapped_offset + stashed_size = size + stashed_mapped_size = mapped_size + stashed_map_layer = map_layer + else: + # Part of an existing block + stashed_size += size + stashed_mapped_size += mapped_size + # Yield whatever's left + if (stashed_offset is not None and stashed_mapped_offset is not None and stashed_size is not None + and stashed_mapped_size is not None and stashed_map_layer is not None): + yield stashed_offset, stashed_size, stashed_mapped_offset, stashed_mapped_size, stashed_map_layer + + def _mapping(self, + offset: int, + length: int, + ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + """Returns a sorted iterable of (offset, sublength, mapped_offset, mapped_length, layer) + mappings. + This allows translation layers to provide maps of contiguous regions in one layer """ @@ -320,6 +363,11 @@ def _translate(self, offset: int) -> Tuple[int, int, str]: class WindowsIntel32e(WindowsMixin, Intel32e): + # TODO: Fix appropriately in a future release. + # Currently just a temporary workaround to deal with custom bit flag + # in the PFN field for pages in transition state. + # See https://github.com/volatilityfoundation/volatility3/pull/475 + _maxphyaddr = 45 def _translate(self, offset: int) -> Tuple[int, int, str]: return self._translate_swap(self, offset, self._bits_per_register // 2) diff --git a/volatility3/framework/layers/leechcore.py b/volatility3/framework/layers/leechcore.py new file mode 100644 index 0000000000..8c492ca852 --- /dev/null +++ b/volatility3/framework/layers/leechcore.py @@ -0,0 +1,174 @@ +import io +import logging +import urllib.parse +from typing import Optional, Any, List + +try: + import leechcorepyc + HAS_LEECHCORE = True +except ImportError: + HAS_LEECHCORE = False + +from volatility3.framework import exceptions +from volatility3.framework.layers import resources + +vollog = logging.getLogger(__file__) + +if HAS_LEECHCORE: + + class LeechCoreFile(io.RawIOBase): + """Class to mimic python-native file access to a LeechCore memory space""" + + _leechcore = None + + def __init__(self, leechcore_device): + self._chunk_size = 0x1000000 + self._device = leechcore_device + self._cursor = 0 + self._handle = None + self._pad = True + self._chunk_size = 0x1000000 + + @property + def maxaddr(self): + return self.handle.maxaddr + + @property + def handle(self): + """The actual LeechCore file object returned by leechcorepyc + + Accessing this attribute will create/attach the handle if it hasn't already been opened + """ + if not self._handle: + try: + self._handle = leechcorepyc.LeechCore(self._device) + except TypeError: + raise IOError(f"Unable to open LeechCore device {self._device}") + return self._handle + + def fileno(self): + raise OSError + + def flush(self): + pass + + def isatty(self): + return False + + def readable(self): + """This returns whether the handle is open + + This doesn't access self.handle so that it doesn't accidentally attempt to open the device + """ + return bool(self._handle) + + def seek(self, offset, whence = io.SEEK_SET): + if whence == io.SEEK_SET: + self._cursor = offset + elif whence == io.SEEK_CUR: + self._cursor += offset + elif whence == io.SEEK_END: + self._cursor = self.maxaddr + offset + + def tell(self): + """Return how far into the memory we are""" + return self._cursor + + def writable(self): + """Leechcore supports writing, so this is always true""" + return True + + def writelines(self, lines: List[bytes]): + return self.write(b"".join(lines)) + + def in_memmap(self, start, size): + chunk_start = start + chunk_size = size + output = [] + for entry in self.handle.memmap: + + if entry['base'] + entry['size'] <= chunk_start or entry['base'] >= chunk_start + chunk_size: + continue + output += [(max(entry['base'], chunk_start), min(entry['size'], chunk_size))] + chunk_start = output[-1][0] + output[-1][1] + chunk_size = max(0, size - chunk_start) + + if chunk_size <= 0: + break + return output + + def write(self, b: bytes): + result = self.handle.write(self._cursor, b) + self._cursor += len(b) + return result + + def read(self, size: int = -1) -> bytes: + """We ask leechcore to pad the data, because otherwise determining holes in the underlying file would + be extremely inefficient borderline impossible to do consistently""" + data = self.handle.read(self._cursor, size, True) + + if len(data) > size: + data = data[:size] + else: + data = data + b'\x00' * (size - len(data)) + self._cursor += len(data) + if not len(data): + raise exceptions.InvalidAddressException('LeechCore layer read failure', self._cursor + len(data)) + return data + + def readline(self, __size: Optional[int] = ...) -> bytes: + data = b'' + while __size > self._chunk_size or __size < 0: + data += self.read(self._chunk_size) + index = data.find(b"\n") + __size -= self._chunk_size + if index >= 0: + __size = 0 + break + data += self.read(__size) + index = data.find(b"\n") + return data[:index] + + def readlines(self, __hint: int = ...) -> List[bytes]: + counter = 0 + result = [] + while counter < __hint or __hint < 0: + line = self.readline() + counter += len(line) + result += [line] + return result + + def readall(self) -> bytes: + return self.read() + + def readinto(self, b: bytearray) -> Optional[int]: + data = self.read() + for index in range(len(data)): + b[index] = data[index] + return len(data) + + def close(self): + if self._handle: + self._handle.close() + self._handle = None + + def closed(self): + return self._handle + + + class LeechCoreHandler(resources.VolatilityHandler): + """Handler for the invented `leechcore` scheme. This is an unofficial scheme and not registered with IANA + """ + + @classmethod + def non_cached_schemes(cls) -> List[str]: + """We need to turn caching *off* for a live filesystem""" + return ['leechcore'] + + @staticmethod + def default_open(req: urllib.request.Request) -> Optional[Any]: + """Handles the request if it's the leechcore scheme.""" + if req.type == 'leechcore': + device_uri = '://'.join(req.full_url.split('://')[1:]) + return LeechCoreFile(device_uri) + return None diff --git a/volatility3/framework/layers/lime.py b/volatility3/framework/layers/lime.py index ae93f7b7a6..4f4a66f185 100644 --- a/volatility3/framework/layers/lime.py +++ b/volatility3/framework/layers/lime.py @@ -45,7 +45,7 @@ def _load_segments(self) -> None: if start < maxaddr or end < start: raise LimeFormatException( - self.name, "Bad start/end 0x{:x}/0x{:x} at file offset 0x{:x}".format(start, end, offset)) + self.name, f"Bad start/end 0x{start:x}/0x{end:x} at file offset 0x{offset:x}") segment_length = end - start + 1 segments.append((start, offset + header_size, segment_length, segment_length)) @@ -53,7 +53,7 @@ def _load_segments(self) -> None: offset = offset + header_size + segment_length if len(segments) == 0: - raise LimeFormatException(self.name, "No LiME segments defined in {}".format(self._base_layer)) + raise LimeFormatException(self.name, f"No LiME segments defined in {self._base_layer}") self._segments = segments @@ -63,13 +63,13 @@ def _check_header(cls, base_layer: interfaces.layers.DataLayerInterface, offset: header_data = base_layer.read(offset, cls._header_struct.size) except exceptions.InvalidAddressException: raise LimeFormatException(base_layer.name, - "Offset 0x{:0x} does not exist within the base layer".format(offset)) + f"Offset 0x{offset:0x} does not exist within the base layer") (magic, version, start, end, reserved) = cls._header_struct.unpack(header_data) if magic != cls.MAGIC: - raise LimeFormatException(base_layer.name, "Bad magic 0x{:x} at file offset 0x{:x}".format(magic, offset)) + raise LimeFormatException(base_layer.name, f"Bad magic 0x{magic:x} at file offset 0x{offset:x}") if version != cls.VERSION: raise LimeFormatException(base_layer.name, - "Unexpected version {:d} at file offset 0x{:x}".format(version, offset)) + f"Unexpected version {version:d} at file offset 0x{offset:x}") return start, end diff --git a/volatility3/framework/layers/linear.py b/volatility3/framework/layers/linear.py index 80f71ec35e..c5cb47bdc2 100644 --- a/volatility3/framework/layers/linear.py +++ b/volatility3/framework/layers/linear.py @@ -16,13 +16,13 @@ def translate(self, offset: int, ignore_errors: bool = False) -> Tuple[Optional[ original_offset, _, mapped_offset, _, layer = mapping[0] if original_offset != offset: raise exceptions.LayerException(self.name, - "Layer {} claims to map linearly but does not".format(self.name)) + f"Layer {self.name} claims to map linearly but does not") else: if ignore_errors: # We should only hit this if we ignored errors, but check anyway return None, None raise exceptions.InvalidAddressException(self.name, offset, - "Cannot translate {} in layer {}".format(offset, self.name)) + f"Cannot translate {offset} in layer {self.name}") return mapped_offset, layer # ## Read/Write functions for mapped pages @@ -33,11 +33,11 @@ def read(self, offset: int, length: int, pad: bool = False) -> bytes: """Reads an offset for length bytes and returns 'bytes' (not 'str') of length size.""" current_offset = offset - output = [] # type: List[bytes] + output: List[bytes] = [] for (offset, _, mapped_offset, mapped_length, layer) in self.mapping(offset, length, ignore_errors = pad): if not pad and offset > current_offset: raise exceptions.InvalidAddressException( - self.name, current_offset, "Layer {} cannot map offset: {}".format(self.name, current_offset)) + self.name, current_offset, f"Layer {self.name} cannot map offset: {current_offset}") elif offset > current_offset: output += [b"\x00" * (offset - current_offset)] current_offset = offset @@ -57,7 +57,7 @@ def write(self, offset: int, value: bytes) -> None: for (offset, _, mapped_offset, length, layer) in self.mapping(offset, length): if offset > current_offset: raise exceptions.InvalidAddressException( - self.name, current_offset, "Layer {} cannot map offset: {}".format(self.name, current_offset)) + self.name, current_offset, f"Layer {self.name} cannot map offset: {current_offset}") elif offset < current_offset: raise exceptions.LayerException(self.name, "Mapping returned an overlapping element") self._context.layers.write(layer, mapped_offset, value[:length]) diff --git a/volatility3/framework/layers/msf.py b/volatility3/framework/layers/msf.py index 442ba87289..02fc570bcb 100644 --- a/volatility3/framework/layers/msf.py +++ b/volatility3/framework/layers/msf.py @@ -34,7 +34,7 @@ def __init__(self, if response is None: raise PDBFormatException(name, "Could not find a suitable header") self._version, self._header = response - self._streams = {} # type: Dict[int, str] + self._streams: Dict[int, str] = {} @property def pdb_symbol_table(self) -> str: @@ -224,7 +224,7 @@ def maximum_address(self) -> int: def _pdb_layer(self) -> PdbMultiStreamFormat: if self._base_layer not in self._context.layers: raise PDBFormatException(self._base_layer, - "No PdbMultiStreamFormat layer found: {}".format(self._base_layer)) + f"No PdbMultiStreamFormat layer found: {self._base_layer}") result = self._context.layers[self._base_layer] if isinstance(result, PdbMultiStreamFormat): return result diff --git a/volatility3/framework/layers/physical.py b/volatility3/framework/layers/physical.py index 7e4e76a6e8..73d46b2119 100644 --- a/volatility3/framework/layers/physical.py +++ b/volatility3/framework/layers/physical.py @@ -1,6 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import logging import threading from typing import Any, Dict, IO, List, Optional, Union @@ -8,6 +9,8 @@ from volatility3.framework.configuration import requirements from volatility3.framework.layers import resources +vollog = logging.getLogger(__name__) + class BufferDataLayer(interfaces.layers.DataLayerInterface): """A DataLayer class backed by a buffer in memory, designed for testing and @@ -80,12 +83,13 @@ def __init__(self, metadata: Optional[Dict[str, Any]] = None) -> None: super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) + self._write_warning = False self._location = self.config["location"] self._accessor = resources.ResourceAccessor() - self._file_ = None # type: Optional[IO[Any]] - self._size = None # type: Optional[int] + self._file_: Optional[IO[Any]] = None + self._size: Optional[int] = None # Construct the lock now (shared if made before threading) in case we ever need it - self._lock = DummyLock() # type: Union[DummyLock, threading.Lock] + self._lock: Union[DummyLock, threading.Lock] = DummyLock() if constants.PARALLELISM == constants.Parallelism.Threading: self._lock = threading.Lock() # Instantiate the file to throw exceptions if the file doesn't open @@ -157,6 +161,11 @@ def write(self, offset: int, data: bytes) -> None: This will technically allow writes beyond the extent of the file """ + if not self._file.writable(): + if not self._write_warning: + self._write_warning = True + vollog.warning(f"Try to write to unwritable layer: {self.name}") + return None if not self.is_valid(offset, len(data)): invalid_address = offset if self.minimum_address < offset <= self.maximum_address: @@ -180,6 +189,9 @@ def destroy(self) -> None: """Closes the file handle.""" self._file.close() + def __del__(self) -> None: + self.destroy() + @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [requirements.StringRequirement(name = 'location', optional = False)] diff --git a/volatility3/framework/layers/qemu.py b/volatility3/framework/layers/qemu.py index 8a2621fa8d..4ce17bb634 100644 --- a/volatility3/framework/layers/qemu.py +++ b/volatility3/framework/layers/qemu.py @@ -1,6 +1,7 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import bisect import functools import json import math @@ -40,7 +41,7 @@ def __init__(self, metadata: Optional[Dict[str, Any]] = None) -> None: self._qemu_table_name = intermed.IntermediateSymbolTable.create(context, config_path, 'generic', 'qemu') self._configuration = None - self._compressed = set() # type: Set[int] + self._compressed: Set[int] = set() self._current_segment_name = b'' super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) @@ -58,12 +59,15 @@ def _read_configuration(self, base_layer: interfaces.layers.DataLayerInterface, data = b'' for i in range(base_layer.maximum_address, base_layer.minimum_address, -chunk_size): if i != base_layer.maximum_address: - data = base_layer.read(i, chunk_size) + data + data = (base_layer.read(i, chunk_size) + data).rstrip(b'\x00') if b'\x00' in data: - start = data.rfind(b'\x00') - data = data[data.find(b'{', start):] - return json.loads(data) - raise exceptions.LayerException(name, "Could not load JSON configuration from the end of the file") + last_null_byte = data.rfind(b'\x00') + start_of_json = data.find(b'{', last_null_byte) + if start_of_json >= 0: + data = data[start_of_json:] + return json.loads(data) + return dict() + raise exceptions.LayerException(name, "Invalid JSON configuration at the end of the file") def _get_ram_segments(self, index: int, page_size: int) -> Tuple[List[Tuple[int, int, int, int]], int]: """Recovers the new index and any sections of memory from a ram section""" @@ -176,21 +180,21 @@ def _load_segments(self): index += 4 if section_id != current_section_id: raise exceptions.LayerException( - self._name, 'QEMU section footer mismatch: {} and {}'.format(current_section_id, section_id)) + self._name, f'QEMU section footer mismatch: {current_section_id} and {section_id}') elif section_byte == self.QEVM_EOF: pass else: - raise exceptions.LayerException(self._name, 'QEMU unknown section encountered: {}'.format(section_byte)) + raise exceptions.LayerException(self._name, f'QEMU unknown section encountered: {section_byte}') def extract_data(self, index, name, version_id): if name == 'ram': if version_id != 4: - raise exceptions.LayerException("QEMU unknown RAM version_id {}".format(version_id)) + raise exceptions.LayerException(f"QEMU unknown RAM version_id {version_id}") new_segments, index = self._get_ram_segments(index, self._configuration.get('page_size', None) or 4096) self._segments += new_segments elif name == 'spapr/htab': if version_id != 1: - raise exceptions.LayerException("QEMU unknown HTAB version_id {}".format(version_id)) + raise exceptions.LayerException(f"QEMU unknown HTAB version_id {version_id}") header = self.context.object(self._qemu_table_name + constants.BANG + 'unsigned long', offset = index, layer_name = self._base_layer) @@ -208,9 +212,18 @@ def extract_data(self, index, name, version_id): return index def _decode_data(self, data: bytes, mapped_offset: int, offset: int, output_length: int) -> bytes: - if mapped_offset in self._compressed: - return (data * 0x1000)[:output_length] - return data + """Takes the full segment from the base_layer that the data occurs in, checks whether it's compressed + (by locating it in the segment list and verifying if that address is compressed), then reading/expanding the + data, and finally cutting it to the right size. Offset may be the address requested rather than the location + of the starting data. It is the responsibility of the layer to turn the provided data chunk into the right + portion of data necessary. + """ + start_offset, _, start_mapped_offset, _ = self._segments[ + bisect.bisect_right(self._segments, (offset, 0xffffffffffffff,)) - 1] + if start_mapped_offset in self._compressed: + data = (data * 0x1000) + result = data[offset - start_offset:output_length + offset - start_offset] + return result @functools.lru_cache(maxsize = 512) def read(self, offset: int, length: int, pad: bool = False) -> bytes: diff --git a/volatility3/framework/layers/registry.py b/volatility3/framework/layers/registry.py index 32acbc4a9c..55a6e51863 100644 --- a/volatility3/framework/layers/registry.py +++ b/volatility3/framework/layers/registry.py @@ -48,7 +48,7 @@ def __init__(self, # TODO: Check the checksum if self.hive.Signature != 0xbee0bee0: raise RegistryFormatException( - self.name, "Registry hive at {} does not have a valid signature".format(self._hive_offset)) + self.name, f"Registry hive at {self._hive_offset} does not have a valid signature") # Win10 17063 introduced the Registry process to map most hives. Check # if it exists and update RegistryHive._base_layer @@ -66,13 +66,13 @@ def __init__(self, self._hive_maxaddr_non_volatile = self.hive.Storage[0].Length self._hive_maxaddr_volatile = self.hive.Storage[1].Length self._maxaddr = 0x80000000 | self._hive_maxaddr_volatile - vollog.log(constants.LOGLEVEL_VVV, "Setting hive max address to {}".format(hex(self._maxaddr))) + vollog.log(constants.LOGLEVEL_VVVV, f"Setting hive {self.name} max address to {hex(self._maxaddr)}") except exceptions.InvalidAddressException: self._hive_maxaddr_non_volatile = 0x7fffffff self._hive_maxaddr_volatile = 0x7fffffff self._maxaddr = 0x80000000 | self._hive_maxaddr_volatile - vollog.log(constants.LOGLEVEL_VVV, - "Exception when setting hive max address, using {}".format(hex(self._maxaddr))) + vollog.log(constants.LOGLEVEL_VVVV, + f"Exception when setting hive {self.name} max address, using {hex(self._maxaddr)}") def _get_hive_maxaddr(self, volatile): return self._hive_maxaddr_volatile if volatile else self._hive_maxaddr_non_volatile @@ -141,7 +141,7 @@ def get_key(self, key: str, return_list: bool = False) -> Union[List[objects.Str if key.endswith("\\"): key = key[:-1] key_array = key.split('\\') - found_key = [] # type: List[str] + found_key: List[str] = [] while key_array and node_key: subkeys = node_key[-1].get_subkeys() for subkey in subkeys: @@ -199,6 +199,13 @@ def _translate(self, offset: int) -> int: # Ignore the volatile bit when determining maxaddr validity volatile = self._mask(offset, 31, 31) >> 31 if offset & 0x7fffffff > self._get_hive_maxaddr(volatile): + vollog.log(constants.LOGLEVEL_VVV, + "Layer {} couldn't translate offset {}, greater than {} in {} store of {}".format( + self.name, + hex(offset & 0x7fffffff), + hex(self._get_hive_maxaddr(volatile)), + "volative" if volatile else "non-volatile", + self.get_name())) raise RegistryInvalidIndex(self.name, "Mapping request for value greater than maxaddr") storage = self.hive.Storage[volatile] diff --git a/volatility3/framework/layers/resources.py b/volatility3/framework/layers/resources.py index d99f7135c2..b6ef1b6bad 100644 --- a/volatility3/framework/layers/resources.py +++ b/volatility3/framework/layers/resources.py @@ -13,11 +13,11 @@ import urllib.parse import urllib.request import zipfile -from typing import Optional, Any, IO +from typing import Optional, Any, IO, List from urllib import error from volatility3 import framework -from volatility3.framework import constants +from volatility3.framework import constants, exceptions try: import magic @@ -34,6 +34,7 @@ vollog = logging.getLogger(__name__) + # TODO: Type-annotating the ResourceAccessor.open method is difficult because HTTPResponse is not actually an IO[Any] type # fix this @@ -55,14 +56,15 @@ def close(): class ResourceAccessor(object): - """Object for openning URLs as files (downloading locally first if + """Object for opening URLs as files (downloading locally first if necessary)""" list_handlers = True def __init__(self, progress_callback: Optional[constants.ProgressCallback] = None, - context: Optional[ssl.SSLContext] = None) -> None: + context: Optional[ssl.SSLContext] = None, + enable_cache: bool = True) -> None: """Creates a resource accessor. Note: context is an SSL context, not a volatility context @@ -70,16 +72,25 @@ def __init__(self, self._progress_callback = progress_callback self._context = context self._handlers = list(framework.class_subclasses(urllib.request.BaseHandler)) + self._enable_cache = enable_cache if self.list_handlers: vollog.log(constants.LOGLEVEL_VVV, - "Available URL handlers: {}".format(", ".join([x.__name__ for x in self._handlers]))) + f"Available URL handlers: {', '.join([x.__name__ for x in self._handlers])}") self.__class__.list_handlers = False def uses_cache(self, url: str) -> bool: """Determines whether a URLs contents should be cached""" parsed_url = urllib.parse.urlparse(url) - return not parsed_url.scheme in ['file', 'jar'] + return self._enable_cache and not parsed_url.scheme in self._non_cached_schemes() + + @staticmethod + def _non_cached_schemes() -> List[str]: + """Returns the list of schemes not to be cached""" + result = ['file'] + for clazz in framework.class_subclasses(VolatilityHandler): + result += clazz.non_cached_schemes() + return result # Current urllib.request.urlopen returns Any, so we do the same def open(self, url: str, mode: str = "rb") -> Any: @@ -107,6 +118,9 @@ def open(self, url: str, mode: str = "rb") -> Any: raise excp else: raise excp + except exceptions.OfflineException: + vollog.info(f"Not accessing {url} in offline mode") + raise with contextlib.closing(fp) as fp: # Cache the file locally @@ -122,7 +136,7 @@ def open(self, url: str, mode: str = "rb") -> Any: "data_" + hashlib.sha512(bytes(url, 'raw_unicode_escape')).hexdigest() + ".cache") if not os.path.exists(temp_filename): - vollog.debug("Caching file at: {}".format(temp_filename)) + vollog.debug(f"Caching file at: {temp_filename}") try: content_length = fp.info().get('Content-Length', -1) @@ -137,11 +151,13 @@ def open(self, url: str, mode: str = "rb") -> Any: count += len(block) if self._progress_callback: self._progress_callback(count * 100 / max(count, int(content_length)), - "Reading file {}".format(url)) + f"Reading file {url}") cache_file.write(block) block = fp.read(block_size) cache_file.close() # Re-open the cache with a different mode + # Since we don't want people thinking they're able to save to the cache file, + # open it in read mode only and allow breakages to happen if they wanted to write curfile = open(temp_filename, mode = "rb") # Determine whether the file is a particular type of file, and if so, open it as such @@ -199,7 +215,14 @@ def open(self, url: str, mode: str = "rb") -> Any: return curfile -class JarHandler(urllib.request.BaseHandler): +class VolatilityHandler(urllib.request.BaseHandler): + + @classmethod + def non_cached_schemes(cls) -> List[str]: + return [] + + +class JarHandler(VolatilityHandler): """Handles the jar scheme for URIs. Reference used for the schema syntax: @@ -209,21 +232,33 @@ class JarHandler(urllib.request.BaseHandler): http://developer.java.sun.com/developer/onlineTraining/protocolhandlers/ """ + @classmethod + def non_cached_schemes(cls) -> List[str]: + return ['jar'] + @staticmethod def default_open(req: urllib.request.Request) -> Optional[Any]: """Handles the request if it's the jar scheme.""" if req.type == 'jar': subscheme, remainder = req.full_url.split(":")[1], ":".join(req.full_url.split(":")[2:]) if subscheme != 'file': - vollog.log(constants.LOGLEVEL_VVV, "Unsupported jar subscheme {}".format(subscheme)) + vollog.log(constants.LOGLEVEL_VVV, f"Unsupported jar subscheme {subscheme}") return None zipsplit = remainder.split("!") if len(zipsplit) != 2: vollog.log(constants.LOGLEVEL_VVV, - "Path did not contain exactly one fragment indicator: {}".format(remainder)) + f"Path did not contain exactly one fragment indicator: {remainder}") return None zippath, filepath = zipsplit return zipfile.ZipFile(zippath).open(filepath) return None + + +class OfflineHandler(VolatilityHandler): + @staticmethod + def default_open(req: urllib.request.Request) -> Optional[Any]: + if constants.OFFLINE and req.type in ['http', 'https']: + raise exceptions.OfflineException(req.full_url) + return None diff --git a/volatility3/framework/layers/scanners/__init__.py b/volatility3/framework/layers/scanners/__init__.py index 3b57d32e44..85c8390d95 100644 --- a/volatility3/framework/layers/scanners/__init__.py +++ b/volatility3/framework/layers/scanners/__init__.py @@ -1,9 +1,8 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - import re -from typing import Generator, List, Tuple +from typing import Generator, List, Tuple, Dict, Optional from volatility3.framework.interfaces import layers from volatility3.framework.layers.scanners import multiregexp @@ -12,6 +11,8 @@ class BytesScanner(layers.ScannerInterface): thread_safe = True + _required_framework_version = (2, 0, 0) + def __init__(self, needle: bytes) -> None: super().__init__() self.needle = needle @@ -21,15 +22,24 @@ def __call__(self, data: bytes, data_offset: int) -> Generator[int, None, None]: where the needle is found.""" find_pos = data.find(self.needle) while find_pos >= 0: + # Ensure that if we're in the overlap, we don't report it + # It'll be returned when the next block is scanned if find_pos < self.chunk_size: yield find_pos + data_offset find_pos = data.find(self.needle, find_pos + 1) class RegExScanner(layers.ScannerInterface): + """A scanner that can be provided with a bytes-object regular expression pattern + The scanner will scqn all blocks for the regular expression and report the absolute offset of any finds + + The default flags include DOTALL, since the searches are through binary data and the newline character should + have no specific significance in such searches""" thread_safe = True - def __init__(self, pattern: bytes, flags: int = 0) -> None: + _required_framework_version = (2, 0, 0) + + def __init__(self, pattern: bytes, flags: int = re.DOTALL) -> None: super().__init__() self.regex = re.compile(pattern, flags) @@ -46,15 +56,78 @@ def __call__(self, data: bytes, data_offset: int) -> Generator[int, None, None]: class MultiStringScanner(layers.ScannerInterface): thread_safe = True + _required_framework_version = (2, 0, 0) + def __init__(self, patterns: List[bytes]) -> None: super().__init__() - self._patterns = multiregexp.MultiRegexp() + self._pattern_trie: Optional[Dict[int, Optional[Dict]]] = {} for pattern in patterns: - self._patterns.add_pattern(pattern) - self._patterns.preprocess() + self._process_pattern(pattern) + self._regex = self._process_trie(self._pattern_trie) + + def _process_pattern(self, value: bytes) -> None: + trie = self._pattern_trie + if trie is None: + return None + + for char in value: + trie[char] = trie.get(char, {}) + trie = trie[char] + + # Mark the end of a string + trie[-1] = None + + def _process_trie(self, trie: Optional[Dict[int, Optional[Dict]]]) -> bytes: + if trie is None or len(trie) == 1 and -1 in trie: + # We've reached the end of this path, return the empty byte string + return b'' + + choices = [] + suffixes = [] + finished = False + + for entry in sorted(trie): + # Clump together different paths + if entry >= 0: + remainder = self._process_trie(trie[entry]) + if remainder: + choices.append(re.escape(bytes([entry])) + remainder) + else: + suffixes.append(re.escape(bytes([entry]))) + else: + # If we've fininshed one of the strings at this point, remember it for later + finished = True + + if len(suffixes) == 1: + choices.append(suffixes[0]) + elif len(suffixes) > 1: + choices.append(b'[' + b''.join(suffixes) + b']') + + if len(choices) == 0: + # If there's none, return the empty byte string + response = b'' + elif len(choices) == 1: + # If there's only one return it + response = choices[0] + else: + response = b'(?:' + b'|'.join(choices) + b')' + + if finished: + # We finished one string, so everything after this is optional + response = b"(?:" + response + b")?" + + return response def __call__(self, data: bytes, data_offset: int) -> Generator[Tuple[int, bytes], None, None]: """Runs through the data looking for the needles.""" - for offset, pattern in self._patterns.search(data): + for offset, pattern in self.search(data): if offset < self.chunk_size: yield offset + data_offset, pattern + + def search(self, haystack: bytes) -> Generator[Tuple[int, bytes], None, None]: + if not isinstance(haystack, bytes): + raise TypeError("Search haystack must be a byte string") + if not self._regex: + raise ValueError("MultiRegexp cannot be used with an empty set of search strings") + for match in re.finditer(self._regex, haystack): + yield match.start(0), match.group() diff --git a/volatility3/framework/layers/scanners/multiregexp.py b/volatility3/framework/layers/scanners/multiregexp.py index 36c6410aa8..45feb51d1c 100644 --- a/volatility3/framework/layers/scanners/multiregexp.py +++ b/volatility3/framework/layers/scanners/multiregexp.py @@ -10,7 +10,7 @@ class MultiRegexp(object): """Algorithm for multi-string matching.""" def __init__(self) -> None: - self._pattern_strings = [] # type: List[bytes] + self._pattern_strings: List[bytes] = [] self._regex = re.compile(b'') def add_pattern(self, pattern: bytes) -> None: diff --git a/volatility3/framework/layers/segmented.py b/volatility3/framework/layers/segmented.py index 8334c722df..05fc01b977 100644 --- a/volatility3/framework/layers/segmented.py +++ b/volatility3/framework/layers/segmented.py @@ -25,9 +25,9 @@ def __init__(self, super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) self._base_layer = self.config["base_layer"] - self._segments = [] # type: List[Tuple[int, int, int, int]] - self._minaddr = None # type: Optional[int] - self._maxaddr = None # type: Optional[int] + self._segments: List[Tuple[int, int, int, int]] = [] + self._minaddr: Optional[int] = None + self._maxaddr: Optional[int] = None self._load_segments() @@ -35,7 +35,7 @@ def __init__(self, def _load_segments(self) -> None: """Populates the _segments variable. - Segments must be (address, mapped address, length) and must be + Segments must be (address, mapped address, length, mapped_length) and must be sorted by address when this method exits """ @@ -67,7 +67,11 @@ def _find_segment(self, offset: int, next: bool = False) -> Tuple[int, int, int, if next: if i < len(self._segments): return self._segments[i] - raise exceptions.InvalidAddressException(self.name, offset, "Invalid address at {:0x}".format(offset)) + raise exceptions.InvalidAddressException(self.name, offset, f"Invalid address at {offset:0x}") + + # Determines whether larger segments are in use and the offsets within them should be tracked linearly + # When no decoding of the data occurs, this should be set to true + _track_offset = False def mapping(self, offset: int, @@ -85,7 +89,8 @@ def mapping(self, if current_offset > logical_offset: difference = current_offset - logical_offset logical_offset += difference - mapped_offset += difference + if self._track_offset: + mapped_offset += difference size -= difference except exceptions.InvalidAddressException: if not ignore_errors: @@ -103,7 +108,7 @@ def mapping(self, return # Crop it to the amount we need left chunk_size = min(size, length + offset - logical_offset) - yield logical_offset, chunk_size, mapped_offset, chunk_size, self._base_layer + yield logical_offset, chunk_size, mapped_offset, mapped_size, self._base_layer current_offset += chunk_size # Terminate if we've gone (or reached) our required limit if current_offset >= offset + length: @@ -139,4 +144,12 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] class SegmentedLayer(NonLinearlySegmentedLayer, linear.LinearlyMappedLayer, metaclass = ABCMeta): - pass + _track_offset = True + + def mapping(self, + offset: int, + length: int, + ignore_errors: bool = False) -> Iterable[Tuple[int, int, int, int, str]]: + # Linear mappings must return the same length of segment as that requested + for offset, length, mapped_offset, mapped_length, layer in super().mapping(offset, length, ignore_errors): + yield offset, length, mapped_offset, length, layer diff --git a/volatility3/framework/layers/vmware.py b/volatility3/framework/layers/vmware.py index e0d6b172da..85e961b246 100644 --- a/volatility3/framework/layers/vmware.py +++ b/volatility3/framework/layers/vmware.py @@ -2,6 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import logging import struct from typing import Any, Dict, List, Optional @@ -10,6 +11,8 @@ from volatility3.framework.layers import physical, segmented, resources from volatility3.framework.symbols import native +vollog = logging.getLogger(__name__) + class VmwareFormatException(exceptions.LayerException): """Thrown when an error occurs with the underlying VMware vmem file format.""" @@ -36,6 +39,10 @@ def _load_segments(self) -> None: """Loads up the segments from the meta_layer.""" self._read_header() + @staticmethod + def _choose_type(size: int) -> str: + return "vmware!unsigned int" if size == 4 else "vmware!unsigned long long" + def _read_header(self) -> None: """Checks the vmware header to make sure it's valid.""" if "vmware" not in self._context.symbol_space: @@ -45,12 +52,10 @@ def _read_header(self) -> None: header_size = struct.calcsize(self.header_structure) data = meta_layer.read(0, header_size) magic, unknown, groupCount = struct.unpack(self.header_structure, data) - if magic not in [b"\xD2\xBE\xD2\xBE"]: - raise VmwareFormatException(self.name, "Wrong magic bytes for Vmware layer: {}".format(repr(magic))) - - # TODO: Change certain structure sizes based on the version - # version = magic[1] & 0xf + if magic not in [b"\xD0\xBE\xD2\xBE", b"\xD1\xBA\xD1\xBA", b"\xD2\xBE\xD2\xBE", b"\xD3\xBE\xD3\xBE"]: + raise VmwareFormatException(self.name, f"Wrong magic bytes for Vmware layer: {repr(magic)}") + version = magic[0] & 0xf group_size = struct.calcsize(self.group_structure) groups = {} @@ -74,19 +79,36 @@ def _read_header(self) -> None: layer_name = self._meta_layer, offset = offset + 2, max_length = name_len) - indicies_len = (flags >> 6) & 3 - indicies = [] - for index in range(indicies_len): - indicies.append( + indices_len = (flags >> 6) & 3 + indices = [] + for index in range(indices_len): + indices.append( self._context.object("vmware!unsigned int", offset = offset + name_len + 2 + (index * index_len), layer_name = self._meta_layer)) - data = self._context.object("vmware!unsigned int", + data_len = flags & 0x3f + + if data_len in [62, 63]: # Handle special data sizes that indicate a longer data stream + data_len = 4 if version == 0 else 8 + # Read the size of the data + data_size = self._context.object(self._choose_type(data_len), layer_name = self._meta_layer, - offset = offset + 2 + name_len + (indicies_len * index_len)) - tags[(name, tuple(indicies))] = (flags, data) - offset += 2 + name_len + (indicies_len * - index_len) + self._context.symbol_space.get_type("vmware!unsigned int").size + offset = offset + 2 + name_len + (indices_len * index_len)) + # Skip two bytes of padding (as it seems?) + # Read the actual data + data = self._context.object("vmware!bytes", + layer_name = self._meta_layer, + offset = offset + 2 + name_len + (indices_len * index_len) + + 2 * data_len + 2, + length = data_size) + offset += 2 + name_len + (indices_len * index_len) + 2 * data_len + 2 + data_size + else: # Handle regular cases + data = self._context.object(self._choose_type(data_len), + layer_name = self._meta_layer, + offset = offset + 2 + name_len + (indices_len * index_len)) + offset += 2 + name_len + (indices_len * index_len) + data_len + + tags[(name, tuple(indices))] = (flags, data) if tags[("regionsCount", ())][1] == 0: raise VmwareFormatException(self.name, "VMware VMEM is not split into regions") @@ -130,14 +152,16 @@ def stack(cls, current_config_path = interfaces.configuration.path_join("automagic", "layer_stacker", "stack", current_layer_name) + vmss_success = False try: _ = resources.ResourceAccessor().open(vmss).read(10) context.config[interfaces.configuration.path_join(current_config_path, "location")] = vmss context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) vmss_success = True except IOError: - vmss_success = False + pass + vmsn_success = False if not vmss_success: try: _ = resources.ResourceAccessor().open(vmsn).read(10) @@ -145,7 +169,9 @@ def stack(cls, context.layers.add_layer(physical.FileLayer(context, current_config_path, current_layer_name)) vmsn_success = True except IOError: - vmsn_success = False + pass + + vollog.log(constants.LOGLEVEL_VVVV, f"Metadata found: VMSS ({vmss_success}) or VMSN ({vmsn_success})") if not vmss_success and not vmsn_success: return None diff --git a/volatility3/framework/objects/__init__.py b/volatility3/framework/objects/__init__.py index 6504b12591..6ee24407f6 100644 --- a/volatility3/framework/objects/__init__.py +++ b/volatility3/framework/objects/__init__.py @@ -3,6 +3,7 @@ # import collections +import collections.abc import logging import struct from typing import Any, ClassVar, Dict, List, Iterable, Optional, Tuple, Type, Union as TUnion, overload @@ -30,7 +31,7 @@ def convert_data_to_value(data: bytes, struct_type: Type[TUnion[int, float, byte elif struct_type in [bytes, str]: struct_format = str(data_format.length) + "s" else: - raise TypeError("Cannot construct struct format for type {}".format(type(struct_type))) + raise TypeError(f"Cannot construct struct format for type {type(struct_type)}") return struct.unpack(struct_format, data)[0] @@ -40,7 +41,7 @@ def convert_value_to_data(value: TUnion[int, float, bytes, str, bool], struct_ty data_format: DataFormatInfo) -> bytes: """Converts a particular value to a series of bytes.""" if not isinstance(value, struct_type): - raise TypeError("Written value is not of the correct type for {}".format(struct_type.__class__.__name__)) + raise TypeError(f"Written value is not of the correct type for {struct_type.__name__}") if struct_type == int and isinstance(value, int): # Doubling up on the isinstance is for mypy @@ -56,9 +57,11 @@ def convert_value_to_data(value: TUnion[int, float, bytes, str, bool], struct_ty raise ValueError("Invalid float size") struct_format = ("<" if data_format.byteorder == 'little' else ">") + float_vals[data_format.length] elif struct_type in [bytes, str]: + if isinstance(value, str): + value = bytes(value, 'latin-1') struct_format = str(data_format.length) + "s" else: - raise TypeError("Cannot construct struct format for type {}".format(type(struct_type))) + raise TypeError(f"Cannot construct struct format for type {type(struct_type)}") return struct.pack(struct_format, value) @@ -92,7 +95,7 @@ class Function(interfaces.objects.ObjectInterface): class PrimitiveObject(interfaces.objects.ObjectInterface): """PrimitiveObject is an interface for any objects that should simulate a Python primitive.""" - _struct_type = int # type: ClassVar[Type] + _struct_type: ClassVar[Type] = int def __init__(self, context: interfaces.context.ContextInterface, type_name: str, object_info: interfaces.objects.ObjectInformation, data_format: DataFormatInfo) -> None: @@ -109,7 +112,7 @@ def __new__(cls: Type, """Creates the appropriate class and returns it so that the native type is inherited. - The only reason the kwargs is added, is so that the inherriting types can override __init__ + The only reason the kwargs is added, is so that the inheriting types can override __init__ without needing to override __new__ We also sneak in new_value, so that we don't have to do expensive (read: impossible) context reads @@ -126,7 +129,7 @@ def __new__(cls: Type, return result def __getnewargs_ex__(self): - """Make sure that when pickling, all appropiate parameters for new are + """Make sure that when pickling, all appropriate parameters for new are provided.""" kwargs = {} for k, v in self._vol.maps[-1].items(): @@ -148,11 +151,12 @@ def size(cls, template: interfaces.objects.Template) -> int: """Returns the size of the templated object.""" return template.vol.data_format.length - def write(self, value: TUnion[int, float, bool, bytes, str]) -> None: + def write(self, value: TUnion[int, float, bool, bytes, str]) -> interfaces.objects.ObjectInterface: """Writes the object into the layer of the context at the current offset.""" data = convert_value_to_data(value, self._struct_type, self._data_format) - return self._context.layers.write(self.vol.layer_name, self.vol.offset, data) + self._context.layers.write(self.vol.layer_name, self.vol.offset, data) + return self.cast(self.vol.type_name) # This must be int (and the _struct_type must be int) because bool cannot be inherited from: @@ -160,7 +164,7 @@ def write(self, value: TUnion[int, float, bool, bytes, str]) -> None: # https://mail.python.org/pipermail/python-dev/2004-February/042537.html class Boolean(PrimitiveObject, int): """Primitive Object that handles boolean types.""" - _struct_type = int # type: ClassVar[Type] + _struct_type: ClassVar[Type] = int class Integer(PrimitiveObject, int): @@ -169,17 +173,17 @@ class Integer(PrimitiveObject, int): class Float(PrimitiveObject, float): """Primitive Object that handles double or floating point numbers.""" - _struct_type = float # type: ClassVar[Type] + _struct_type: ClassVar[Type] = float class Char(PrimitiveObject, int): """Primitive Object that handles characters.""" - _struct_type = int # type: ClassVar[Type] + _struct_type: ClassVar[Type] = int class Bytes(PrimitiveObject, bytes): """Primitive Object that handles specific series of bytes.""" - _struct_type = bytes # type: ClassVar[Type] + _struct_type: ClassVar[Type] = bytes def __init__(self, context: interfaces.context.ContextInterface, @@ -202,7 +206,7 @@ def __new__(cls: Type, is inherritted. The only reason the kwargs is added, is so that the - inherriting types can override __init__ without needing to + inheriting types can override __init__ without needing to override __new__ """ return cls._struct_type.__new__( @@ -223,7 +227,7 @@ class String(PrimitiveObject, str): max_length: specifies the maximum possible length that the string could hold within memory (for multibyte characters, this will not be the maximum length of the string) """ - _struct_type = str # type: ClassVar[Type] + _struct_type: ClassVar[Type] = str def __init__(self, context: interfaces.context.ContextInterface, @@ -252,7 +256,7 @@ def __new__(cls: Type, is inherited. The only reason the kwargs is added, is so that the - inherriting types can override __init__ without needing to + inheriting types can override __init__ without needing to override __new__ """ params = {} @@ -288,6 +292,7 @@ def __init__(self, subtype: Optional[templates.ObjectTemplate] = None) -> None: super().__init__(context = context, object_info = object_info, type_name = type_name, data_format = data_format) self._vol['subtype'] = subtype + self._cache: Dict[str, interfaces.objects.ObjectInterface] = {} @classmethod def _unmarshall(cls, context: interfaces.context.ContextInterface, data_format: DataFormatInfo, @@ -315,14 +320,22 @@ def dereference(self, layer_name: Optional[str] = None) -> interfaces.objects.Ob defaults to the same layer that the pointer is currently instantiated in. """ - layer_name = layer_name or self.vol.native_layer_name - mask = self._context.layers[layer_name].address_mask - offset = self & mask - return self.vol.subtype(context = self._context, - object_info = interfaces.objects.ObjectInformation(layer_name = layer_name, - offset = offset, - parent = self, - size = self.vol.subtype.size)) + # Do our own caching because lru_cache doesn't seem to memoize correctly across multiple uses + # Cache clearing should be done by a cast (we can add a specific method to reset a pointer, + # but hopefully it's not necessary) + if layer_name is None: + layer_name = self.vol.native_layer_name + if self._cache.get(layer_name, None) is None: + layer_name = layer_name or self.vol.native_layer_name + mask = self._context.layers[layer_name].address_mask + offset = self & mask + self._cache[layer_name] = self.vol.subtype(context = self._context, + object_info = interfaces.objects.ObjectInformation( + layer_name = layer_name, + offset = offset, + parent = self, + size = self.vol.subtype.size)) + return self._cache[layer_name] def is_readable(self, layer_name: Optional[str] = None) -> bool: """Determines whether the address of this pointer can be read from @@ -333,7 +346,7 @@ def is_readable(self, layer_name: Optional[str] = None) -> bool: def __getattr__(self, attr: str) -> Any: """Convenience function to access unknown attributes by getting them from the subtype object.""" - if attr in ['vol', '_vol']: + if attr in ['vol', '_vol', '_cache']: raise AttributeError("Pointer not initialized before use") return getattr(self.dereference(), attr) @@ -448,13 +461,13 @@ def __hash__(self): @classmethod def _generate_inverse_choices(cls, choices: Dict[str, int]) -> Dict[int, str]: """Generates the inverse choices for the object.""" - inverse_choices = {} # type: Dict[int, str] + inverse_choices: Dict[int, str] = {} for k, v in choices.items(): if v in inverse_choices: # Technically this shouldn't be a problem, but since we inverse cache # and can't map one value to two possibilities we throw an exception during build # We can remove/work around this if it proves a common issue - raise ValueError("Enumeration value {} duplicated as {} and {}".format(v, k, inverse_choices[v])) + raise ValueError(f"Enumeration value {v} duplicated as {k} and {inverse_choices[v]}") inverse_choices[v] = k return inverse_choices @@ -484,7 +497,7 @@ def __getattr__(self, attr: str) -> str: """Returns the value for a specific name.""" if attr in self._vol['choices']: return self._vol['choices'][attr] - raise AttributeError("Unknown attribute {} for Enumeration {}".format(attr, self._vol['type_name'])) + raise AttributeError(f"Unknown attribute {attr} for Enumeration {self._vol['type_name']}") def write(self, value: bytes): raise NotImplementedError("Writing to Enumerations is not yet implemented") @@ -549,6 +562,10 @@ def count(self, value: int) -> None: self._vol['count'] = value self._vol['size'] = value * self._vol['subtype'].size + def __repr__(self) -> str: + """Describes the object appropriately""" + return AggregateType.__repr__(self) + class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): @classmethod @@ -580,7 +597,7 @@ def relative_child_offset(cls, template: interfaces.objects.Template, child: str the child member.""" if 'subtype' in template.vol and child == 'subtype': return 0 - raise IndexError("Member not present in array template: {}".format(child)) + raise IndexError(f"Member not present in array template: {child}") @overload def __getitem__(self, i: int) -> interfaces.objects.Template: @@ -592,7 +609,7 @@ def __getitem__(self, s: slice) -> List[interfaces.objects.Template]: def __getitem__(self, i): """Returns the i-th item from the array.""" - result = [] # type: List[interfaces.objects.Template] + result: List[interfaces.objects.Template] = [] mask = self._context.layers[self.vol.layer_name].address_mask # We use the range function to deal with slices for us series = range(self.vol.count)[i] @@ -617,7 +634,11 @@ def __len__(self) -> int: return self.vol.count def write(self, value) -> None: - raise NotImplementedError("Writing to Arrays is not yet implemented") + if not isinstance(value, collections.abc.Sequence): + raise TypeError("Only Sequences can be written to arrays") + self.count = len(value) + for index in range(len(value)): + self[index].write(value[index]) class AggregateType(interfaces.objects.ObjectInterface): @@ -636,13 +657,22 @@ def __init__(self, context: interfaces.context.ContextInterface, type_name: str, size = size, members = members) # self._check_members(members) - self._concrete_members = {} # type: Dict[str, Dict] + self._concrete_members: Dict[str, Dict] = {} def has_member(self, member_name: str) -> bool: """Returns whether the object would contain a member called member_name.""" return member_name in self.vol.members + def __repr__(self) -> str: + """Describes the object appropriately""" + extras = member_name = '' + if self.vol.native_layer_name != self.vol.layer_name: + extras += f" (Native: {self.vol.native_layer_name})" + if self.vol.member_name: + member_name = f" (.{self.vol.member_name})" + return f"<{self.__class__.__name__} {self.vol.type_name}{member_name}: {self.vol.layer_name} @ 0x{self.vol.offset:x} #{self.vol.size}{extras}>" + class VolTemplateProxy(interfaces.objects.ObjectInterface.VolTemplateProxy): @classmethod @@ -679,7 +709,7 @@ def relative_child_offset(cls, template: interfaces.objects.Template, child: str """Returns the relative offset of a child to its parent.""" retlist = template.vol.members.get(child, None) if retlist is None: - raise IndexError("Member not present in template: {}".format(child)) + raise IndexError(f"Member not present in template: {child}") return retlist[0] @classmethod @@ -700,9 +730,9 @@ def _check_members(cls, members: Dict[str, Tuple[int, interfaces.objects.Templat agg_name = agg_type.__name__ assert isinstance(members, collections.abc.Mapping) - "{} members parameter must be a mapping: {}".format(agg_name, type(members)) + f"{agg_name} members parameter must be a mapping: {type(members)}" assert all([(isinstance(member, tuple) and len(member) == 2) for member in members.values()]) - "{} members must be a tuple of relative_offsets and templates".format(agg_name) + f"{agg_name} members must be a tuple of relative_offsets and templates" def member(self, attr: str = 'member') -> object: """Specifically named method for retrieving members.""" @@ -710,11 +740,14 @@ def member(self, attr: str = 'member') -> object: def __getattr__(self, attr: str) -> Any: """Method for accessing members of the type.""" + if attr in ['_concrete_members', 'vol']: raise AttributeError("Object has not been properly initialized") if attr in self._concrete_members: return self._concrete_members[attr] - elif attr in self.vol.members: + if attr.startswith("_") and not attr.startswith("__") and "__" in attr: + attr = attr[attr.find("__", 1):] # See issue #522 + if attr in self.vol.members: mask = self._context.layers[self.vol.layer_name].address_mask relative_offset, template = self.vol.members[attr] if isinstance(template, templates.ReferenceTemplate): @@ -733,7 +766,18 @@ def __getattr__(self, attr: str) -> Any: for agg_type in AggregateTypes: if isinstance(self, agg_type): agg_name = agg_type.__name__ - raise AttributeError("{} has no attribute: {}.{}".format(agg_name, self.vol.type_name, attr)) + raise AttributeError(f"{agg_name} has no attribute: {self.vol.type_name}.{attr}") + + # Disable messing around with setattr until the consequences have been considered properly + # For example pdbutil constructs objects and then sets values for them + # Some don't always match the type (for example, the data read is encoded and interpreted) + # + # def __setattr__(self, name, value): + # """Method for writing specific members of a structure""" + # if name in ['_concrete_members', 'vol', '_vol'] or not self.has_member(name): + # return super().__setattr__(name, value) + # attr = self.__getattr__(name) + # return attr.write(value) def __dir__(self) -> Iterable[str]: """Returns a complete list of members when dir is called.""" @@ -746,7 +790,7 @@ def write(self, value): if isinstance(self, agg_type): agg_name = agg_type.__name__ raise TypeError( - "{}s cannot be written to directly, individual members must be written instead".format(agg_name)) + f"{agg_name}s cannot be written to directly, individual members must be written instead") class StructType(AggregateType): diff --git a/volatility3/framework/objects/templates.py b/volatility3/framework/objects/templates.py index 05938ff9c5..b544d117fb 100644 --- a/volatility3/framework/objects/templates.py +++ b/volatility3/framework/objects/templates.py @@ -65,7 +65,7 @@ def __call__(self, context: interfaces.context.ContextInterface, Returns: an object adhereing to the :class:`~volatility3.framework.interfaces.objects.ObjectInterface` """ - arguments = {} # type: Dict[str, Any] + arguments: Dict[str, Any] = {} for arg in self.vol: if arg != 'object_class': arguments[arg] = self.vol[arg] @@ -94,12 +94,12 @@ def _unresolved(self, *args, **kwargs) -> Any: symbol_name = type_name[-1] raise exceptions.SymbolError( symbol_name, table_name, - "Template contains no information about its structure: {}".format(self.vol.type_name)) + f"Template contains no information about its structure: {self.vol.type_name}") - size = property(_unresolved) # type: ClassVar[Any] - replace_child = _unresolved # type: ClassVar[Any] - relative_child_offset = _unresolved # type: ClassVar[Any] - has_member = _unresolved # type: ClassVar[Any] + size: ClassVar[Any] = property(_unresolved) + replace_child: ClassVar[Any] = _unresolved + relative_child_offset: ClassVar[Any] = _unresolved + has_member: ClassVar[Any] = _unresolved def __call__(self, context: interfaces.context.ContextInterface, object_info: interfaces.objects.ObjectInformation): template = context.symbol_space.get_type(self.vol.type_name) diff --git a/volatility3/framework/plugins/__init__.py b/volatility3/framework/plugins/__init__.py index aa9701a4cd..7dbce52085 100644 --- a/volatility3/framework/plugins/__init__.py +++ b/volatility3/framework/plugins/__init__.py @@ -44,7 +44,7 @@ def construct_plugin(context: interfaces.context.ContextInterface, if unsatisfied: for error in errors: error_string = [x for x in error.format_exception_only()][-1] - vollog.warning("Automagic exception occurred: {}".format(error_string[:-1])) + vollog.warning(f"Automagic exception occurred: {error_string[:-1]}") vollog.log(constants.LOGLEVEL_V, "".join(error.format(chain = True))) raise exceptions.UnsatisfiedException(unsatisfied) diff --git a/volatility3/framework/plugins/banners.py b/volatility3/framework/plugins/banners.py index c907497ed8..ac20062074 100644 --- a/volatility3/framework/plugins/banners.py +++ b/volatility3/framework/plugins/banners.py @@ -15,7 +15,7 @@ class Banners(interfaces.plugins.PluginInterface): """Attempts to identify potential linux banners in an image""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/plugins/configwriter.py b/volatility3/framework/plugins/configwriter.py index 6dc504fb5c..f0979eb4de 100644 --- a/volatility3/framework/plugins/configwriter.py +++ b/volatility3/framework/plugins/configwriter.py @@ -17,7 +17,7 @@ class ConfigWriter(plugins.PluginInterface): """Runs the automagics and both prints and outputs configuration in the output directory.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -42,7 +42,7 @@ def _generator(self): with self.open(filename) as file_data: file_data.write(bytes(json.dumps(config, sort_keys = True, indent = 2), 'raw_unicode_escape')) except Exception as excp: - vollog.warning("Unable to JSON encode configuration: {}".format(excp)) + vollog.warning(f"Unable to JSON encode configuration: {excp}") for k, v in config.items(): yield (0, (k, json.dumps(v))) diff --git a/volatility3/framework/plugins/frameworkinfo.py b/volatility3/framework/plugins/frameworkinfo.py index 8d9f3b0137..b7c887d5cf 100644 --- a/volatility3/framework/plugins/frameworkinfo.py +++ b/volatility3/framework/plugins/frameworkinfo.py @@ -8,7 +8,7 @@ class FrameworkInfo(plugins.PluginInterface): """Plugin to list the various modular components of Volatility""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/framework/plugins/isfinfo.py b/volatility3/framework/plugins/isfinfo.py index c1f5daef1b..575f254269 100644 --- a/volatility3/framework/plugins/isfinfo.py +++ b/volatility3/framework/plugins/isfinfo.py @@ -22,7 +22,7 @@ class IsfInfo(plugins.PluginInterface): """Determines information about the currently available ISF files, or a specific one""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod @@ -117,7 +117,7 @@ def check_valid(data): windows_info = os.path.splitext(os.path.basename(entry))[0] valid = check_valid(data) except (UnicodeDecodeError, json.decoder.JSONDecodeError): - vollog.warning("Invalid ISF: {}".format(entry)) + vollog.warning(f"Invalid ISF: {entry}") yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, windows_info, linux_banner, mac_banner)) diff --git a/volatility3/framework/plugins/layerwriter.py b/volatility3/framework/plugins/layerwriter.py index 721845fd3d..b2a02116e9 100644 --- a/volatility3/framework/plugins/layerwriter.py +++ b/volatility3/framework/plugins/layerwriter.py @@ -17,15 +17,13 @@ class LayerWriter(plugins.PluginInterface): default_block_size = 0x500000 - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), + requirements.TranslationLayerRequirement(name = 'primary', description = 'Memory layer for the kernel'), requirements.IntRequirement(name = 'block_size', description = "Size of blocks to copy over", default = cls.default_block_size, @@ -75,7 +73,7 @@ def write_layer( data = layer.read(i, current_chunk_size, pad = True) file_handle.write(data) if progress_callback: - progress_callback((i / layer.maximum_address) * 100, 'Writing layer {}'.format(layer_name)) + progress_callback((i / layer.maximum_address) * 100, f'Writing layer {layer_name}') return file_handle def _generator(self): @@ -93,7 +91,7 @@ def _generator(self): for name in self.config['layers']: # Check the layer exists and validate the output file if name not in self.context.layers: - yield 0, ('Layer Name {} does not exist'.format(name), ) + yield 0, (f'Layer Name {name} does not exist', ) else: output_name = self.config.get('output', ".".join([name, "raw"])) try: @@ -105,9 +103,9 @@ def _generator(self): progress_callback = self._progress_callback) file_handle.close() except IOError as excp: - yield 0, ('Layer cannot be written to {}: {}'.format(self.config['output_name'], excp), ) + yield 0, (f"Layer cannot be written to {self.config['output_name']}: {excp}", ) - yield 0, ('Layer has been written to {}'.format(output_name), ) + yield 0, (f'Layer has been written to {output_name}', ) def _generate_layers(self): """List layer names from this run""" diff --git a/volatility3/framework/plugins/linux/bash.py b/volatility3/framework/plugins/linux/bash.py index 8e6de7241b..7f606115cf 100644 --- a/volatility3/framework/plugins/linux/bash.py +++ b/volatility3/framework/plugins/linux/bash.py @@ -21,16 +21,14 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): """Recovers bash command history from memory.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.ListRequirement(name = 'pid', element_type = int, description = "Process IDs to include (all other processes are excluded)", @@ -38,7 +36,8 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] ] def _generator(self, tasks): - is_32bit = not symbols.symbol_table_is_64bit(self.context, self.config["vmlinux"]) + vmlinux = self.context.modules[self.config["kernel"]] + is_32bit = not symbols.symbol_table_is_64bit(self.context, vmlinux.symbol_table_name) if is_32bit: pack_format = "I" bash_json_file = "bash32" @@ -72,15 +71,16 @@ def _generator(self, tasks): history_entries = [] - for address, _ in proc_layer.scan(self.context, - scanners.MultiStringScanner(bang_addrs), - sections = task.get_process_memory_sections(heap_only = True)): - hist = self.context.object(bash_table_name + constants.BANG + "hist_entry", - offset = address - ts_offset, - layer_name = proc_layer_name) + if bang_addrs: + for address, _ in proc_layer.scan(self.context, + scanners.MultiStringScanner(bang_addrs), + sections = task.get_process_memory_sections(heap_only = True)): + hist = self.context.object(bash_table_name + constants.BANG + "hist_entry", + offset = address - ts_offset, + layer_name = proc_layer_name) - if hist.is_valid(): - history_entries.append(hist) + if hist.is_valid(): + history_entries.append(hist) for hist in sorted(history_entries, key = lambda x: x.get_time_as_integer()): yield (0, (task.pid, task_name, hist.get_time_object(), hist.get_command())) @@ -92,8 +92,7 @@ def run(self): ("Command", str)], self._generator( pslist.PsList.list_tasks(self.context, - self.config['primary'], - self.config['vmlinux'], + self.config['kernel'], filter_func = filter_func))) def generate_timeline(self): @@ -101,9 +100,8 @@ def generate_timeline(self): for row in self._generator( pslist.PsList.list_tasks(self.context, - self.config['primary'], - self.config['vmlinux'], + self.config['kernel'], filter_func = filter_func)): _depth, row_data = row - description = "{} ({}): \"{}\"".format(row_data[0], row_data[1], row_data[3]) + description = f"{row_data[0]} ({row_data[1]}): \"{row_data[3]}\"" yield (description, timeliner.TimeLinerType.CREATED, row_data[2]) diff --git a/volatility3/framework/plugins/linux/check_afinfo.py b/volatility3/framework/plugins/linux/check_afinfo.py index 29e6975408..4cd065c7ea 100644 --- a/volatility3/framework/plugins/linux/check_afinfo.py +++ b/volatility3/framework/plugins/linux/check_afinfo.py @@ -6,7 +6,7 @@ import logging from typing import List -from volatility3.framework import exceptions, interfaces, contexts +from volatility3.framework import exceptions, interfaces from volatility3.framework import renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins @@ -18,15 +18,13 @@ class Check_afinfo(plugins.PluginInterface): """Verifies the operation function pointers of network protocols.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols") + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), ] # returns whether the symbol is found within the kernel (system.map) or not @@ -63,7 +61,8 @@ def _check_afinfo(self, var_name, var, op_members, seq_members): yield var_name, "show", var.seq_show def _generator(self): - vmlinux = contexts.Module(self.context, self.config['vmlinux'], self.config['primary'], 0) + + vmlinux = self.context.modules[self.config['kernel']] op_members = vmlinux.get_type('file_operations').members seq_members = vmlinux.get_type('seq_operations').members diff --git a/volatility3/framework/plugins/linux/check_creds.py b/volatility3/framework/plugins/linux/check_creds.py index 2333eb1231..613469eed1 100644 --- a/volatility3/framework/plugins/linux/check_creds.py +++ b/volatility3/framework/plugins/linux/check_creds.py @@ -4,7 +4,7 @@ import logging -from volatility3.framework import interfaces, renderers, constants +from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.plugins.linux import pslist @@ -14,22 +14,20 @@ class Check_creds(interfaces.plugins.PluginInterface): """Checks if any processes are sharing credential structures""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)) + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)) ] def _generator(self): - # vmlinux = contexts.Module(self.context, self.config['vmlinux'], self.config['primary'], 0) + vmlinux = self.context.modules[self.config['kernel']] - type_task = self.context.symbol_space.get_type(self.config['vmlinux'] + constants.BANG + "task_struct") + type_task = vmlinux.get_type("task_struct") if not type_task.has_member("cred"): raise TypeError( @@ -40,7 +38,7 @@ def _generator(self): creds = {} - tasks = pslist.PsList.list_tasks(self.context, self.config['primary'], self.config['vmlinux']) + tasks = pslist.PsList.list_tasks(self.context, vmlinux.name) for task in tasks: @@ -55,7 +53,7 @@ def _generator(self): if len(pids) > 1: pid_str = "" for pid in pids: - pid_str = pid_str + "{0:d}, ".format(pid) + pid_str = pid_str + f"{pid:d}, " pid_str = pid_str[:-2] yield (0, [str(pid_str)]) diff --git a/volatility3/framework/plugins/linux/check_idt.py b/volatility3/framework/plugins/linux/check_idt.py index f171ab8464..1764b63645 100644 --- a/volatility3/framework/plugins/linux/check_idt.py +++ b/volatility3/framework/plugins/linux/check_idt.py @@ -5,7 +5,7 @@ import logging from typing import List -from volatility3.framework import interfaces, renderers, contexts, symbols +from volatility3.framework import interfaces, renderers, symbols from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import linux @@ -17,32 +17,29 @@ class Check_idt(interfaces.plugins.PluginInterface): """ Checks if the IDT has been altered """ - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.VersionRequirement(name = 'linuxutils', component = linux.LinuxUtilities, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (1, 0, 0)) + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.VersionRequirement(name = 'linuxutils', component = linux.LinuxUtilities, version = (2, 0, 0)), + requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) ] def _generator(self): - vmlinux = contexts.Module(self.context, self.config['vmlinux'], self.config['primary'], 0) + vmlinux = self.context.modules[self.config['kernel']] - modules = lsmod.Lsmod.list_modules(self.context, self.config['primary'], self.config['vmlinux']) + modules = lsmod.Lsmod.list_modules(self.context, vmlinux.name) - handlers = linux.LinuxUtilities.generate_kernel_handler_info(self.context, self.config['primary'], - self.config['vmlinux'], modules) + handlers = linux.LinuxUtilities.generate_kernel_handler_info(self.context, vmlinux.name, modules) - is_32bit = not symbols.symbol_table_is_64bit(self.context, self.config["vmlinux"]) + is_32bit = not symbols.symbol_table_is_64bit(self.context, vmlinux.symbol_table_name) idt_table_size = 256 - address_mask = self.context.layers[self.config['primary']].address_mask + address_mask = self.context.layers[vmlinux.layer_name].address_mask # hw handlers + system call check_idxs = list(range(0, 20)) + [128] @@ -65,7 +62,8 @@ def _generator(self): table = vmlinux.object(object_type = 'array', offset = addrs.vol.offset, subtype = vmlinux.get_type(idt_type), - count = idt_table_size) + count = idt_table_size, + absolute = True) for i in check_idxs: ent = table[i] @@ -88,7 +86,7 @@ def _generator(self): idt_addr = idt_addr & address_mask - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address(self.context, handlers, idt_addr) + module_name, symbol_name = linux.LinuxUtilities.lookup_module_address(vmlinux, handlers, idt_addr) yield (0, [format_hints.Hex(i), format_hints.Hex(idt_addr), module_name, symbol_name]) diff --git a/volatility3/framework/plugins/linux/check_modules.py b/volatility3/framework/plugins/linux/check_modules.py index 449046e866..6af8dec968 100644 --- a/volatility3/framework/plugins/linux/check_modules.py +++ b/volatility3/framework/plugins/linux/check_modules.py @@ -5,7 +5,7 @@ import logging from typing import List -from volatility3.framework import interfaces, renderers, exceptions, constants, contexts +from volatility3.framework import interfaces, renderers, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -18,19 +18,20 @@ class Check_modules(plugins.PluginInterface): """Compares module list to sysfs info, if available""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (1, 0, 0)) + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) ] - def get_kset_modules(self, vmlinux): + @classmethod + def get_kset_modules(self, context: interfaces.context.ContextInterface, vmlinux_name: str): + + vmlinux = context.modules[vmlinux_name] try: module_kset = vmlinux.object_from_symbol("module_kset") @@ -44,12 +45,12 @@ def get_kset_modules(self, vmlinux): ret = {} - kobj_off = self.context.symbol_space.get_type(self.config['vmlinux'] + constants.BANG + - 'module_kobject').relative_child_offset('kobj') + kobj_off = vmlinux.get_type('module_kobject').relative_child_offset('kobj') - for kobj in module_kset.list.to_list(vmlinux.name + constants.BANG + "kobject", "entry"): + for kobj in module_kset.list.to_list(vmlinux.symbol_table_name + constants.BANG + "kobject", "entry"): - mod_kobj = vmlinux.object(object_type = "module_kobject", offset = kobj.vol.offset - kobj_off) + mod_kobj = vmlinux.object(object_type = "module_kobject", offset = kobj.vol.offset - kobj_off, + absolute = True) mod = mod_kobj.mod @@ -60,13 +61,11 @@ def get_kset_modules(self, vmlinux): return ret def _generator(self): - vmlinux = contexts.Module(self.context, self.config['vmlinux'], self.config['primary'], 0) - - kset_modules = self.get_kset_modules(vmlinux) + kset_modules = self.get_kset_modules(self.context, self.config['kernel']) lsmod_modules = set( str(utility.array_to_string(modules.name)) - for modules in lsmod.Lsmod.list_modules(self.context, self.config['primary'], self.config['vmlinux'])) + for modules in lsmod.Lsmod.list_modules(self.context, self.config['kernel'])) for mod_name in set(kset_modules.keys()).difference(lsmod_modules): yield (0, (format_hints.Hex(kset_modules[mod_name]), str(mod_name))) diff --git a/volatility3/framework/plugins/linux/check_syscall.py b/volatility3/framework/plugins/linux/check_syscall.py index b845ad4aab..729a0bec69 100644 --- a/volatility3/framework/plugins/linux/check_syscall.py +++ b/volatility3/framework/plugins/linux/check_syscall.py @@ -6,7 +6,7 @@ import logging from typing import List -from volatility3.framework import exceptions, interfaces, contexts +from volatility3.framework import exceptions, interfaces from volatility3.framework import renderers, constants from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins @@ -25,24 +25,27 @@ class Check_syscall(plugins.PluginInterface): """Check system call table for hooks.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols") + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), ] def _get_table_size_next_symbol(self, table_addr, ptr_sz, vmlinux): """Returns the size of the table based on the next symbol.""" ret = 0 - sym_table = self.context.symbol_space[vmlinux.name] - - sorted_symbols = sorted([(sym_table.get_symbol(sn).address, sn) for sn in sym_table.symbols]) + symbol_list = [] + for sn in vmlinux.symbols: + try: + # When requesting the symbol from the module, a full resolve is performed + symbol_list.append((vmlinux.get_symbol(sn).address, sn)) + except exceptions.SymbolError: + pass + sorted_symbols = sorted(symbol_list) sym_address = 0 @@ -62,7 +65,8 @@ def _get_table_size_meta(self, vmlinux): accurate.""" return len( - [sym for sym in self.context.symbol_space[vmlinux.name].symbols if sym.startswith("__syscall_meta__")]) + [sym for sym in self.context.symbol_space[vmlinux.symbol_table_name].symbols if + sym.startswith("__syscall_meta__")]) def _get_table_info_other(self, table_addr, ptr_sz, vmlinux): table_size_meta = self._get_table_size_meta(vmlinux) @@ -93,12 +97,13 @@ def _get_table_info_disassembly(self, ptr_sz, vmlinux): md = capstone.Cs(capstone.CS_ARCH_X86, mode) try: - func_addr = self.context.symbol_space.get_symbol(vmlinux.name + constants.BANG + syscall_entry_func).address + func_addr = vmlinux.get_symbol(syscall_entry_func).address except exceptions.SymbolError as e: # if we can't find the disassemble function then bail and rely on a different method return 0 - data = self.context.layers.read(self.config['primary'], func_addr, 6) + vmlinux = self.context.modules[self.config['kernel']] + data = self.context.layers.read(vmlinux.layer_name, func_addr, 6) for (address, size, mnemonic, op_str) in md.disasm_lite(data, func_addr): if mnemonic == 'CMP': @@ -108,7 +113,7 @@ def _get_table_info_disassembly(self, ptr_sz, vmlinux): return table_size def _get_table_info(self, vmlinux, table_name, ptr_sz): - table_sym = self.context.symbol_space.get_symbol(vmlinux.name + constants.BANG + table_name) + table_sym = vmlinux.get_symbol(table_name) table_size = self._get_table_info_disassembly(ptr_sz, vmlinux) @@ -123,7 +128,7 @@ def _get_table_info(self, vmlinux, table_name, ptr_sz): # TODO - add finding and parsing unistd.h once cached file enumeration is added def _generator(self): - vmlinux = contexts.Module(self.context, self.config['vmlinux'], self.config['primary'], 0) + vmlinux = self.context.modules[self.config['kernel']] ptr_sz = vmlinux.get_type("pointer").size if ptr_sz == 4: @@ -143,7 +148,7 @@ def _generator(self): # enabled in order to support 32 bit programs and libraries # if the symbol isn't there then the support isn't in the kernel and so we skip it try: - ia32_symbol = self.context.symbol_space.get_symbol(vmlinux.name + constants.BANG + "ia32_sys_call_table") + ia32_symbol = vmlinux.get_symbol("ia32_sys_call_table") except exceptions.SymbolError: ia32_symbol = None @@ -161,7 +166,7 @@ def _generator(self): if not call_addr: continue - symbols = list(self.context.symbol_space.get_symbols_by_location(call_addr)) + symbols = list(vmlinux.get_symbols_by_absolute_location(call_addr)) if len(symbols) > 0: sym_name = str(symbols[0].split(constants.BANG)[1]) if constants.BANG in symbols[0] else \ diff --git a/volatility3/framework/plugins/linux/elfs.py b/volatility3/framework/plugins/linux/elfs.py index 89909f4ff5..d2380817d4 100644 --- a/volatility3/framework/plugins/linux/elfs.py +++ b/volatility3/framework/plugins/linux/elfs.py @@ -17,16 +17,14 @@ class Elfs(plugins.PluginInterface): """Lists all memory mapped ELF files for all processes.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -59,6 +57,5 @@ def run(self): ("End", format_hints.Hex), ("File Path", str)], self._generator( pslist.PsList.list_tasks(self.context, - self.config['primary'], - self.config['vmlinux'], + self.config['kernel'], filter_func = filter_func))) diff --git a/volatility3/framework/plugins/linux/keyboard_notifiers.py b/volatility3/framework/plugins/linux/keyboard_notifiers.py index 290c0a180b..51e684e6f4 100644 --- a/volatility3/framework/plugins/linux/keyboard_notifiers.py +++ b/volatility3/framework/plugins/linux/keyboard_notifiers.py @@ -4,7 +4,7 @@ import logging -from volatility3.framework import interfaces, renderers, contexts, exceptions +from volatility3.framework import interfaces, renderers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import linux @@ -16,26 +16,23 @@ class Keyboard_notifiers(interfaces.plugins.PluginInterface): """Parses the keyboard notifier call chain""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (1, 0, 0)), - requirements.VersionRequirement(name = 'linuxutils', component = linux.LinuxUtilities, version = (1, 0, 0)) + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)), + requirements.VersionRequirement(name = 'linuxutils', component = linux.LinuxUtilities, version = (2, 0, 0)) ] def _generator(self): - vmlinux = contexts.Module(self.context, self.config['vmlinux'], self.config['primary'], 0) + vmlinux = self.context.modules[self.config['kernel']] - modules = lsmod.Lsmod.list_modules(self.context, self.config['primary'], self.config['vmlinux']) + modules = lsmod.Lsmod.list_modules(self.context, vmlinux.name) - handlers = linux.LinuxUtilities.generate_kernel_handler_info(self.context, self.config['primary'], - self.config['vmlinux'], modules) + handlers = linux.LinuxUtilities.generate_kernel_handler_info(self.context, vmlinux.name, modules) try: knl_addr = vmlinux.object_from_symbol("keyboard_notifier_list") @@ -49,12 +46,12 @@ def _generator(self): "This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt." ) - knl = vmlinux.object(object_type = "atomic_notifier_head", offset = knl_addr.vol.offset) + knl = vmlinux.object(object_type = "atomic_notifier_head", offset = knl_addr.vol.offset, absolute = True) for call_back in linux.LinuxUtilities.walk_internal_list(vmlinux, "notifier_block", "next", knl.head): call_addr = call_back.notifier_call - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address(self.context, handlers, call_addr) + module_name, symbol_name = linux.LinuxUtilities.lookup_module_address(vmlinux, handlers, call_addr) yield (0, [format_hints.Hex(call_addr), module_name, symbol_name]) diff --git a/volatility3/framework/plugins/linux/kmsg.py b/volatility3/framework/plugins/linux/kmsg.py new file mode 100644 index 0000000000..a729c1695a --- /dev/null +++ b/volatility3/framework/plugins/linux/kmsg.py @@ -0,0 +1,387 @@ +# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# +import logging +from abc import ABC, abstractmethod +from enum import Enum +from typing import List, Iterator, Tuple, Generator + +from volatility3.framework import renderers, interfaces, constants, contexts, class_subclasses +from volatility3.framework.configuration import requirements +from volatility3.framework.interfaces import plugins +from volatility3.framework.objects import utility + +vollog = logging.getLogger(__name__) + + +class DescStateEnum(Enum): + desc_miss = -1 # ID mismatch (pseudo state) + desc_reserved = 0x0 # reserved, in use by writer + desc_committed = 0x1 # committed by writer, could get reopened + desc_finalized = 0x2 # committed, no further modification allowed + desc_reusable = 0x3 # free, not yet used by any writer + + +class ABCKmsg(ABC): + """Kernel log buffer reader""" + LEVELS = ( + "emerg", # system is unusable + "alert", # action must be taken immediately + "crit", # critical conditions + "err", # error conditions + "warn", # warning conditions + "notice", # normal but significant condition + "info", # informational + "debug", # debug-level messages + ) + + FACILITIES = ( + "kern", # kernel messages + "user", # random user-level messages + "mail", # mail system + "daemon", # system daemons + "auth", # security/authorization messages + "syslog", # messages generated internally by syslogd + "lpr", # line printer subsystem + "news", # network news subsystem + "uucp", # UUCP subsystem + "cron", # clock daemon + "authpriv", # security/authorization messages (private) + "ftp" # FTP daemon + ) + + def __init__( + self, + context: interfaces.context.ContextInterface, + config: interfaces.configuration.HierarchicalDict + ): + self._context = context + self._config = config + vmlinux = context.modules[self._config['kernel']] + self.layer_name = vmlinux.layer_name # type: ignore + symbol_table_name = vmlinux.symbol_table_name # type: ignore + self.vmlinux = contexts.Module.create(context, symbol_table_name, self.layer_name, 0) # type: ignore + self.long_unsigned_int_size = self.vmlinux.get_type('long unsigned int').size + + @classmethod + def run_all( + cls, + context: interfaces.context.ContextInterface, + config: interfaces.configuration.HierarchicalDict + ) -> Iterator[Tuple[str, str, str, str, str]]: + """It calls each subclass symtab_checks() to test the required + conditions to that specific kernel implementation. + + Args: + context: The volatility3 context on which to operate + config: Core configuration + + Yields: + kmsg records + """ + vmlinux = context.modules[config['kernel']] + + kmsg_inst = None # type: ignore + for subclass in class_subclasses(cls): + if not subclass.symtab_checks(vmlinux = vmlinux): + vollog.log(constants.LOGLEVEL_VVVV, + "Kmsg implementation '%s' doesn't match this memory dump", subclass.__name__) + continue + + vollog.log(constants.LOGLEVEL_VVVV, "Kmsg implementation '%s' matches!", subclass.__name__) + kmsg_inst = subclass(context = context, config = config) + # More than one class could be executed for an specific kernel + # version i.e. Netfilter Ingress hooks + # We expect just one implementation to be executed for an specific kernel + yield from kmsg_inst.run() + break + + if kmsg_inst is None: + vollog.error("Unsupported Netfilter kernel implementation") + + @abstractmethod + def run(self) -> Iterator[Tuple[str, str, str, str, str]]: + """Walks through the specific kernel implementation.""" + + @classmethod + @abstractmethod + def symtab_checks(cls, vmlinux: interfaces.context.ModuleInterface) -> bool: + """This method on each sublasss will be called to evaluate if the kernel + being analyzed fulfill the type & symbols requirements for the implementation. + The first class returning True will be instantiated and called via the + run() method. + + :return: True is the kernel being analysed fulfill the class requirements. + """ + + def get_string(self, addr: int, length: int) -> str: + txt = self._context.layers[self.layer_name].read(addr, length) # type: ignore + return txt.decode(encoding = 'utf8', errors = 'replace') + + def nsec_to_sec_str(self, nsec: int) -> str: + # See kernel/printk/printk.c:print_time() + # Here, we could simply do: + # "%.6f" % (nsec / 1000000000.0) + # However, that will cause a roundoff error. For instance, using + # 17110365556 as input, the above will result in 17.110366. + # While the kernel print_time function will result in 17.110365. + # This might seem insignificant but it could cause some issues + # when compared with userland tool results or when used in + # timelines. + return "%lu.%06lu" % (nsec / 1000000000, (nsec % 1000000000) / 1000) + + def get_timestamp_in_sec_str(self, obj) -> str: + # obj could be printk_log or printk_info + return self.nsec_to_sec_str(obj.ts_nsec) + + def get_caller(self, obj): + # In some kernel versions, it's only available if CONFIG_PRINTK_CALLER is defined. + # caller_id is a member of printk_log struct from 5.1 to the latest 5.9 + # From kernels 5.10 on, it's a member of printk_info struct + if obj.has_member('caller_id'): + return self.get_caller_text(obj.caller_id) + else: + return "" + + def get_caller_text(self, caller_id): + caller_name = 'CPU' if caller_id & 0x80000000 else 'Task' + caller = "%s(%u)" % (caller_name, caller_id & ~0x80000000) + return caller + + def get_prefix(self, obj) -> Tuple[int, int, str, str]: + # obj could be printk_log or printk_info + return obj.facility, obj.level, self.get_timestamp_in_sec_str(obj), self.get_caller(obj) + + @classmethod + def get_level_text(cls, level: int) -> str: + if level < len(cls.LEVELS): + return cls.LEVELS[level] + else: + vollog.debug(f"Level {level} unknown") + return str(level) + + @classmethod + def get_facility_text(cls, facility: int) -> str: + if facility < len(cls.FACILITIES): + return cls.FACILITIES[facility] + else: + vollog.debug(f"Facility {facility} unknown") + return str(facility) + + +class KmsgLegacy(ABCKmsg): + """Linux kernels prior to v5.10, the ringbuffer is initially kept in + __log_buf, and log_buf is a pointer to the former. __log_buf is declared as + a char array but it actually contains an array of printk_log structs. + The length of this array is defined in the kernel KConfig configuration via + the CONFIG_LOG_BUF_SHIFT value as a power of 2. + This can also be modified by the log_buf_len kernel boot parameter. + In SMP systems with more than 64 CPUs this ringbuffer size is dynamically + allocated according the number of CPUs based on the value of + CONFIG_LOG_CPU_MAX_BUF_SHIFT, and the log_buf pointer is updated + consequently to the new buffer. + In that case, the original static buffer in __log_buf is unused. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return vmlinux.has_type('printk_log') + + def get_text_from_printk_log(self, msg) -> str: + msg_offset = msg.vol.offset + self.vmlinux.get_type('printk_log').size + return self.get_string(msg_offset, msg.text_len) + + def get_log_lines(self, msg) -> Generator[str, None, None]: + if msg.text_len > 0: + text = self.get_text_from_printk_log(msg) + yield from text.splitlines() + + def get_dict_lines(self, msg) -> Generator[str, None, None]: + if msg.dict_len == 0: + return None + dict_offset = msg.vol.offset + self.vmlinux.get_type('printk_log').size + msg.text_len + dict_data = self._context.layers[self.layer_name].read(dict_offset, msg.dict_len) + for chunk in dict_data.split(b'\x00'): + yield " " + chunk.decode() + + def run(self) -> Iterator[Tuple[str, str, str, str, str]]: + log_buf_ptr = self.vmlinux.object_from_symbol(symbol_name = 'log_buf') + if log_buf_ptr == 0: + # This is weird, let's fallback to check the static ringbuffer. + log_buf_ptr = self.vmlinux.object_from_symbol(symbol_name = '__log_buf').vol.offset + if log_buf_ptr == 0: + raise ValueError("Log buffer is not available") + + log_first_idx = int(self.vmlinux.object_from_symbol(symbol_name = 'log_first_idx')) + cur_idx = log_first_idx + end_idx = None # We don't need log_next_idx here. See below msg.len == 0 + while cur_idx != end_idx: + end_idx = log_first_idx + msg_offset = log_buf_ptr + cur_idx # type: ignore + msg = self.vmlinux.object(object_type = 'printk_log', offset = msg_offset) + if msg.len == 0: + # As per kernel/printk/printk.c: + # A length == 0 for the next message indicates a wrap-around to + # the beginning of the buffer. + cur_idx = 0 + else: + facility, level, timestamp, caller = self.get_prefix(msg) + level_txt = self.get_level_text(level) + facility_txt = self.get_facility_text(facility) + + for line in self.get_log_lines(msg): + yield facility_txt, level_txt, timestamp, caller, line + for line in self.get_dict_lines(msg): + yield facility_txt, level_txt, timestamp, caller, line + + cur_idx += msg.len + + +class KmsgFiveTen(ABCKmsg): + """In 5.10 the kernel ringbuffer implementation changed. + Previously only one process should read /proc/kmsg and it is permanently + open and periodically read by the syslog daemon. + A high level structure 'printk_ringbuffer' was added to represent the printk + ringbuffer which actually contains two ringbuffers. The descriptor ring + 'desc_ring' contains the records' metadata, text offsets and states. + The data block ring 'text_data_ring' contains the records' text strings. + A pointer to the high level structure is kept in the prb pointer which is + initialized to a static ringbuffer. + static struct printk_ringbuffer *prb = &printk_rb_static; + In SMP systems with more than 64 CPUs this ringbuffer size is dynamically + allocated according the number of CPUs based on the value of + CONFIG_LOG_CPU_MAX_BUF_SHIFT. The prb pointer is updated consequently to + this dynamic ringbuffer in setup_log_buf(). + prb = &printk_rb_dynamic; + Behind scenes, log_buf is still used as external buffer. + When the static printk_ringbuffer struct is initialized, _DEFINE_PRINTKRB + sets text_data_ring.data pointer to the address in log_buf which points to + the static buffer __log_buff. + If a dynamic ringbuffer takes place, setup_log_buf() sets + text_data_ring.data of printk_rb_dynamic to the new allocated external + buffer via the prb_init function. + In that case, the original external static buffer in __log_buf and + printk_rb_static are unused. + ... + new_log_buf = memblock_alloc(new_log_buf_len, LOG_ALIGN); + prb_init(&printk_rb_dynamic, new_log_buf, ...); + log_buf = new_log_buf; + prb = &printk_rb_dynamic; + ... + See printk.c and printk_ringbuffer.c in kernel/printk/ folder for more + details. + """ + + @classmethod + def symtab_checks(cls, vmlinux) -> bool: + return vmlinux.has_symbol('prb') + + def get_text_from_data_ring(self, text_data_ring, desc, info) -> str: + text_data_sz = text_data_ring.size_bits + text_data_mask = 1 << text_data_sz + + begin = desc.text_blk_lpos.begin % text_data_mask + end = desc.text_blk_lpos.next % text_data_mask + + # This record doesn't contain text + if begin & 1: + return "" + + # This means a wrap-around to the beginning of the buffer + if begin > end: + begin = 0 + + # Each element in the ringbuffer is "ID + data". + # See prb_data_ring struct + desc_id_size = self.long_unsigned_int_size + text_start = begin + desc_id_size + offset = text_data_ring.data + text_start + + # Safety first ;) + text_len = min(info.text_len, end - begin) + + return self.get_string(offset, text_len) + + def get_log_lines(self, text_data_ring, desc, info) -> Generator[str, None, None]: + text = self.get_text_from_data_ring(text_data_ring, desc, info) + yield from text.splitlines() + + def get_dict_lines(self, info) -> Generator[str, None, None]: + dict_text = utility.array_to_string(info.dev_info.subsystem) + if dict_text: + yield f" SUBSYSTEM={dict_text}" + + dict_text = utility.array_to_string(info.dev_info.device) + if dict_text: + yield f" DEVICE={dict_text}" + + def run(self) -> Iterator[Tuple[str, str, str, str, str]]: + # static struct printk_ringbuffer *prb = &printk_rb_static; + ringbuffers = self.vmlinux.object_from_symbol(symbol_name = 'prb').dereference() + + desc_ring = ringbuffers.desc_ring + text_data_ring = ringbuffers.text_data_ring + + desc_count = 1 << desc_ring.count_bits + desc_arr = self.vmlinux.object(object_type = "array", + offset = desc_ring.descs, + subtype = self.vmlinux.get_type("prb_desc"), + count = desc_count) + info_arr = self.vmlinux.object(object_type = "array", + offset = desc_ring.infos, + subtype = self.vmlinux.get_type("printk_info"), + count = desc_count) + + # See kernel/printk/printk_ringbuffer.h + desc_state_var_bytes_sz = self.long_unsigned_int_size + desc_state_var_bits_sz = desc_state_var_bytes_sz * 8 + desc_flags_shift = desc_state_var_bits_sz - 2 + desc_flags_mask = 3 << desc_flags_shift + desc_id_mask = ~desc_flags_mask + + cur_id = desc_ring.tail_id.counter + end_id = None + while cur_id != end_id: + end_id = desc_ring.head_id.counter + desc = desc_arr[cur_id % desc_count] # type: ignore + info = info_arr[cur_id % desc_count] # type: ignore + desc_state = DescStateEnum((desc.state_var.counter >> desc_flags_shift) & 3) + if desc_state in (DescStateEnum.desc_committed, DescStateEnum.desc_finalized): + facility, level, timestamp, caller = self.get_prefix(info) + level_txt = self.get_level_text(level) + facility_txt = self.get_facility_text(facility) + + for line in self.get_log_lines(text_data_ring, desc, info): + yield facility_txt, level_txt, timestamp, caller, line + for line in self.get_dict_lines(info): + yield facility_txt, level_txt, timestamp, caller, line + + cur_id += 1 + cur_id &= desc_id_mask + + +class Kmsg(plugins.PluginInterface): + """Kernel log buffer reader""" + + _required_framework_version = (2, 0, 0) + + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ['Intel32', 'Intel64']), + ] + + def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str, str]]]: + for values in ABCKmsg.run_all(context = self.context, config = self.config): + yield (0, values) + + def run(self): + return renderers.TreeGrid([("facility", str), + ("level", str), + ("timestamp", str), + ("caller", str), + ("line", str)], + self._generator()) # type: ignore diff --git a/volatility3/framework/plugins/linux/lsmod.py b/volatility3/framework/plugins/linux/lsmod.py index 2d390cc824..ecb262d008 100644 --- a/volatility3/framework/plugins/linux/lsmod.py +++ b/volatility3/framework/plugins/linux/lsmod.py @@ -7,7 +7,6 @@ import logging from typing import List, Iterable -from volatility3.framework import contexts from volatility3.framework import exceptions, renderers, constants, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins @@ -20,21 +19,19 @@ class Lsmod(plugins.PluginInterface): """Lists loaded kernel modules.""" - _required_framework_version = (1, 0, 0) - _version = (1, 0, 0) + _required_framework_version = (2, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols") + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), ] @classmethod - def list_modules(cls, context: interfaces.context.ContextInterface, layer_name: str, - vmlinux_symbols: str) -> Iterable[interfaces.objects.ObjectInterface]: + def list_modules(cls, context: interfaces.context.ContextInterface, vmlinux_module_name: str) -> Iterable[ + interfaces.objects.ObjectInterface]: """Lists all the modules in the primary layer. Args: @@ -47,7 +44,7 @@ def list_modules(cls, context: interfaces.context.ContextInterface, layer_name: This function will throw a SymbolError exception if kernel module support is not enabled. """ - vmlinux = contexts.Module(context, vmlinux_symbols, layer_name, 0) + vmlinux = context.modules[vmlinux_module_name] modules = vmlinux.object_from_symbol(symbol_name = "modules").cast("list_head") @@ -58,7 +55,7 @@ def list_modules(cls, context: interfaces.context.ContextInterface, layer_name: def _generator(self): try: - for module in self.list_modules(self.context, self.config['primary'], self.config['vmlinux']): + for module in self.list_modules(self.context, self.config['kernel']): mod_size = module.get_init_size() + module.get_core_size() diff --git a/volatility3/framework/plugins/linux/lsof.py b/volatility3/framework/plugins/linux/lsof.py index 153d28f6a1..a074f57446 100644 --- a/volatility3/framework/plugins/linux/lsof.py +++ b/volatility3/framework/plugins/linux/lsof.py @@ -19,17 +19,15 @@ class Lsof(plugins.PluginInterface): """Lists all memory maps for all processes.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), - requirements.VersionRequirement(name = 'linuxutils', component = linux.LinuxUtilities, version = (1, 0, 0)), + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.VersionRequirement(name = 'linuxutils', component = linux.LinuxUtilities, version = (2, 0, 0)), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -57,6 +55,5 @@ def run(self): return renderers.TreeGrid([("PID", int), ("Process", str), ("FD", int), ("Path", str)], self._generator( pslist.PsList.list_tasks(self.context, - self.config['primary'], - self.config['vmlinux'], + self.config['kernel'], filter_func = filter_func))) diff --git a/volatility3/framework/plugins/linux/malfind.py b/volatility3/framework/plugins/linux/malfind.py index 149222f37a..abc2cf7d25 100644 --- a/volatility3/framework/plugins/linux/malfind.py +++ b/volatility3/framework/plugins/linux/malfind.py @@ -15,16 +15,14 @@ class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -48,7 +46,8 @@ def _list_injections(self, task): def _generator(self, tasks): # determine if we're on a 32 or 64 bit kernel - if self.context.symbol_space.get_type(self.config["vmlinux"] + constants.BANG + "pointer").size == 4: + vmlinux = self.context.modules[self.config['kernel']] + if self.context.symbol_space.get_type(vmlinux.symbol_table_name + constants.BANG + "pointer").size == 4: is_32bit_arch = True else: is_32bit_arch = False @@ -75,6 +74,5 @@ def run(self): ("Disasm", interfaces.renderers.Disassembly)], self._generator( pslist.PsList.list_tasks(self.context, - self.config['primary'], - self.config['vmlinux'], + self.config['kernel'], filter_func = filter_func))) diff --git a/volatility3/framework/plugins/linux/proc.py b/volatility3/framework/plugins/linux/proc.py index 8bd0db538c..13fd87f531 100644 --- a/volatility3/framework/plugins/linux/proc.py +++ b/volatility3/framework/plugins/linux/proc.py @@ -15,17 +15,15 @@ class Maps(plugins.PluginInterface): """Lists all memory maps for all processes.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (1, 0, 0)), + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -68,6 +66,5 @@ def run(self): ("File Path", str)], self._generator( pslist.PsList.list_tasks(self.context, - self.config['primary'], - self.config['vmlinux'], + self.config['kernel'], filter_func = filter_func))) diff --git a/volatility3/framework/plugins/linux/pslist.py b/volatility3/framework/plugins/linux/pslist.py index 78fce978be..5672bb56e7 100644 --- a/volatility3/framework/plugins/linux/pslist.py +++ b/volatility3/framework/plugins/linux/pslist.py @@ -1,10 +1,9 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - from typing import Callable, Iterable, List, Any -from volatility3.framework import renderers, interfaces, contexts +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility @@ -12,17 +11,15 @@ class PsList(interfaces.plugins.PluginInterface): """Lists the processes present in a particular linux memory image.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -53,8 +50,7 @@ def filter_func(x): def _generator(self): for task in self.list_tasks(self.context, - self.config['primary'], - self.config['vmlinux'], + self.config['kernel'], filter_func = self.create_pid_filter(self.config.get('pid', None))): pid = task.pid ppid = 0 @@ -67,23 +63,22 @@ def _generator(self): def list_tasks( cls, context: interfaces.context.ContextInterface, - layer_name: str, - vmlinux_symbols: str, + vmlinux_module_name: str, filter_func: Callable[[int], bool] = lambda _: False) -> Iterable[interfaces.objects.ObjectInterface]: """Lists all the tasks in the primary layer. Args: context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - vmlinux_symbols: The name of the table containing the kernel symbols + vmlinux_module_name: The name of the kernel module on which to operate Yields: Process objects """ - vmlinux = contexts.Module(context, vmlinux_symbols, layer_name, 0) + vmlinux = context.modules[vmlinux_module_name] init_task = vmlinux.object_from_symbol(symbol_name = "init_task") + # Note that the init_task itself is not yielded, since "ps" also never shows it. for task in init_task.tasks: if not filter_func(task): yield task diff --git a/volatility3/framework/plugins/linux/pstree.py b/volatility3/framework/plugins/linux/pstree.py index a187ea9078..3b95a344c6 100644 --- a/volatility3/framework/plugins/linux/pstree.py +++ b/volatility3/framework/plugins/linux/pstree.py @@ -10,8 +10,6 @@ class PsTree(pslist.PsList): """Plugin for listing processes in a tree based on their parent process ID.""" - _required_framework_version = (1, 0, 0) - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._processes = {} @@ -36,7 +34,8 @@ def find_level(self, pid): def _generator(self): """Generates the.""" - for proc in self.list_tasks(self.context, self.config['primary'], self.config['vmlinux']): + vmlinux = self.context.modules[self.config['kernel']] + for proc in self.list_tasks(self.context, vmlinux.name): self._processes[proc.pid] = proc # Build the child/level maps diff --git a/volatility3/framework/plugins/linux/tty_check.py b/volatility3/framework/plugins/linux/tty_check.py index fe35d8fa32..e1d8339e86 100644 --- a/volatility3/framework/plugins/linux/tty_check.py +++ b/volatility3/framework/plugins/linux/tty_check.py @@ -5,7 +5,7 @@ import logging from typing import List -from volatility3.framework import interfaces, renderers, exceptions, constants, contexts +from volatility3.framework import interfaces, renderers, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -19,29 +19,26 @@ class tty_check(plugins.PluginInterface): """Checks tty devices for hooks""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "vmlinux", description = "Linux kernel symbols"), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (1, 0, 0)), - requirements.VersionRequirement(name = 'linuxutils', component = linux.LinuxUtilities, version = (1, 0, 0)) + requirements.ModuleRequirement(name = 'kernel', description = 'Linux kernel', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)), + requirements.VersionRequirement(name = 'linuxutils', component = linux.LinuxUtilities, version = (2, 0, 0)) ] def _generator(self): - vmlinux = contexts.Module(self.context, self.config['vmlinux'], self.config['primary'], 0) + vmlinux = self.context.modules[self.config['kernel']] - modules = lsmod.Lsmod.list_modules(self.context, self.config['primary'], self.config['vmlinux']) + modules = lsmod.Lsmod.list_modules(self.context, vmlinux.name) - handlers = linux.LinuxUtilities.generate_kernel_handler_info(self.context, self.config['primary'], - self.config['vmlinux'], modules) + handlers = linux.LinuxUtilities.generate_kernel_handler_info(self.context, vmlinux.name, modules) try: - tty_drivers = vmlinux.object_from_symbol("tty_drivers") + tty_drivers = vmlinux.object_from_symbol("tty_drivers").cast("list_head") except exceptions.SymbolError: tty_drivers = None @@ -52,12 +49,12 @@ def _generator(self): "This means you are either analyzing an unsupported kernel version or that your symbol table is corrupt." ) - for tty in tty_drivers.to_list(vmlinux.name + constants.BANG + "tty_driver", "tty_drivers"): + for tty in tty_drivers.to_list(vmlinux.symbol_table_name + constants.BANG + "tty_driver", "tty_drivers"): try: ttys = utility.array_of_pointers(tty.ttys.dereference(), count = tty.num, - subtype = vmlinux.name + constants.BANG + "tty_struct", + subtype = vmlinux.symbol_table_name + constants.BANG + "tty_struct", context = self.context) except exceptions.PagedInvalidAddressException: continue @@ -71,7 +68,7 @@ def _generator(self): recv_buf = tty_dev.ldisc.ops.receive_buf - module_name, symbol_name = linux.LinuxUtilities.lookup_module_address(self.context, handlers, recv_buf) + module_name, symbol_name = linux.LinuxUtilities.lookup_module_address(vmlinux, handlers, recv_buf) yield (0, (name, format_hints.Hex(recv_buf), module_name, symbol_name)) diff --git a/volatility3/framework/plugins/mac/bash.py b/volatility3/framework/plugins/mac/bash.py index 6b453759e6..e16a0d79f8 100644 --- a/volatility3/framework/plugins/mac/bash.py +++ b/volatility3/framework/plugins/mac/bash.py @@ -20,16 +20,14 @@ class Bash(plugins.PluginInterface, timeliner.TimeLinerInterface): """Recovers bash command history from memory.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -37,7 +35,8 @@ def get_requirements(cls): ] def _generator(self, tasks): - is_32bit = not symbols.symbol_table_is_64bit(self.context, self.config["darwin"]) + darwin = self.context.modules[self.config['kernel']] + is_32bit = not symbols.symbol_table_is_64bit(self.context, darwin.symbol_table_name) if is_32bit: pack_format = "I" bash_json_file = "bash32" @@ -67,7 +66,7 @@ def _generator(self, tasks): for address in proc_layer.scan(self.context, scanners.BytesScanner(b"#"), sections = task.get_process_memory_sections(self.context, - self.config['darwin'], + self.config['kernel'], rw_no_file = True)): bang_addrs.append(struct.pack(pack_format, address)) @@ -76,7 +75,7 @@ def _generator(self, tasks): for address, _ in proc_layer.scan(self.context, scanners.MultiStringScanner(bang_addrs), sections = task.get_process_memory_sections(self.context, - self.config['darwin'], + self.config['kernel'], rw_no_file = True)): hist = self.context.object(bash_table_name + constants.BANG + "hist_entry", offset = address - ts_offset, @@ -96,8 +95,7 @@ def run(self): ("Command", str)], self._generator( list_tasks(self.context, - self.config['primary'], - self.config['darwin'], + self.config['kernel'], filter_func = filter_func))) def generate_timeline(self): @@ -105,7 +103,9 @@ def generate_timeline(self): list_tasks = pslist.PsList.get_list_tasks(self.config.get('pslist_method', pslist.PsList.pslist_methods[0])) for row in self._generator( - list_tasks(self.context, self.config['primary'], self.config['darwin'], filter_func = filter_func)): + list_tasks(self.context, + self.config['kernel'], + filter_func = filter_func)): _depth, row_data = row - description = "{} ({}): \"{}\"".format(row_data[0], row_data[1], row_data[3]) + description = f"{row_data[0]} ({row_data[1]}): \"{row_data[3]}\"" yield (description, timeliner.TimeLinerType.CREATED, row_data[2]) diff --git a/volatility3/framework/plugins/mac/check_syscall.py b/volatility3/framework/plugins/mac/check_syscall.py index 4f367001eb..9072e76d1a 100644 --- a/volatility3/framework/plugins/mac/check_syscall.py +++ b/volatility3/framework/plugins/mac/check_syscall.py @@ -5,7 +5,7 @@ from typing import List from volatility3.framework import exceptions, interfaces -from volatility3.framework import renderers, contexts +from volatility3.framework import renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.renderers import format_hints @@ -18,34 +18,26 @@ class Check_syscall(plugins.PluginInterface): """Check system call table for hooks.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel symbols"), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (1, 0, 0)) + requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) ] def _generator(self): - kernel = contexts.Module(self._context, self.config['darwin'], self.config['primary'], 0) + kernel = self.context.modules[self.config['kernel']] - mods = lsmod.Lsmod.list_modules(self.context, self.config['primary'], self.config['darwin']) + mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, self.config['primary'], kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) - nsysent = kernel.object_from_symbol(symbol_name = "nsysent") table = kernel.object_from_symbol(symbol_name = "sysent") - # smear help - num_ents = min(nsysent, table.count) - if num_ents > 1024: - num_ents = 1024 - for (i, ent) in enumerate(table): try: call_addr = ent.sy_call.dereference().vol.offset @@ -55,7 +47,8 @@ def _generator(self): if not call_addr or call_addr == 0: continue - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, call_addr) + module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, + call_addr, self.config['kernel']) yield (0, (format_hints.Hex(table.vol.offset), "SysCall", i, format_hints.Hex(call_addr), module_name, symbol_name)) diff --git a/volatility3/framework/plugins/mac/check_sysctl.py b/volatility3/framework/plugins/mac/check_sysctl.py index 4ea1d8af03..fc6ab5c373 100644 --- a/volatility3/framework/plugins/mac/check_sysctl.py +++ b/volatility3/framework/plugins/mac/check_sysctl.py @@ -6,7 +6,7 @@ import volatility3 from volatility3.framework import exceptions, interfaces -from volatility3.framework import renderers, contexts +from volatility3.framework import renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -20,17 +20,15 @@ class Check_sysctl(plugins.PluginInterface): """Check sysctl handlers for hooks.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel symbols"), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (1, 0, 0)) + requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) ] def _parse_global_variable_sysctls(self, kernel, name): @@ -115,11 +113,11 @@ def _process_sysctl_list(self, kernel, sysctl_list, recursive = 0): break def _generator(self): - kernel = contexts.Module(self._context, self.config['darwin'], self.config['primary'], 0) + kernel = self.context.modules[self.config['kernel']] - mods = lsmod.Lsmod.list_modules(self.context, self.config['primary'], self.config['darwin']) + mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, self.config['primary'], kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) sysctl_list = kernel.object_from_symbol(symbol_name = "sysctl__children") @@ -129,7 +127,8 @@ def _generator(self): except exceptions.InvalidAddressException: continue - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, check_addr) + module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, check_addr, + self.config['kernel']) yield (0, (name, sysctl.oid_number, sysctl.get_perms(), format_hints.Hex(check_addr), val, module_name, symbol_name)) diff --git a/volatility3/framework/plugins/mac/check_trap_table.py b/volatility3/framework/plugins/mac/check_trap_table.py index 0584d9ff79..47d0ed57db 100644 --- a/volatility3/framework/plugins/mac/check_trap_table.py +++ b/volatility3/framework/plugins/mac/check_trap_table.py @@ -6,7 +6,7 @@ from typing import List from volatility3.framework import exceptions, interfaces -from volatility3.framework import renderers, contexts +from volatility3.framework import renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.renderers import format_hints @@ -19,25 +19,23 @@ class Check_trap_table(plugins.PluginInterface): """Check mach trap table for hooks.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel symbols"), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (1, 0, 0)), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)), requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), ] def _generator(self): - kernel = contexts.Module(self._context, self.config['darwin'], self.config['primary'], 0) + kernel = self.context.modules[self.config['kernel']] - mods = lsmod.Lsmod.list_modules(self.context, self.config['primary'], self.config['darwin']) + mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, self.config['primary'], kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) table = kernel.object_from_symbol(symbol_name = "mach_trap_table") @@ -50,7 +48,8 @@ def _generator(self): if not call_addr or call_addr == 0: continue - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, call_addr) + module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, call_addr, + self.config['kernel']) yield (0, (format_hints.Hex(table.vol.offset), "TrapTable", i, format_hints.Hex(call_addr), module_name, symbol_name)) diff --git a/volatility3/framework/plugins/mac/ifconfig.py b/volatility3/framework/plugins/mac/ifconfig.py index 5c4e5728e4..c366a19f06 100644 --- a/volatility3/framework/plugins/mac/ifconfig.py +++ b/volatility3/framework/plugins/mac/ifconfig.py @@ -1,7 +1,7 @@ # This file is Copyright 2019 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework import exceptions, renderers, contexts +from volatility3.framework import exceptions, renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -11,20 +11,18 @@ class Ifconfig(plugins.PluginInterface): """Lists loaded kernel modules""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel symbols"), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)) ] def _generator(self): - kernel = contexts.Module(self._context, self.config['darwin'], self.config['primary'], 0) + kernel = self.context.modules[self.config['kernel']] try: list_head = kernel.object_from_symbol(symbol_name = "ifnet_head") @@ -45,7 +43,7 @@ def _generator(self): for ifaddr in mac.MacUtilities.walk_tailq(ifnet.if_addrhead, "ifa_link"): ip = ifaddr.ifa_addr.get_address() - yield (0, ("{0}{1}".format(name, unit), ip, mac_addr, prom)) + yield (0, (f"{name}{unit}", ip, mac_addr, prom)) def run(self): return renderers.TreeGrid([("Interface", str), ("IP Address", str), ("Mac Address", str), diff --git a/volatility3/framework/plugins/mac/kauth_listeners.py b/volatility3/framework/plugins/mac/kauth_listeners.py index 930239db9d..7002d88e29 100644 --- a/volatility3/framework/plugins/mac/kauth_listeners.py +++ b/volatility3/framework/plugins/mac/kauth_listeners.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework import renderers, interfaces, contexts +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints @@ -13,34 +13,31 @@ class Kauth_listeners(interfaces.plugins.PluginInterface): """ Lists kauth listeners and their status """ - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel"), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 1, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (1, 0, 0)), + requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)), requirements.PluginRequirement(name = 'kauth_scopes', plugin = kauth_scopes.Kauth_scopes, - version = (1, 0, 0)) + version = (2, 0, 0)) ] def _generator(self): """ Enumerates the listeners for each kauth scope """ - kernel = contexts.Module(self.context, self.config['darwin'], self.config['primary'], 0) + kernel = self.context.modules[self.config['kernel']] - mods = lsmod.Lsmod.list_modules(self.context, self.config['primary'], self.config['darwin']) + mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, self.config['primary'], kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) - for scope in kauth_scopes.Kauth_scopes.list_kauth_scopes(self.context, self.config['primary'], - self.config['darwin']): + for scope in kauth_scopes.Kauth_scopes.list_kauth_scopes(self.context, self.config['kernel']): scope_name = utility.pointer_to_string(scope.ks_identifier, 128) @@ -49,7 +46,8 @@ def _generator(self): if callback == 0: continue - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, callback) + module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, callback, + self.config['kernel']) yield (0, (scope_name, format_hints.Hex(listener.kll_idata), format_hints.Hex(callback), module_name, symbol_name)) diff --git a/volatility3/framework/plugins/mac/kauth_scopes.py b/volatility3/framework/plugins/mac/kauth_scopes.py index a5f57d9a13..910de35fd4 100644 --- a/volatility3/framework/plugins/mac/kauth_scopes.py +++ b/volatility3/framework/plugins/mac/kauth_scopes.py @@ -1,69 +1,67 @@ # This file is opyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # +import logging +from typing import Iterable, Callable -from typing import Iterable, Callable, Tuple - -from volatility3.framework import renderers, interfaces, contexts +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import mac from volatility3.plugins.mac import lsmod +vollog = logging.getLogger(__name__) + class Kauth_scopes(interfaces.plugins.PluginInterface): """ Lists kauth scopes and their status """ - _version = (1, 0, 0) - _required_framework_version = (1, 0, 0) + _version = (2, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel"), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 1, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (1, 0, 0)) + requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) ] @classmethod def list_kauth_scopes(cls, context: interfaces.context.ContextInterface, - layer_name: str, - darwin_symbols: str, + kernel_module_name: str, filter_func: Callable[[int], bool] = lambda _: False) -> \ - Iterable[Tuple[interfaces.objects.ObjectInterface, - interfaces.objects.ObjectInterface, - interfaces.objects.ObjectInterface]]: + Iterable[interfaces.objects.ObjectInterface]: """ Enumerates the registered kauth scopes and yields each object Uses smear-safe enumeration API """ - - kernel = contexts.Module(context, darwin_symbols, layer_name, 0) + kernel = context.modules[kernel_module_name] scopes = kernel.object_from_symbol("kauth_scopes") for scope in mac.MacUtilities.walk_tailq(scopes, "ks_link"): - yield scope + if not filter_func(scope): + yield scope def _generator(self): - kernel = contexts.Module(self.context, self.config['darwin'], self.config['primary'], 0) + kernel = self.context.modules[self.config['kernel']] - mods = lsmod.Lsmod.list_modules(self.context, self.config['primary'], self.config['darwin']) + mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, self.config['primary'], kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) - for scope in self.list_kauth_scopes(self.context, self.config['primary'], self.config['darwin']): + for scope in self.list_kauth_scopes(self.context, self.config['kernel']): callback = scope.ks_callback if callback == 0: continue - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, callback) + module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, callback, + self.config['kernel']) identifier = utility.pointer_to_string(scope.ks_identifier, 128) diff --git a/volatility3/framework/plugins/mac/kevents.py b/volatility3/framework/plugins/mac/kevents.py index a6d9051021..16fa51fa44 100644 --- a/volatility3/framework/plugins/mac/kevents.py +++ b/volatility3/framework/plugins/mac/kevents.py @@ -4,7 +4,7 @@ from typing import Iterable, Callable, Tuple -from volatility3.framework import renderers, interfaces, exceptions, contexts +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.symbols import mac @@ -14,7 +14,8 @@ class Kevents(interfaces.plugins.PluginInterface): """ Lists event handlers registered by processes """ - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) event_types = { 1: "EVFILT_READ", @@ -47,11 +48,9 @@ class Kevents(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 2, 0)), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', @@ -99,7 +98,7 @@ def _get_task_kevents(cls, kernel, task): """ Enumerates event filters per task. Uses smear-safe APIs throughout as these data structures - see a signifcant amount of smear + see a significant amount of smear """ fdp = task.p_fd @@ -120,8 +119,7 @@ def _get_task_kevents(cls, kernel, task): @classmethod def list_kernel_events(cls, context: interfaces.context.ContextInterface, - layer_name: str, - darwin_symbols: str, + kernel_module_name: str, filter_func: Callable[[int], bool] = lambda _: False) -> \ Iterable[Tuple[interfaces.objects.ObjectInterface, interfaces.objects.ObjectInterface, @@ -135,11 +133,11 @@ def list_kernel_events(cls, 2) The process ID of the process that registered the filter 3) The object of the associated kernel event filter """ - kernel = contexts.Module(context, darwin_symbols, layer_name, 0) + kernel = context.modules[kernel_module_name] list_tasks = pslist.PsList.get_list_tasks(pslist.PsList.pslist_methods[0]) - for task in list_tasks(context, layer_name, darwin_symbols, filter_func): + for task in list_tasks(context, kernel_module_name, filter_func): task_name = utility.array_to_string(task.p_comm) pid = task.p_pid @@ -150,8 +148,7 @@ def _generator(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) for task_name, pid, kn in self.list_kernel_events(self.context, - self.config['primary'], - self.config['darwin'], + self.config['kernel'], filter_func = filter_func): filter_index = kn.kn_kevent.filter * -1 diff --git a/volatility3/framework/plugins/mac/list_files.py b/volatility3/framework/plugins/mac/list_files.py index 557e735eda..19f28b18f9 100644 --- a/volatility3/framework/plugins/mac/list_files.py +++ b/volatility3/framework/plugins/mac/list_files.py @@ -18,16 +18,14 @@ class List_Files(plugins.PluginInterface): """Lists all open file descriptors for all processes.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Kernel Address Space', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac Kernel"), - requirements.PluginRequirement(name = 'mount', plugin = mount.Mount, version = (1, 0, 0)), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'mount', plugin = mount.Mount, version = (2, 0, 0)), ] @classmethod @@ -44,20 +42,22 @@ def _vnode_name(cls, vnode: interfaces.objects.ObjectInterface) -> Optional[str] return v_name @classmethod - def _get_parent(cls, vnode): - parent = None - + def _get_parent(cls, context, vnode): # root entries do not have parents # and parents of normal files can be smeared try: - parent = vnode.v_parent + parent = vnode.v_parent.dereference() except exceptions.InvalidAddressException: - pass + return None + + if parent and not context.layers[vnode.vol.native_layer_name].is_valid(parent.vol.offset, + parent.vol.size): + return None return parent @classmethod - def _add_vnode(cls, vnode, loop_vnodes): + def _add_vnode(cls, context, vnode, loop_vnodes): """ Adds the given vnode to loop_vnodes. @@ -65,7 +65,11 @@ def _add_vnode(cls, vnode, loop_vnodes): and holds its name, parent address, and object """ - key = vnode + if not context.layers[vnode.vol.native_layer_name].is_valid(vnode.vol.offset, + vnode.vol.size): + return False + + key = vnode.vol.offset added = False if not key in loop_vnodes: @@ -74,9 +78,9 @@ def _add_vnode(cls, vnode, loop_vnodes): if v_name is None: return added - parent = cls._get_parent(vnode) + parent = cls._get_parent(context, vnode) if parent: - parent_val = parent + parent_val = parent.vol.offset else: parent_val = None @@ -87,61 +91,79 @@ def _add_vnode(cls, vnode, loop_vnodes): return added @classmethod - def _walk_vnode(cls, vnode, loop_vnodes): + def _walk_vnode(cls, context, vnode, loop_vnodes): """ Iterates over the list of vnodes associated with the given one. Also traverses the parent chain for the vnode and adds each one. """ + added = False + while vnode: - if not cls._add_vnode(vnode, loop_vnodes): + if vnode in loop_vnodes: + return added + + if not cls._add_vnode(context, vnode, loop_vnodes): break + + added = True - parent = cls._get_parent(vnode) - while parent: - cls._walk_vnode(parent, loop_vnodes) - parent = cls._get_parent(parent) + parent = cls._get_parent(context, vnode) + while parent and not parent in loop_vnodes: + if not cls._walk_vnode(context, parent, loop_vnodes): + break + + parent = cls._get_parent(context, parent) try: - vnode = vnode.v_mntvnodes.tqe_next + vnode = vnode.v_mntvnodes.tqe_next.dereference() except exceptions.InvalidAddressException: break + return added + @classmethod - def _walk_vnodelist(cls, list_head, loop_vnodes): + def _walk_vnodelist(cls, context, list_head, loop_vnodes): for vnode in mac.MacUtilities.walk_tailq(list_head, "v_mntvnodes"): - cls._walk_vnode(vnode, loop_vnodes) + cls._walk_vnode(context, vnode, loop_vnodes) @classmethod def _walk_mounts(cls, context: interfaces.context.ContextInterface, - layer_name: str, - darwin_symbols: str) -> \ + kernel_module_name: str) -> \ Iterable[interfaces.objects.ObjectInterface]: loop_vnodes = {} # iterate each vnode source from each mount - list_mounts = mount.Mount.list_mounts(context, layer_name, darwin_symbols) + list_mounts = mount.Mount.list_mounts(context, kernel_module_name) for mnt in list_mounts: - cls._walk_vnodelist(mnt.mnt_vnodelist, loop_vnodes) - cls._walk_vnodelist(mnt.mnt_workerqueue, loop_vnodes) - cls._walk_vnodelist(mnt.mnt_newvnodes, loop_vnodes) - - cls._walk_vnode(mnt.mnt_vnodecovered, loop_vnodes) - cls._walk_vnode(mnt.mnt_realrootvp, loop_vnodes) - cls._walk_vnode(mnt.mnt_devvp, loop_vnodes) + cls._walk_vnodelist(context, mnt.mnt_vnodelist, loop_vnodes) + cls._walk_vnodelist(context, mnt.mnt_workerqueue, loop_vnodes) + cls._walk_vnodelist(context, mnt.mnt_newvnodes, loop_vnodes) + cls._walk_vnode(context, mnt.mnt_vnodecovered, loop_vnodes) + cls._walk_vnode(context, mnt.mnt_realrootvp, loop_vnodes) + cls._walk_vnode(context, mnt.mnt_devvp, loop_vnodes) return loop_vnodes @classmethod def _build_path(cls, vnodes, vnode_name, parent_offset): path = [vnode_name] + seen_offsets = set() while parent_offset in vnodes: parent_name, parent_offset, _ = vnodes[parent_offset] if parent_offset is None: parent_offset = 0 + # circular references from smear + elif parent_offset in seen_offsets: + path = [] + break + + else: + seen_offsets.add(parent_offset) + path.insert(0, parent_name) if len(path) > 1: @@ -157,11 +179,10 @@ def _build_path(cls, vnodes, vnode_name, parent_offset): @classmethod def list_files(cls, context: interfaces.context.ContextInterface, - layer_name: str, - darwin_symbols: str) -> \ + kernel_module_name: str) -> \ Iterable[interfaces.objects.ObjectInterface]: - vnodes = cls._walk_mounts(context, layer_name, darwin_symbols) + vnodes = cls._walk_mounts(context, kernel_module_name) for voff, (vnode_name, parent_offset, vnode) in vnodes.items(): full_path = cls._build_path(vnodes, vnode_name, parent_offset) @@ -169,9 +190,9 @@ def list_files(cls, yield vnode, full_path def _generator(self): - for vnode, full_path in self.list_files(self.context, self.config['primary'], self.config['darwin']): + for vnode, full_path in self.list_files(self.context, self.config['kernel']): - yield (0, (format_hints.Hex(vnode), full_path)) + yield (0, (format_hints.Hex(vnode.vol.offset), full_path)) def run(self): return renderers.TreeGrid([("Address", format_hints.Hex), ("File Path", str)], self._generator()) diff --git a/volatility3/framework/plugins/mac/lsmod.py b/volatility3/framework/plugins/mac/lsmod.py index b536a112be..095fbc6630 100644 --- a/volatility3/framework/plugins/mac/lsmod.py +++ b/volatility3/framework/plugins/mac/lsmod.py @@ -3,7 +3,9 @@ # """A module containing a collection of plugins that produce data typically found in Mac's lsmod command.""" -from volatility3.framework import renderers, interfaces, contexts +from typing import Set + +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -13,21 +15,19 @@ class Lsmod(plugins.PluginInterface): """Lists loaded kernel modules.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel") + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), ] @classmethod - def list_modules(cls, context: interfaces.context.ContextInterface, layer_name: str, darwin_symbols: str): + def list_modules(cls, context: interfaces.context.ContextInterface, darwin_module_name: str): """Lists all the modules in the primary layer. Args: @@ -38,18 +38,45 @@ def list_modules(cls, context: interfaces.context.ContextInterface, layer_name: Returns: A list of modules from the `layer_name` layer """ - kernel = contexts.Module(context, darwin_symbols, layer_name, 0) + kernel = context.modules[darwin_module_name] + kernel_layer = context.layers[kernel.layer_name] kmod_ptr = kernel.object_from_symbol(symbol_name = "kmod") - # TODO - use smear-proof list walking API after dev release - kmod = kmod_ptr.dereference().cast("kmod_info") - while kmod != 0: - yield kmod + try: + kmod = kmod_ptr.dereference().cast("kmod_info") + except exceptions.InvalidAddressException: + return [] + + yield kmod + + try: kmod = kmod.next + except exceptions.InvalidAddressException: + return [] + + seen: Set = set() + + while kmod != 0 and \ + kmod not in seen and \ + len(seen) < 1024: + + kmod_obj = kmod.dereference() + + if not kernel_layer.is_valid(kmod_obj.vol.offset, kmod_obj.vol.size): + break + + seen.add(kmod) + + yield kmod + + try: + kmod = kmod.next + except exceptions.InvalidAddressException: + return def _generator(self): - for module in self.list_modules(self.context, self.config['primary'], self.config['darwin']): + for module in self.list_modules(self.context, self.config['kernel']): mod_name = utility.array_to_string(module.name) mod_size = module.size diff --git a/volatility3/framework/plugins/mac/lsof.py b/volatility3/framework/plugins/mac/lsof.py index f4fb725a00..c3941ec277 100644 --- a/volatility3/framework/plugins/mac/lsof.py +++ b/volatility3/framework/plugins/mac/lsof.py @@ -16,17 +16,15 @@ class Lsof(plugins.PluginInterface): """Lists all open file descriptors for all processes.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Kernel Address Space', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac Kernel"), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -34,10 +32,12 @@ def get_requirements(cls): ] def _generator(self, tasks): + darwin = self.context.modules[self.config['kernel']] for task in tasks: pid = task.p_pid - for _, filepath, fd in mac.MacUtilities.files_descriptors_for_process(self.context, self.config['darwin'], + for _, filepath, fd in mac.MacUtilities.files_descriptors_for_process(self.context, + darwin.symbol_table_name, task): if filepath and len(filepath) > 0: yield (0, (pid, fd, filepath)) @@ -49,6 +49,5 @@ def run(self): return renderers.TreeGrid([("PID", int), ("File Descriptor", int), ("File Path", str)], self._generator( list_tasks(self.context, - self.config['primary'], - self.config['darwin'], + self.config['kernel'], filter_func = filter_func))) diff --git a/volatility3/framework/plugins/mac/malfind.py b/volatility3/framework/plugins/mac/malfind.py index 20b46528b6..7a42a0c5f1 100644 --- a/volatility3/framework/plugins/mac/malfind.py +++ b/volatility3/framework/plugins/mac/malfind.py @@ -2,7 +2,6 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework import constants from volatility3.framework import interfaces from volatility3.framework import renderers from volatility3.framework.configuration import requirements @@ -14,16 +13,14 @@ class Malfind(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -41,13 +38,13 @@ def _list_injections(self, task): proc_layer = self.context.layers[proc_layer_name] for vma in task.get_map_iter(): - if not vma.is_suspicious(self.context, self.config['darwin']): + if not vma.is_suspicious(self.context, self.context.modules[self.config['kernel']].symbol_table_name): data = proc_layer.read(vma.links.start, 64, pad = True) yield vma, data def _generator(self, tasks): # determine if we're on a 32 or 64 bit kernel - if self.context.symbol_space.get_type(self.config["darwin"] + constants.BANG + "pointer").size == 4: + if self.context.modules[self.config['kernel']].get_type("pointer").size == 4: is_32bit_arch = True else: is_32bit_arch = False @@ -75,6 +72,5 @@ def run(self): ("Disasm", interfaces.renderers.Disassembly)], self._generator( list_tasks(self.context, - self.config['primary'], - self.config['darwin'], + self.config['kernel'], filter_func = filter_func))) diff --git a/volatility3/framework/plugins/mac/mount.py b/volatility3/framework/plugins/mac/mount.py index 3500e5ff18..3985594463 100644 --- a/volatility3/framework/plugins/mac/mount.py +++ b/volatility3/framework/plugins/mac/mount.py @@ -3,7 +3,7 @@ # """A module containing a collection of plugins that produce data typically found in Mac's mount command.""" -from volatility3.framework import renderers, interfaces, contexts +from volatility3.framework import renderers, interfaces from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -14,22 +14,20 @@ class Mount(plugins.PluginInterface): """A module containing a collection of plugins that produce data typically foundin Mac's mount command""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) - _version = (1, 0, 0) + _version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel symbols") ] @classmethod - def list_mounts(cls, context: interfaces.context.ContextInterface, layer_name: str, darwin_symbols: str): + def list_mounts(cls, context: interfaces.context.ContextInterface, kernel_module_name: str): """Lists all the mount structures in the primary layer. Args: @@ -40,7 +38,7 @@ def list_mounts(cls, context: interfaces.context.ContextInterface, layer_name: s Returns: A list of mount structures from the `layer_name` layer """ - kernel = contexts.Module(context, darwin_symbols, layer_name, 0) + kernel = context.modules[kernel_module_name] list_head = kernel.object_from_symbol(symbol_name = "mountlist") @@ -48,7 +46,7 @@ def list_mounts(cls, context: interfaces.context.ContextInterface, layer_name: s yield mount def _generator(self): - for mount in self.list_mounts(self.context, self.config['primary'], self.config['darwin']): + for mount in self.list_mounts(self.context, self.config['kernel']): vfs = mount.mnt_vfsstat device_name = utility.array_to_string(vfs.f_mntonname) mount_point = utility.array_to_string(vfs.f_mntfromname) diff --git a/volatility3/framework/plugins/mac/netstat.py b/volatility3/framework/plugins/mac/netstat.py index 4b9f539fdd..e231b8082c 100644 --- a/volatility3/framework/plugins/mac/netstat.py +++ b/volatility3/framework/plugins/mac/netstat.py @@ -19,16 +19,14 @@ class Netstat(plugins.PluginInterface): """Lists all network connections for all processes.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Kernel Address Space', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac Kernel"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', @@ -39,8 +37,7 @@ def get_requirements(cls): @classmethod def list_sockets(cls, context: interfaces.context.ContextInterface, - layer_name: str, - darwin_symbols: str, + kernel_module_name: str, filter_func: Callable[[int], bool] = lambda _: False) -> \ Iterable[Tuple[interfaces.objects.ObjectInterface, interfaces.objects.ObjectInterface, @@ -56,12 +53,13 @@ def list_sockets(cls, """ # This is hardcoded, since a change in the default method would change the expected results list_tasks = pslist.PsList.get_list_tasks(pslist.PsList.pslist_methods[0]) - for task in list_tasks(context, layer_name, darwin_symbols, filter_func): + for task in list_tasks(context, kernel_module_name, filter_func): task_name = utility.array_to_string(task.p_comm) pid = task.p_pid - for filp, _, _ in mac.MacUtilities.files_descriptors_for_process(context, darwin_symbols, task): + for filp, _, _ in mac.MacUtilities.files_descriptors_for_process(context, context.modules[ + kernel_module_name].symbol_table_name, task): try: ftype = filp.f_fglob.get_fg_type() except exceptions.InvalidAddressException: @@ -75,14 +73,17 @@ def list_sockets(cls, except exceptions.InvalidAddressException: continue + if not context.layers[task.vol.native_layer_name].is_valid(socket.vol.offset, + socket.vol.size): + continue + yield task_name, pid, socket def _generator(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) for task_name, pid, socket in self.list_sockets(self.context, - self.config['primary'], - self.config['darwin'], + self.config['kernel'], filter_func = filter_func): family = socket.get_family() @@ -95,7 +96,7 @@ def _generator(self): continue yield (0, (format_hints.Hex(socket.vol.offset), "UNIX", path, 0, "", 0, "", - "{}/{:d}".format(task_name, pid))) + f"{task_name}/{pid:d}")) elif family in [2, 30]: state = socket.get_state() @@ -107,7 +108,7 @@ def _generator(self): (lip, lport, rip, rport) = vals yield (0, (format_hints.Hex(socket.vol.offset), proto, lip, lport, rip, rport, state, - "{}/{:d}".format(task_name, pid))) + f"{task_name}/{pid:d}")) def run(self): return renderers.TreeGrid([("Offset", format_hints.Hex), ("Proto", str), ("Local IP", str), ("Local Port", int), diff --git a/volatility3/framework/plugins/mac/proc_maps.py b/volatility3/framework/plugins/mac/proc_maps.py index 2ff80c47e4..70c9684cd5 100644 --- a/volatility3/framework/plugins/mac/proc_maps.py +++ b/volatility3/framework/plugins/mac/proc_maps.py @@ -12,16 +12,14 @@ class Maps(interfaces.plugins.PluginInterface): """Lists process memory ranges that potentially contain injected code.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -34,7 +32,7 @@ def _generator(self, tasks): process_pid = task.p_pid for vma in task.get_map_iter(): - path = vma.get_path(self.context, self.config['darwin']) + path = vma.get_path(self.context, self.context.modules[self.config['kernel']].symbol_table_name) if path == "": path = vma.get_special_path() @@ -49,6 +47,5 @@ def run(self): ("End", format_hints.Hex), ("Protection", str), ("Map Name", str)], self._generator( list_tasks(self.context, - self.config['primary'], - self.config['darwin'], + self.config['kernel'], filter_func = filter_func))) diff --git a/volatility3/framework/plugins/mac/psaux.py b/volatility3/framework/plugins/mac/psaux.py index 97f61c02ed..e3fcdc0bbd 100644 --- a/volatility3/framework/plugins/mac/psaux.py +++ b/volatility3/framework/plugins/mac/psaux.py @@ -14,16 +14,14 @@ class Psaux(plugins.PluginInterface): """Recovers program command line arguments.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -52,7 +50,7 @@ def _generator(self, tasks: Iterator[Any]) -> Generator[Tuple[int, Tuple[int, st task_name = utility.array_to_string(task.p_comm) - args = [] # type: List[bytes] + args: List[bytes] = [] while argc > 0: try: @@ -98,6 +96,5 @@ def run(self) -> renderers.TreeGrid: return renderers.TreeGrid([("PID", int), ("Process", str), ("Argc", int), ("Arguments", str)], self._generator( list_tasks(self.context, - self.config['primary'], - self.config['darwin'], + self.config['kernel'], filter_func = filter_func))) diff --git a/volatility3/framework/plugins/mac/pslist.py b/volatility3/framework/plugins/mac/pslist.py index 8ef7876f14..e92609b3aa 100644 --- a/volatility3/framework/plugins/mac/pslist.py +++ b/volatility3/framework/plugins/mac/pslist.py @@ -5,7 +5,7 @@ import logging from typing import Callable, Iterable, List, Dict -from volatility3.framework import renderers, interfaces, contexts, exceptions +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.symbols import mac @@ -16,17 +16,15 @@ class PsList(interfaces.plugins.PluginInterface): """Lists the processes present in a particular mac memory image.""" - _required_framework_version = (1, 0, 0) - _version = (2, 0, 0) + _required_framework_version = (2, 0, 0) + _version = (3, 0, 0) pslist_methods = ['tasks', 'allproc', 'process_group', 'sessions', 'pid_hash_table'] @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel symbols"), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 1, 0)), requirements.ChoiceRequirement(name = 'pslist_method', description = 'Method to determine for processes', @@ -41,8 +39,8 @@ def get_requirements(cls): @classmethod def get_list_tasks( - cls, method: str - ) -> Callable[[interfaces.context.ContextInterface, str, str, Callable[[int], bool]], + cls, method: str + ) -> Callable[[interfaces.context.ContextInterface, str, Callable[[int], bool]], Iterable[interfaces.objects.ObjectInterface]]: """Returns the list_tasks method based on the selector @@ -68,7 +66,7 @@ def get_list_tasks( list_tasks = cls.list_tasks_pid_hash_table else: raise ValueError("Impossible method choice chosen") - vollog.debug("Using method {}".format(method)) + vollog.debug(f"Using method {method}") return list_tasks @@ -91,8 +89,7 @@ def _generator(self): list_tasks = self.get_list_tasks(self.config.get('pslist_method', self.pslist_methods[0])) for task in list_tasks(self.context, - self.config['primary'], - self.config['darwin'], + self.config['kernel'], filter_func = self.create_pid_filter(self.config.get('pid', None))): pid = task.p_pid ppid = task.p_ppid @@ -102,29 +99,27 @@ def _generator(self): @classmethod def list_tasks_allproc(cls, context: interfaces.context.ContextInterface, - layer_name: str, - darwin_symbols: str, + kernel_module_name: str, filter_func: Callable[[int], bool] = lambda _: False) -> \ Iterable[interfaces.objects.ObjectInterface]: """Lists all the processes in the primary layer based on the allproc method Args: context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - darwin_symbols: The name of the table containing the kernel symbols + kernel_module_name: The name of the the kernel module on which to operate filter_func: A function which takes a process object and returns True if the process should be ignored/filtered Returns: The list of process objects from the processes linked list after filtering """ - kernel = contexts.Module(context, darwin_symbols, layer_name, 0) + kernel = context.modules[kernel_module_name] - kernel_layer = context.layers[layer_name] + kernel_layer = context.layers[kernel.layer_name] proc = kernel.object_from_symbol(symbol_name = "allproc").lh_first - seen = {} # type: Dict[int, int] + seen: Dict[int, int] = {} while proc is not None and proc.vol.offset != 0: if proc.vol.offset in seen: vollog.log(logging.INFO, "Recursive process list detected (a result of non-atomic acquisition).") @@ -143,29 +138,26 @@ def list_tasks_allproc(cls, @classmethod def list_tasks_tasks(cls, context: interfaces.context.ContextInterface, - layer_name: str, - darwin_symbols: str, + kernel_module_name: str, filter_func: Callable[[int], bool] = lambda _: False) -> \ Iterable[interfaces.objects.ObjectInterface]: """Lists all the tasks in the primary layer based on the tasks queue Args: context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - darwin_symbols: The name of the table containing the kernel symbols + kernel_module_name: The name of the the kernel module on which to operate filter_func: A function which takes a task object and returns True if the task should be ignored/filtered Returns: The list of task objects from the `layer_name` layer's `tasks` list after filtering """ + kernel = context.modules[kernel_module_name] - kernel = contexts.Module(context, darwin_symbols, layer_name, 0) - - kernel_layer = context.layers[layer_name] + kernel_layer = context.layers[kernel.layer_name] queue_entry = kernel.object_from_symbol(symbol_name = "tasks") - seen = {} # type: Dict[int, int] + seen: Dict[int, int] = {} for task in queue_entry.walk_list(queue_entry, "tasks", "task"): if task.vol.offset in seen: vollog.log(logging.INFO, "Recursive process list detected (a result of non-atomic acquisition).") @@ -184,22 +176,20 @@ def list_tasks_tasks(cls, @classmethod def list_tasks_sessions(cls, context: interfaces.context.ContextInterface, - layer_name: str, - darwin_symbols: str, + kernel_module_name: str, filter_func: Callable[[int], bool] = lambda _: False) -> \ Iterable[interfaces.objects.ObjectInterface]: """Lists all the tasks in the primary layer using sessions Args: context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - darwin_symbols: The name of the table containing the kernel symbols + kernel_module_name: The name of the the kernel module on which to operate filter_func: A function which takes a task object and returns True if the task should be ignored/filtered Returns: The list of task objects from the `layer_name` layer's `tasks` list after filtering """ - kernel = contexts.Module(context, darwin_symbols, layer_name, 0) + kernel = context.modules[kernel_module_name] table_size = kernel.object_from_symbol(symbol_name = "sesshash") @@ -218,23 +208,20 @@ def list_tasks_sessions(cls, @classmethod def list_tasks_process_group(cls, context: interfaces.context.ContextInterface, - layer_name: str, - darwin_symbols: str, + kernel_module_name: str, filter_func: Callable[[int], bool] = lambda _: False) -> \ Iterable[interfaces.objects.ObjectInterface]: """Lists all the tasks in the primary layer using process groups Args: context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - darwin_symbols: The name of the table containing the kernel symbols + kernel_module_name: The name of the the kernel module on which to operate filter_func: A function which takes a task object and returns True if the task should be ignored/filtered Returns: The list of task objects from the `layer_name` layer's `tasks` list after filtering """ - - kernel = contexts.Module(context, darwin_symbols, layer_name, 0) + kernel = context.modules[kernel_module_name] table_size = kernel.object_from_symbol(symbol_name = "pgrphash") @@ -254,23 +241,21 @@ def list_tasks_process_group(cls, @classmethod def list_tasks_pid_hash_table(cls, context: interfaces.context.ContextInterface, - layer_name: str, - darwin_symbols: str, + kernel_module_name: str, filter_func: Callable[[int], bool] = lambda _: False) -> \ Iterable[interfaces.objects.ObjectInterface]: """Lists all the tasks in the primary layer using the pid hash table Args: context: The context to retrieve required elements (layers, symbol tables) from - layer_name: The name of the layer on which to operate - darwin_symbols: The name of the table containing the kernel symbols + kernel_module_name: The name of the the kernel module on which to operate filter_func: A function which takes a task object and returns True if the task should be ignored/filtered Returns: The list of task objects from the `layer_name` layer's `tasks` list after filtering """ - kernel = contexts.Module(context, darwin_symbols, layer_name, 0) + kernel = context.modules[kernel_module_name] table_size = kernel.object_from_symbol(symbol_name = "pidhash") diff --git a/volatility3/framework/plugins/mac/pstree.py b/volatility3/framework/plugins/mac/pstree.py index ea70a5d4c6..d7fb0eab4c 100644 --- a/volatility3/framework/plugins/mac/pstree.py +++ b/volatility3/framework/plugins/mac/pstree.py @@ -13,7 +13,7 @@ class PsTree(plugins.PluginInterface): """Plugin for listing processes in a tree based on their parent process ID.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -24,11 +24,9 @@ def __init__(self, *args, **kwargs): @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel symbols"), - requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)) + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), + requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (3, 0, 0)) ] def _find_level(self, pid): @@ -50,7 +48,7 @@ def _generator(self): """Generates the tree list of processes""" list_tasks = pslist.PsList.get_list_tasks(self.config.get('pslist_method', pslist.PsList.pslist_methods[0])) - for proc in list_tasks(self.context, self.config['primary'], self.config['darwin']): + for proc in list_tasks(self.context, self.config['kernel']): self._processes[proc.p_pid] = proc # Build the child/level maps diff --git a/volatility3/framework/plugins/mac/socket_filters.py b/volatility3/framework/plugins/mac/socket_filters.py index 40fc6ffe0d..a6e9d11fd8 100644 --- a/volatility3/framework/plugins/mac/socket_filters.py +++ b/volatility3/framework/plugins/mac/socket_filters.py @@ -5,7 +5,7 @@ from typing import List from volatility3.framework import exceptions, interfaces -from volatility3.framework import renderers, contexts +from volatility3.framework import renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -19,25 +19,23 @@ class Socket_filters(plugins.PluginInterface): """Enumerates kernel socket filters.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel symbols"), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (1, 0, 0)) + requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) ] def _generator(self): - kernel = contexts.Module(self._context, self.config['darwin'], self.config['primary'], 0) + kernel = self.context.modules[self.config['kernel']] - mods = lsmod.Lsmod.list_modules(self.context, self.config['primary'], self.config['darwin']) + mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, self.config['primary'], kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) members_to_check = [ "sf_unregistered", "sf_attach", "sf_detach", "sf_notify", "sf_getpeername", "sf_getsockname", "sf_data_in", diff --git a/volatility3/framework/plugins/mac/timers.py b/volatility3/framework/plugins/mac/timers.py index 912388ffb1..7bc5fd5d0b 100644 --- a/volatility3/framework/plugins/mac/timers.py +++ b/volatility3/framework/plugins/mac/timers.py @@ -5,7 +5,7 @@ from typing import List from volatility3.framework import exceptions, interfaces -from volatility3.framework import renderers, contexts +from volatility3.framework import renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.renderers import format_hints @@ -18,36 +18,36 @@ class Timers(plugins.PluginInterface): """Check for malicious kernel timers.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel symbols"), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (1, 0, 0)) + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), + requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 3, 0)), + requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) ] def _generator(self): - kernel = contexts.Module(self.context, self.config['darwin'], self.config['primary'], 0) + kernel = self.context.modules[self.config['kernel']] - mods = lsmod.Lsmod.list_modules(self.context, self.config['primary'], self.config['darwin']) + mods = lsmod.Lsmod.list_modules(self.context, self.config['kernel']) - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, self.config['primary'], kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) real_ncpus = kernel.object_from_symbol(symbol_name = "real_ncpus") cpu_data_ptrs_ptr = kernel.get_symbol("cpu_data_ptr").address + # Returns the a pointer to the absolute address cpu_data_ptrs_addr = kernel.object(object_type = "pointer", offset = cpu_data_ptrs_ptr, subtype = kernel.get_type('long unsigned int')) cpu_data_ptrs = kernel.object(object_type = "array", offset = cpu_data_ptrs_addr, + absolute = True, subtype = kernel.get_type('cpu_data'), count = real_ncpus) @@ -68,9 +68,10 @@ def _generator(self): else: entry_time = -1 - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, handler) + module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, handler, + self.config['kernel']) - yield (0, (format_hints.Hex(handler), format_hints.Hex(timer.param0), format_hints.Hex(timer.param1), \ + yield (0, (format_hints.Hex(handler), format_hints.Hex(timer.param0), format_hints.Hex(timer.param1), timer.deadline, entry_time, module_name, symbol_name)) def run(self): diff --git a/volatility3/framework/plugins/mac/trustedbsd.py b/volatility3/framework/plugins/mac/trustedbsd.py index b03663625b..9efb1b9d60 100644 --- a/volatility3/framework/plugins/mac/trustedbsd.py +++ b/volatility3/framework/plugins/mac/trustedbsd.py @@ -6,7 +6,7 @@ from typing import List, Iterator, Any from volatility3.framework import exceptions, interfaces -from volatility3.framework import renderers, contexts +from volatility3.framework import renderers from volatility3.framework.configuration import requirements from volatility3.framework.interfaces import plugins from volatility3.framework.objects import utility @@ -20,29 +20,28 @@ class Trustedbsd(plugins.PluginInterface): """Checks for malicious trustedbsd modules""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel symbols"), - requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (1, 0, 0)) + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), + requirements.VersionRequirement(name = 'macutils', component = mac.MacUtilities, version = (1, 3, 0)), + requirements.PluginRequirement(name = 'lsmod', plugin = lsmod.Lsmod, version = (2, 0, 0)) ] def _generator(self, mods: Iterator[Any]): - kernel = contexts.Module(self._context, self.config['darwin'], self.config['primary'], 0) + kernel = self.context.modules[self.config['kernel']] - handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, self.config['primary'], kernel, mods) + handlers = mac.MacUtilities.generate_kernel_handler_info(self.context, kernel.layer_name, kernel, mods) policy_list = kernel.object_from_symbol(symbol_name = "mac_policy_list").cast("mac_policy_list") entries = kernel.object(object_type = "array", offset = policy_list.entries.dereference().vol.offset, subtype = kernel.get_type('mac_policy_list_element'), + absolute = True, count = policy_list.staticmax + 1) for i, ent in enumerate(entries): @@ -65,7 +64,8 @@ def _generator(self, mods: Iterator[Any]): if call_addr is None or call_addr == 0: continue - module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, call_addr) + module_name, symbol_name = mac.MacUtilities.lookup_module_address(self.context, handlers, call_addr, + self.config['kernel']) yield (0, (check, ent_name, format_hints.Hex(call_addr), module_name, symbol_name)) @@ -73,5 +73,4 @@ def run(self): return renderers.TreeGrid([("Member", str), ("Policy Name", str), ("Handler Address", format_hints.Hex), ("Handler Module", str), ("Handler Symbol", str)], self._generator( - lsmod.Lsmod.list_modules(self.context, self.config['primary'], - self.config['darwin']))) + lsmod.Lsmod.list_modules(self.context, self.config['kernel']))) diff --git a/volatility3/framework/plugins/mac/vfsevents.py b/volatility3/framework/plugins/mac/vfsevents.py index 97b23168c2..38f4172ce7 100644 --- a/volatility3/framework/plugins/mac/vfsevents.py +++ b/volatility3/framework/plugins/mac/vfsevents.py @@ -2,7 +2,7 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # -from volatility3.framework import renderers, interfaces, exceptions, contexts +from volatility3.framework import renderers, interfaces, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility @@ -10,7 +10,7 @@ class VFSevents(interfaces.plugins.PluginInterface): """ Lists processes that are filtering file system events """ - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) event_types = [ "CREATE_FILE", "DELETE", "STAT_CHANGED", "RENAME", "CONTENT_MODIFIED", "EXCHANGE", "FINDER_INFO_CHANGED", @@ -20,10 +20,8 @@ class VFSevents(interfaces.plugins.PluginInterface): @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "darwin", description = "Mac kernel"), + requirements.ModuleRequirement(name = 'kernel', description = 'Kernel module for the OS', + architectures = ["Intel32", "Intel64"]), ] def _generator(self): @@ -32,7 +30,7 @@ def _generator(self): Also lists which event(s) a process is registered for """ - kernel = contexts.Module(self.context, self.config['darwin'], self.config['primary'], 0) + kernel = self.context.modules[self.config['kernel']] watcher_table = kernel.object_from_symbol("watcher_table") @@ -48,6 +46,7 @@ def _generator(self): try: event_array = kernel.object(object_type = "array", offset = watcher.event_list, + absolute = True, count = 13, subtype = kernel.get_type("unsigned char")) diff --git a/volatility3/framework/plugins/timeliner.py b/volatility3/framework/plugins/timeliner.py index bd6377ca14..6bf5925046 100644 --- a/volatility3/framework/plugins/timeliner.py +++ b/volatility3/framework/plugins/timeliner.py @@ -42,13 +42,13 @@ class Timeliner(interfaces.plugins.PluginInterface): """Runs all relevant plugins that provide time related information and orders the results by time.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.timeline = {} self.usable_plugins = None - self.automagics = None + self.automagics: Optional[List[interfaces.automagic.AutomagicInterface]] = None @classmethod def get_usable_plugins(cls, selected_list: List[str] = None) -> List[Type]: @@ -113,9 +113,9 @@ def _generator(self, runable_plugins: List[TimeLinerInterface]) -> Optional[Iter for plugin in runable_plugins: plugin_name = plugin.__class__.__name__ self._progress_callback((runable_plugins.index(plugin) * 100) // len(runable_plugins), - "Running plugin {}...".format(plugin_name)) + f"Running plugin {plugin_name}...") try: - vollog.log(logging.INFO, "Running {}".format(plugin_name)) + vollog.log(logging.INFO, f"Running {plugin_name}") for (item, timestamp_type, timestamp) in plugin.generate_timeline(): times = self.timeline.get((plugin_name, item), {}) if times.get(timestamp_type, None) is not None: @@ -131,7 +131,7 @@ def _generator(self, runable_plugins: List[TimeLinerInterface]) -> Optional[Iter times.get(TimeLinerType.CHANGED, renderers.NotApplicableValue()) ])) except Exception: - vollog.log(logging.INFO, "Exception occurred running plugin: {}".format(plugin_name)) + vollog.log(logging.INFO, f"Exception occurred running plugin: {plugin_name}") vollog.log(logging.DEBUG, traceback.format_exc()) for data_item in sorted(data, key = self._sort_function): yield data_item @@ -176,6 +176,7 @@ def run(self): self.usable_plugins = self.usable_plugins or self.get_usable_plugins() self.automagics = self.automagics or automagic.available(self._context) plugins_to_run = [] + requirement_configs = {} filter_list = self.config['plugin-filter'] # Identify plugins that we can run which output datetimes @@ -183,16 +184,29 @@ def run(self): try: automagics = automagic.choose_automagic(self.automagics, plugin_class) + for requirement in plugin_class.get_requirements(): + if requirement.name in requirement_configs: + config_req, config_value = requirement_configs[requirement.name] + if requirement == config_req: + self.context.config[interfaces.configuration.path_join( + self.config_path, plugin_class.__name__)] = config_value + plugin = plugins.construct_plugin(self.context, automagics, plugin_class, self.config_path, self._progress_callback, self.open) + for requirement in plugin.get_requirements(): + if requirement.name not in requirement_configs: + config_value = plugin.config.get(requirement.name, None) + if config_value: + requirement_configs[requirement.name] = (requirement, config_value) + if isinstance(plugin, TimeLinerInterface): if not len(filter_list) or any( [filter in plugin.__module__ + '.' + plugin.__class__.__name__ for filter in filter_list]): plugins_to_run.append(plugin) except exceptions.UnsatisfiedException as excp: # Remove the failed plugin from the list and continue - vollog.debug("Unable to satisfy {}: {}".format(plugin_class.__name__, excp.unsatisfied)) + vollog.debug(f"Unable to satisfy {plugin_class.__name__}: {excp.unsatisfied}") continue if self.config.get('record-config', False): diff --git a/volatility3/framework/plugins/windows/bigpools.py b/volatility3/framework/plugins/windows/bigpools.py index a2c23a31cf..c81125f079 100644 --- a/volatility3/framework/plugins/windows/bigpools.py +++ b/volatility3/framework/plugins/windows/bigpools.py @@ -3,6 +3,7 @@ # import logging +import os from typing import List, Optional, Tuple, Iterator from volatility3.framework import interfaces, renderers, exceptions, symbols @@ -19,17 +20,15 @@ class BigPools(interfaces.plugins.PluginInterface): """List big page pools.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.StringRequirement(name = 'tags', description = "Comma separated list of pool tags to filter pools returned", optional = True, @@ -83,7 +82,7 @@ def list_big_pools(cls, new_table_name = intermed.IntermediateSymbolTable.create( context = context, config_path = configuration.path_join(context.symbol_space[symbol_table].config_path, "bigpools"), - sub_path = "windows", + sub_path = os.path.join("windows", "bigpools"), filename = big_pools_json_filename, table_mapping = {'nt_symbols': symbol_table}, class_types = {'_POOL_TRACKER_BIG_PAGES': extensions.pool.POOL_TRACKER_BIG_PAGES}) @@ -106,10 +105,11 @@ def _generator(self) -> Iterator[Tuple[int, Tuple[int, str]]]: # , str, int]]]: tags = [tag for tag in self.config["tags"].split(',')] else: tags = None + kernel = self.context.modules[self.config['kernel']] for big_pool in self.list_big_pools(context = self.context, - layer_name = self.config["primary"], - symbol_table = self.config["nt_symbols"], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, tags = tags): num_bytes = big_pool.get_number_of_bytes() diff --git a/volatility3/framework/plugins/windows/cachedump.py b/volatility3/framework/plugins/windows/cachedump.py index 42bb93a732..ddfa856b9c 100644 --- a/volatility3/framework/plugins/windows/cachedump.py +++ b/volatility3/framework/plugins/windows/cachedump.py @@ -1,66 +1,74 @@ # This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # - +import logging from struct import unpack +from typing import Tuple from Crypto.Cipher import ARC4, AES from Crypto.Hash import HMAC from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements +from volatility3.framework.layers import registry from volatility3.framework.symbols.windows import versions from volatility3.plugins.windows import hashdump, lsadump from volatility3.plugins.windows.registry import hivelist +vollog = logging.getLogger(__name__) + class Cachedump(interfaces.plugins.PluginInterface): """Dumps lsa secrets from memory""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)), - requirements.PluginRequirement(name = 'lsadump', plugin = lsadump.Lsadump, version = (1, 0, 0)) + requirements.PluginRequirement(name = 'lsadump', plugin = lsadump.Lsadump, version = (1, 0, 0)), + requirements.PluginRequirement(name = 'hashdump', plugin = hashdump.Hashdump, version = (1, 1, 0)) ] - def get_nlkm(self, sechive, lsakey, is_vista_or_later): + @staticmethod + def get_nlkm(sechive: registry.RegistryHive, lsakey: bytes, is_vista_or_later: bool): return lsadump.Lsadump.get_secret_by_name(sechive, 'NL$KM', lsakey, is_vista_or_later) - def decrypt_hash(self, edata, nlkm, ch, xp): + @staticmethod + def decrypt_hash(edata: bytes, nlkm: bytes, ch, xp: bool): if xp: hmac_md5 = HMAC.new(nlkm, ch) rc4key = hmac_md5.digest() rc4 = ARC4.new(rc4key) - data = rc4.encrypt(edata) + data = rc4.encrypt(edata) # lgtm [py/weak-cryptographic-algorithm] else: # based on Based on code from http://lab.mediaservice.net/code/cachedump.rb aes = AES.new(nlkm[16:32], AES.MODE_CBC, ch) - data = "" + data = b"" for i in range(0, len(edata), 16): buf = edata[i:i + 16] if len(buf) < 16: - buf += (16 - len(buf)) * "\00" + buf += (16 - len(buf)) * b"\00" data += aes.decrypt(buf) return data - def parse_cache_entry(self, cache_data): + @staticmethod + def parse_cache_entry(cache_data: bytes) -> Tuple[int, int, int, bytes, bytes]: (uname_len, domain_len) = unpack(" Tuple[str, str, str, bytes]: """Get the data from the cache and separate it into the username, domain name, and hash data""" uname_offset = 72 pad = 2 * ((uname_len / 2) % 2) @@ -68,33 +76,37 @@ def parse_decrypted_cache(self, dec_data, uname_len, domain_len, domain_name_len pad = 2 * ((domain_len / 2) % 2) domain_name_offset = int(domain_offset + domain_len + pad) hashh = dec_data[:0x10] - username = dec_data[uname_offset:uname_offset + uname_len] - username = username.decode('utf-16-le', 'replace') - domain = dec_data[domain_offset:domain_offset + domain_len] - domain = domain.decode('utf-16-le', 'replace') - domain_name = dec_data[domain_name_offset:domain_name_offset + domain_name_len] - domain_name = domain_name.decode('utf-16-le', 'replace') + username = dec_data[uname_offset:uname_offset + uname_len].decode('utf-16-le', 'replace') + domain = dec_data[domain_offset:domain_offset + domain_len].decode('utf-16-le', 'replace') + domain_name = dec_data[domain_name_offset:domain_name_offset + domain_name_len].decode('utf-16-le', 'replace') return (username, domain, domain_name, hashh) def _generator(self, syshive, sechive): bootkey = hashdump.Hashdump.get_bootkey(syshive) if not bootkey: - raise ValueError('Unable to find bootkey') + vollog.warning('Unable to find bootkey') + return + + kernel = self.context.modules[self.config['kernel']] - vista_or_later = versions.is_vista_or_later(context = self.context, symbol_table = self.config['nt_symbols']) + vista_or_later = versions.is_vista_or_later(context = self.context, + symbol_table = kernel.symbol_table_name) lsakey = lsadump.Lsadump.get_lsa_key(sechive, bootkey, vista_or_later) if not lsakey: - raise ValueError('Unable to find lsa key') + vollog.warning('Unable to find lsa key') + return nlkm = self.get_nlkm(sechive, lsakey, vista_or_later) if not nlkm: - raise ValueError('Unable to find nlkma key') + vollog.warning('Unable to find nlkma key') + return - cache = sechive.get_key("Cache") + cache = hashdump.Hashdump.get_hive_key(sechive, "Cache") if not cache: - raise ValueError('Unable to find cache key') + vollog.warning('Unable to find cache key') + return for cache_item in cache.get_values(): if cache_item.Name == "NL$Control": @@ -116,10 +128,13 @@ def _generator(self, syshive, sechive): def run(self): offset = self.config.get('offset', None) + syshive = sechive = None + kernel = self.context.modules[self.config['kernel']] + for hive in hivelist.HiveList.list_hives(self.context, self.config_path, - self.config['primary'], - self.config['nt_symbols'], + kernel.layer_name, + kernel.symbol_table_name, hive_offsets = None if offset is None else [offset]): if hive.get_name().split('\\')[-1].upper() == 'SYSTEM': @@ -127,5 +142,12 @@ def run(self): if hive.get_name().split('\\')[-1].upper() == 'SECURITY': sechive = hive - return renderers.TreeGrid([("Username", str), ("Domain", str), ("Domain name", str), ('Hashh', bytes)], + if syshive is None or sechive is None: + if syshive is None: + vollog.warning('Unable to locate SYSTEM hive') + if sechive is None: + vollog.warning('Unable to locate SECURITY hive') + return + + return renderers.TreeGrid([("Username", str), ("Domain", str), ("Domain name", str), ('Hash', bytes)], self._generator(syshive, sechive)) diff --git a/volatility3/framework/plugins/windows/callbacks.py b/volatility3/framework/plugins/windows/callbacks.py index 545ff9811f..352dba448a 100644 --- a/volatility3/framework/plugins/windows/callbacks.py +++ b/volatility3/framework/plugins/windows/callbacks.py @@ -19,16 +19,14 @@ class Callbacks(interfaces.plugins.PluginInterface): """Lists kernel callbacks and notification routines.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'ssdt', plugin = ssdt.SSDT, version = (1, 0, 0)), requirements.PluginRequirement(name = 'svcscan', plugin = svcscan.SvcScan, version = (1, 0, 0)) ] @@ -90,7 +88,7 @@ def list_notify_routines(cls, context: interfaces.context.ContextInterface, laye try: symbol_offset = ntkrnlmp.get_symbol(symbol_name).address except exceptions.SymbolError: - vollog.debug("Cannot find {}".format(symbol_name)) + vollog.debug(f"Cannot find {symbol_name}") continue if is_vista_or_later and extended_list: @@ -191,9 +189,10 @@ def list_bugcheck_reason_callbacks(cls, context: interfaces.context.ContextInter continue try: - component = ntkrnlmp.object( + component: Union[ + interfaces.renderers.BaseAbsentValue, interfaces.objects.ObjectInterface] = ntkrnlmp.object( "string", absolute = True, offset = callback.Component, max_length = 64, errors = "replace" - ) # type: Union[interfaces.renderers.BaseAbsentValue, interfaces.objects.ObjectInterface] + ) except exceptions.InvalidAddressException: component = renderers.UnreadableValue() @@ -244,17 +243,20 @@ def list_bugcheck_callbacks(cls, context: interfaces.context.ContextInterface, l def _generator(self): - callback_table_name = self.create_callback_table(self.context, self.config["nt_symbols"], self.config_path) + kernel = self.context.modules[self.config['kernel']] - collection = ssdt.SSDT.build_module_collection(self.context, self.config['primary'], self.config['nt_symbols']) + callback_table_name = self.create_callback_table(self.context, kernel.symbol_table_name, + self.config_path) + + collection = ssdt.SSDT.build_module_collection(self.context, kernel.layer_name, kernel.symbol_table_name) callback_methods = (self.list_notify_routines, self.list_bugcheck_callbacks, self.list_bugcheck_reason_callbacks, self.list_registry_callbacks) for callback_method in callback_methods: for callback_type, callback_address, callback_detail in callback_method(self.context, - self.config['primary'], - self.config['nt_symbols'], + kernel.layer_name, + kernel.symbol_table_name, callback_table_name): if callback_detail is None: diff --git a/volatility3/framework/plugins/windows/cmdline.py b/volatility3/framework/plugins/windows/cmdline.py index bf5cc4bfe6..af6035abb3 100644 --- a/volatility3/framework/plugins/windows/cmdline.py +++ b/volatility3/framework/plugins/windows/cmdline.py @@ -15,17 +15,15 @@ class CmdLine(interfaces.plugins.PluginInterface): """Lists process command line arguments.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.ListRequirement(name = 'pid', element_type = int, @@ -56,6 +54,7 @@ def get_cmdline(cls, context: interfaces.context.ContextInterface, kernel_table_ return result_text def _generator(self, procs): + kernel = self.context.modules[self.config['kernel']] for proc in procs: process_name = utility.array_to_string(proc.ImageFileName) @@ -63,13 +62,13 @@ def _generator(self, procs): try: proc_id = proc.UniqueProcessId - result_text = self.get_cmdline(self.context, self.config["nt_symbols"], proc) + result_text = self.get_cmdline(self.context, kernel.symbol_table_name, proc) except exceptions.SwappedInvalidAddressException as exp: - result_text = "Required memory at {0:#x} is inaccessible (swapped)".format(exp.invalid_address) + result_text = f"Required memory at {exp.invalid_address:#x} is inaccessible (swapped)" except exceptions.PagedInvalidAddressException as exp: - result_text = "Required memory at {0:#x} is not valid (process exited?)".format(exp.invalid_address) + result_text = f"Required memory at {exp.invalid_address:#x} is not valid (process exited?)" except exceptions.InvalidAddressException as exp: result_text = "Process {}: Required memory at {:#x} is not valid (incomplete layer {}?)".format( @@ -78,11 +77,12 @@ def _generator(self, procs): yield (0, (proc.UniqueProcessId, process_name, result_text)) def run(self): + kernel = self.context.modules[self.config['kernel']] filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) return renderers.TreeGrid([("PID", int), ("Process", str), ("Args", str)], self._generator( pslist.PsList.list_processes(context = self.context, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, filter_func = filter_func))) diff --git a/volatility3/framework/plugins/windows/crashinfo.py b/volatility3/framework/plugins/windows/crashinfo.py new file mode 100644 index 0000000000..d66d86cd13 --- /dev/null +++ b/volatility3/framework/plugins/windows/crashinfo.py @@ -0,0 +1,97 @@ +# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import logging +import datetime +from volatility3.framework import interfaces, renderers +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints, conversion +from volatility3.framework.objects import utility +from volatility3.framework.layers import crash + +vollog = logging.getLogger(__name__) + + +class Crashinfo(interfaces.plugins.PluginInterface): + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.TranslationLayerRequirement(name = 'primary', + description = 'Memory layer for the kernel', + architectures = ["Intel32", "Intel64"]), + ] + + def _generator(self, layer: crash.WindowsCrashDump32Layer): + header = layer.get_header() + uptime = datetime.timedelta(microseconds = int(header.SystemUpTime) / 10) + + if header.DumpType == 0x1: + dump_type = "Full Dump (0x1)" + elif header.DumpType == 0x5: + dump_type = "Bitmap Dump (0x5)" + else: + # this should never happen since the crash layer only accepts 0x1 and 0x5 + dump_type = f"Unknown/Unsupported ({header.DumpType:#x})" + + if header.DumpType == 0x5: + summary_header = layer.get_summary_header() + bitmap_header_size = format_hints.Hex(summary_header.HeaderSize) + bitmap_size = format_hints.Hex(summary_header.BitmapSize) + bitmap_pages = format_hints.Hex(summary_header.Pages) + else: + bitmap_header_size = bitmap_size = bitmap_pages = renderers.NotApplicableValue() + + yield (0, ( + utility.array_to_string(header.Signature), + header.MajorVersion, + header.MinorVersion, + format_hints.Hex(header.DirectoryTableBase), + format_hints.Hex(header.PfnDataBase), + format_hints.Hex(header.PsLoadedModuleList), + format_hints.Hex(header.PsActiveProcessHead), + header.MachineImageType, + header.NumberProcessors, + format_hints.Hex(header.KdDebuggerDataBlock), + dump_type, + str(uptime), + utility.array_to_string(header.Comment), + conversion.wintime_to_datetime(header.SystemTime), + bitmap_header_size, + bitmap_size, + bitmap_pages, + )) + + def run(self): + crash_layer = None + for layer_name in self._context.layers: + layer = self._context.layers[layer_name] + if isinstance(layer, crash.WindowsCrashDump32Layer): + crash_layer = layer + break + + if crash_layer is None: + vollog.error("This plugin requires a Windows crash dump") + raise + + return renderers.TreeGrid([ + ("Signature", str), + ("MajorVersion", int), + ("MinorVersion", int), + ("DirectoryTableBase", format_hints.Hex), + ("PfnDataBase", format_hints.Hex), + ("PsLoadedModuleList", format_hints.Hex), + ("PsActiveProcessHead", format_hints.Hex), + ("MachineImageType", int), + ("NumberProcessors", int), + ("KdDebuggerDataBlock", format_hints.Hex), + ("DumpType", str), + ("SystemUpTime", str), + ("Comment", str), + ("SystemTime", datetime.datetime), + ("BitmapHeaderSize", format_hints.Hex), + ("BitmapSize", format_hints.Hex), + ("BitmapPages", format_hints.Hex), + ], self._generator(crash_layer)) diff --git a/volatility3/framework/plugins/windows/dlllist.py b/volatility3/framework/plugins/windows/dlllist.py index 4740e5751b..b24f2c0ca7 100644 --- a/volatility3/framework/plugins/windows/dlllist.py +++ b/volatility3/framework/plugins/windows/dlllist.py @@ -20,17 +20,15 @@ class DllList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the loaded modules in a particular windows memory image.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), requirements.ListRequirement(name = 'pid', @@ -83,7 +81,7 @@ def dump_pe(cls, file_handle.seek(offset) file_handle.write(data) except (IOError, exceptions.VolatilityException, OverflowError, ValueError) as excp: - vollog.debug("Unable to dump dll at offset {}: {}".format(dll_entry.DllBase, excp)) + vollog.debug(f"Unable to dump dll at offset {dll_entry.DllBase}: {excp}") return None return file_handle @@ -94,7 +92,9 @@ def _generator(self, procs): "pe", class_types = pe.class_types) - kuser = info.Info.get_kuser_structure(self.context, self.config['primary'], self.config['nt_symbols']) + kernel = self.context.modules[self.config['kernel']] + + kuser = info.Info.get_kuser_structure(self.context, kernel.layer_name, kernel.symbol_table_name) nt_major_version = int(kuser.NtMajorVersion) nt_minor_version = int(kuser.NtMinorVersion) # LoadTime only applies to versions higher or equal to Window 7 (6.1 and higher) @@ -131,23 +131,33 @@ def _generator(self, procs): entry, self.open, proc_layer_name, - prefix = "pid.{}.".format(proc_id)) + prefix = f"pid.{proc_id}.") file_output = "Error outputting file" if file_handle: file_handle.close() file_output = file_handle.preferred_filename + try: + dllbase = format_hints.Hex(entry.DllBase) + except exceptions.InvalidAddressException: + dllbase = renderers.NotAvailableValue() + + try: + size_of_image = format_hints.Hex(entry.SizeOfImage) + except exceptions.InvalidAddressException: + size_of_image = renderers.NotAvailableValue() yield (0, (proc.UniqueProcessId, proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, - errors = 'replace'), format_hints.Hex(entry.DllBase), - format_hints.Hex(entry.SizeOfImage), BaseDllName, FullDllName, DllLoadTime, file_output)) + errors = 'replace'), dllbase, size_of_image, BaseDllName, + FullDllName, DllLoadTime, file_output)) def generate_timeline(self): + kernel = self.context.modules[self.config['kernel']] for row in self._generator( pslist.PsList.list_processes(context = self.context, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'])): + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name)): _depth, row_data = row if not isinstance(row_data[6], datetime.datetime): continue @@ -157,12 +167,13 @@ def generate_timeline(self): def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config['kernel']] return renderers.TreeGrid([("PID", int), ("Process", str), ("Base", format_hints.Hex), ("Size", format_hints.Hex), ("Name", str), ("Path", str), ("LoadTime", datetime.datetime), ("File output", str)], self._generator( pslist.PsList.list_processes(context = self.context, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, filter_func = filter_func))) diff --git a/volatility3/framework/plugins/windows/driverirp.py b/volatility3/framework/plugins/windows/driverirp.py index 64231b9dbd..7f9bc6b088 100644 --- a/volatility3/framework/plugins/windows/driverirp.py +++ b/volatility3/framework/plugins/windows/driverirp.py @@ -22,25 +22,23 @@ class DriverIrp(interfaces.plugins.PluginInterface): """List IRPs for drivers in a particular windows memory image.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.PluginRequirement(name = 'ssdt', plugin = ssdt.SSDT, version = (1, 0, 0)), requirements.PluginRequirement(name = 'driverscan', plugin = driverscan.DriverScan, version = (1, 0, 0)), - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), ] def _generator(self): + kernel = self.context.modules[self.config['kernel']] - collection = ssdt.SSDT.build_module_collection(self.context, self.config['primary'], self.config['nt_symbols']) + collection = ssdt.SSDT.build_module_collection(self.context, kernel.layer_name, kernel.symbol_table_name) - for driver in driverscan.DriverScan.scan_drivers(self.context, self.config['primary'], - self.config['nt_symbols']): + for driver in driverscan.DriverScan.scan_drivers(self.context, kernel.layer_name, kernel.symbol_table_name): try: driver_name = driver.get_driver_name() diff --git a/volatility3/framework/plugins/windows/driverscan.py b/volatility3/framework/plugins/windows/driverscan.py index 7b44d4ebc8..2cf309014d 100644 --- a/volatility3/framework/plugins/windows/driverscan.py +++ b/volatility3/framework/plugins/windows/driverscan.py @@ -13,16 +13,14 @@ class DriverScan(interfaces.plugins.PluginInterface): """Scans for drivers present in a particular windows memory image.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'poolscanner', plugin = poolscanner.PoolScanner, version = (1, 0, 0)), ] @@ -51,7 +49,9 @@ def scan_drivers(cls, yield mem_object def _generator(self): - for driver in self.scan_drivers(self.context, self.config['primary'], self.config['nt_symbols']): + kernel = self.context.modules[self.config['kernel']] + + for driver in self.scan_drivers(self.context, kernel.layer_name, kernel.symbol_table_name): try: driver_name = driver.get_driver_name() diff --git a/volatility3/framework/plugins/windows/dumpfiles.py b/volatility3/framework/plugins/windows/dumpfiles.py index 757dbd3c32..58166ee7f2 100755 --- a/volatility3/framework/plugins/windows/dumpfiles.py +++ b/volatility3/framework/plugins/windows/dumpfiles.py @@ -4,13 +4,13 @@ import logging import ntpath +from typing import List, Tuple, Type, Optional, Generator from volatility3.framework import interfaces, renderers, exceptions, constants -from volatility3.plugins.windows import handles -from volatility3.plugins.windows import pslist from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints -from volatility3.framework.objects import utility -from typing import List, Tuple, Type, Optional +from volatility3.plugins.windows import handles +from volatility3.plugins.windows import pslist + vollog = logging.getLogger(__name__) FILE_DEVICE_DISK = 0x7 @@ -25,17 +25,15 @@ class DumpFiles(interfaces.plugins.PluginInterface): """Dumps cached file contents from Windows memory samples.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.IntRequirement(name = 'pid', description = "Process ID to include (all other processes are excluded)", optional = True), @@ -80,32 +78,32 @@ def dump_file_producer(cls, file_object: interfaces.objects.ObjectInterface, filedata.write(data) if not bytes_written: - vollog.debug("No data is cached for the file at {0:#x}".format(file_object.vol.offset)) + vollog.debug(f"No data is cached for the file at {file_object.vol.offset:#x}") return None else: - vollog.debug("Stored {}".format(filedata.preferred_filename)) + vollog.debug(f"Stored {filedata.preferred_filename}") return filedata except exceptions.InvalidAddressException: - vollog.debug("Unable to dump file at {0:#x}".format(file_object.vol.offset)) + vollog.debug(f"Unable to dump file at {file_object.vol.offset:#x}") return None @classmethod def process_file_object(cls, context: interfaces.context.ContextInterface, primary_layer_name: str, open_method: Type[interfaces.plugins.FileHandlerInterface], - file_obj: interfaces.objects.ObjectInterface) -> Tuple: + file_obj: interfaces.objects.ObjectInterface) -> Generator[Tuple, None, None]: """Given a FILE_OBJECT, dump data to separate files for each of the three file caches. :param context: the context to operate upon :param primary_layer_name: primary/virtual layer to operate on :param open_method: class for constructing output files - :param file_object: the FILE_OBJECT + :param file_obj: the FILE_OBJECT """ # Filtering by these types of devices prevents us from processing other types of devices that # use the "File" object type, such as \Device\Tcp and \Device\NamedPipe. if file_obj.DeviceObject.DeviceType not in [FILE_DEVICE_DISK, FILE_DEVICE_NETWORK_FILE_SYSTEM]: vollog.log(constants.LOGLEVEL_VVV, - "The file object at {0:#x} is not a file on disk".format(file_obj.vol.offset)) + f"The file object at {file_obj.vol.offset:#x} is not a file on disk") return # Depending on the type of object (DataSection, ImageSection, SharedCacheMap) we may need to @@ -134,7 +132,7 @@ def process_file_object(cls, context: interfaces.context.ContextInterface, prima dump_parameters.append((control_area, memory_layer, extension)) except exceptions.InvalidAddressException: vollog.log(constants.LOGLEVEL_VVV, - "{0} is unavailable for file {1:#x}".format(member_name, file_obj.vol.offset)) + f"{member_name} is unavailable for file {file_obj.vol.offset:#x}") # The SharedCacheMap is handled differently than the caches above. # We carve these "pages" from the primary_layer. @@ -145,7 +143,7 @@ def process_file_object(cls, context: interfaces.context.ContextInterface, prima dump_parameters.append((shared_cache_map, primary_layer, "vacb")) except exceptions.InvalidAddressException: vollog.log(constants.LOGLEVEL_VVV, - "SharedCacheMap is unavailable for file {0:#x}".format(file_obj.vol.offset)) + f"SharedCacheMap is unavailable for file {file_obj.vol.offset:#x}") for memory_object, layer, extension in dump_parameters: cache_name = EXTENSION_CACHE_MAP[extension] @@ -167,6 +165,7 @@ def process_file_object(cls, context: interfaces.context.ContextInterface, prima file_output) def _generator(self, procs: List, offsets: List): + kernel = self.context.modules[self.config['kernel']] if procs: # The handles plugin doesn't expose any staticmethod/classmethod, and it also requires stashing @@ -175,11 +174,11 @@ def _generator(self, procs: List, offsets: List): # results instead of just dealing with them as direct objects here. handles_plugin = handles.Handles(context = self.context, config_path = self._config_path) type_map = handles_plugin.get_type_map(context = self.context, - layer_name = self.config["primary"], - symbol_table = self.config["nt_symbols"]) + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name) cookie = handles_plugin.find_cookie(context = self.context, - layer_name = self.config["primary"], - symbol_table = self.config["nt_symbols"]) + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name) for proc in procs: @@ -187,7 +186,7 @@ def _generator(self, procs: List, offsets: List): object_table = proc.ObjectTable except exceptions.InvalidAddressException: vollog.log(constants.LOGLEVEL_VVV, - "Cannot access _EPROCESS.ObjectTable at {0:#x}".format(proc.vol.offset)) + f"Cannot access _EPROCESS.ObjectTable at {proc.vol.offset:#x}") continue for entry in handles_plugin.handles(object_table): @@ -195,12 +194,12 @@ def _generator(self, procs: List, offsets: List): obj_type = entry.get_object_type(type_map, cookie) if obj_type == "File": file_obj = entry.Body.cast("_FILE_OBJECT") - for result in self.process_file_object(self.context, self.config["primary"], self.open, + for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): yield (0, result) except exceptions.InvalidAddressException: vollog.log(constants.LOGLEVEL_VVV, - "Cannot extract file from _OBJECT_HEADER at {0:#x}".format(entry.vol.offset)) + f"Cannot extract file from _OBJECT_HEADER at {entry.vol.offset:#x}") # Pull file objects from the VADs. This will produce DLLs and EXEs that are # mapped into the process as images, but that the process doesn't have an @@ -219,36 +218,38 @@ def _generator(self, procs: List, offsets: List): if not file_obj.is_valid(): continue - for result in self.process_file_object(self.context, self.config["primary"], self.open, + for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): yield (0, result) except exceptions.InvalidAddressException: vollog.log(constants.LOGLEVEL_VVV, - "Cannot extract file from VAD at {0:#x}".format(vad.vol.offset)) + f"Cannot extract file from VAD at {vad.vol.offset:#x}") elif offsets: # Now process any offsets explicitly requested by the user. for offset, is_virtual in offsets: try: - layer_name = self.config["primary"] + layer_name = kernel.layer_name # switch to a memory layer if the user provided --physaddr instead of --virtaddr if not is_virtual: layer_name = self.context.layers[layer_name].config["memory_layer"] - file_obj = self.context.object(self.config["nt_symbols"] + constants.BANG + "_FILE_OBJECT", + file_obj = self.context.object( + kernel.symbol_table_name + constants.BANG + "_FILE_OBJECT", layer_name = layer_name, - native_layer_name = self.config["primary"], + native_layer_name = kernel.layer_name, offset = offset) - for result in self.process_file_object(self.context, self.config["primary"], self.open, file_obj): + for result in self.process_file_object(self.context, kernel.layer_name, self.open, file_obj): yield (0, result) except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, "Cannot extract file at {0:#x}".format(offset)) + vollog.log(constants.LOGLEVEL_VVV, f"Cannot extract file at {offset:#x}") def run(self): # a list of tuples (, ) where is the address and is True for virtual. offsets = [] # a list of processes matching the pid filter. all files for these process(es) will be dumped. procs = [] + kernel = self.context.modules[self.config['kernel']] if self.config.get("virtaddr", None) is not None: offsets.append((self.config["virtaddr"], True)) @@ -257,8 +258,8 @@ def run(self): else: filter_func = pslist.PsList.create_pid_filter([self.config.get("pid", None)]) procs = pslist.PsList.list_processes(self.context, - self.config["primary"], - self.config["nt_symbols"], + kernel.layer_name, + kernel.symbol_table_name, filter_func = filter_func) return renderers.TreeGrid([("Cache", str), ("FileObject", format_hints.Hex), ("FileName", str), diff --git a/volatility3/framework/plugins/windows/envars.py b/volatility3/framework/plugins/windows/envars.py index 8d3ee85062..9791fa5807 100644 --- a/volatility3/framework/plugins/windows/envars.py +++ b/volatility3/framework/plugins/windows/envars.py @@ -16,16 +16,14 @@ class Envars(interfaces.plugins.PluginInterface): "Display process environment variables" _version = (1, 0, 0) - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -48,11 +46,12 @@ def _get_silent_vars(self) -> List[str]: """ values = [] + kernel = self.context.modules[self.config['kernel']] for hive in hivelist.HiveList.list_hives(context = self.context, base_config_path = self.config_path, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, hive_offsets = None): sys = False ntuser = False @@ -192,10 +191,11 @@ def _generator(self, data): def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config['kernel']] return renderers.TreeGrid([("PID", int), ("Process", str), ("Block", str), ("Variable", str), ("Value", str)], self._generator( pslist.PsList.list_processes(context = self.context, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, filter_func = filter_func))) diff --git a/volatility3/framework/plugins/windows/filescan.py b/volatility3/framework/plugins/windows/filescan.py index e1756630db..79d85eb8d9 100644 --- a/volatility3/framework/plugins/windows/filescan.py +++ b/volatility3/framework/plugins/windows/filescan.py @@ -13,15 +13,13 @@ class FileScan(interfaces.plugins.PluginInterface): """Scans for file objects present in a particular windows memory image.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'poolscanner', plugin = poolscanner.PoolScanner, version = (1, 0, 0)), ] @@ -50,7 +48,9 @@ def scan_files(cls, yield mem_object def _generator(self): - for fileobj in self.scan_files(self.context, self.config['primary'], self.config['nt_symbols']): + kernel = self.context.modules[self.config['kernel']] + + for fileobj in self.scan_files(self.context, kernel.layer_name, kernel.symbol_table_name): try: file_name = fileobj.FileName.String diff --git a/volatility3/framework/plugins/windows/getservicesids.py b/volatility3/framework/plugins/windows/getservicesids.py index 2b5ab11f4c..a47b1178a7 100644 --- a/volatility3/framework/plugins/windows/getservicesids.py +++ b/volatility3/framework/plugins/windows/getservicesids.py @@ -31,7 +31,7 @@ class GetServiceSIDs(interfaces.plugins.PluginInterface): """Lists process token sids.""" _version = (1, 0, 0) - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -53,22 +53,22 @@ def __init__(self, *args, **kwargs): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) ] def _generator(self): - # Go all over the hives + kernel = self.context.modules[self.config['kernel']] + # Get the system hive for hive in hivelist.HiveList.list_hives(context = self.context, base_config_path = self.config_path, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, + filter_string = 'machine\\system', hive_offsets = None): - # Get ConrolSet\Services. + # Get ControlSet\Services. try: services = hive.get_key(r"CurrentControlSet\Services") except (KeyError, exceptions.InvalidAddressException): diff --git a/volatility3/framework/plugins/windows/getsids.py b/volatility3/framework/plugins/windows/getsids.py index 30503b2bff..da6fa71c98 100644 --- a/volatility3/framework/plugins/windows/getsids.py +++ b/volatility3/framework/plugins/windows/getsids.py @@ -29,7 +29,7 @@ class GetSIDs(interfaces.plugins.PluginInterface): """Print the SIDs owning each process""" _version = (1, 0, 0) - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -53,10 +53,8 @@ def __init__(self, *args, **kwargs): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -75,12 +73,14 @@ def lookup_user_sids(self) -> Dict[str, str]: key = "Microsoft\\Windows NT\\CurrentVersion\\ProfileList" val = "ProfileImagePath" + kernel = self.context.modules[self.config['kernel']] sids = {} for hive in hivelist.HiveList.list_hives(context = self.context, base_config_path = self.config_path, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, + filter_string = 'config\\software', hive_offsets = None): try: @@ -96,9 +96,9 @@ def lookup_user_sids(self) -> Dict[str, str]: value_data = node.decode_data() if isinstance(value_data, int): value_data = format_hints.MultiTypeData(value_data, encoding = 'utf-8') - elif registry.RegValueTypes.get(node.Type) == registry.RegValueTypes.REG_BINARY: + elif registry.RegValueTypes(node.Type) == registry.RegValueTypes.REG_BINARY: value_data = format_hints.MultiTypeData(value_data, show_hex = True) - elif registry.RegValueTypes.get(node.Type) == registry.RegValueTypes.REG_MULTI_SZ: + elif registry.RegValueTypes(node.Type) == registry.RegValueTypes.REG_MULTI_SZ: value_data = format_hints.MultiTypeData(value_data, encoding = 'utf-16-le', split_nulls = True) @@ -153,10 +153,11 @@ def _generator(self, procs): def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config['kernel']] return renderers.TreeGrid([("PID", int), ("Process", str), ("SID", str), ("Name", str)], self._generator( pslist.PsList.list_processes(context = self.context, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, filter_func = filter_func))) diff --git a/volatility3/framework/plugins/windows/handles.py b/volatility3/framework/plugins/windows/handles.py index d08f888e31..ab11d30d6c 100644 --- a/volatility3/framework/plugins/windows/handles.py +++ b/volatility3/framework/plugins/windows/handles.py @@ -24,7 +24,7 @@ class Handles(interfaces.plugins.PluginInterface): """Lists process open handles.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) def __init__(self, *args, **kwargs): @@ -38,10 +38,8 @@ def __init__(self, *args, **kwargs): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.ListRequirement(name = 'pid', element_type = int, description = "Process IDs to include (all other processes are excluded)", @@ -69,7 +67,9 @@ def _get_item(self, handle_table_entry, handle_value): process' handle table, determine where the corresponding object's _OBJECT_HEADER can be found.""" - virtual = self.config["primary"] + kernel = self.context.modules[self.config['kernel']] + + virtual = kernel.layer_name try: # before windows 7 @@ -80,7 +80,7 @@ def _get_item(self, handle_table_entry, handle_value): object_header.GrantedAccess = handle_table_entry.GrantedAccess except AttributeError: # starting with windows 8 - is_64bit = symbols.symbol_table_is_64bit(self.context, self.config["nt_symbols"]) + is_64bit = symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) if is_64bit: if handle_table_entry.LowValue == 0: @@ -104,8 +104,7 @@ def _get_item(self, handle_table_entry, handle_value): offset = handle_table_entry.InfoTable & ~7 # print("LowValue: {0:#x} Magic: {1:#x} Offset: {2:#x}".format(handle_table_entry.InfoTable, magic, offset)) - object_header = self.context.object(self.config["nt_symbols"] + constants.BANG + "_OBJECT_HEADER", - virtual, + object_header = self.context.object(kernel.symbol_table_name + constants.BANG + "_OBJECT_HEADER", virtual, offset = offset) object_header.GrantedAccess = handle_table_entry.GrantedAccessBits @@ -124,10 +123,11 @@ def find_sar_value(self): if not has_capstone: return None + kernel = self.context.modules[self.config['kernel']] - virtual_layer_name = self.config['primary'] + virtual_layer_name = kernel.layer_name kvo = self.context.layers[virtual_layer_name].config['kernel_virtual_offset'] - ntkrnlmp = self.context.module(self.config["nt_symbols"], layer_name = virtual_layer_name, offset = kvo) + ntkrnlmp = self.context.module(kernel.symbol_table_name, layer_name = virtual_layer_name, offset = kvo) try: func_addr = ntkrnlmp.get_symbol("ObpCaptureHandleInformationEx").address @@ -169,10 +169,10 @@ def get_type_map(cls, context: interfaces.context.ContextInterface, layer_name: symbol_table: The name of the table containing the kernel symbols Returns: - A mapping of type indicies to type names + A mapping of type indices to type names """ - type_map = {} # type: Dict[int, str] + type_map: Dict[int, str] = {} kvo = context.layers[layer_name].config['kernel_virtual_offset'] ntkrnlmp = context.module(symbol_table, layer_name = layer_name, offset = kvo) @@ -203,7 +203,7 @@ def get_type_map(cls, context: interfaces.context.ContextInterface, layer_name: type_name = objt.Name.String except exceptions.InvalidAddressException: vollog.log(constants.LOGLEVEL_VVV, - "Cannot access _OBJECT_HEADER Name at {0:#x}".format(objt.vol.offset)) + f"Cannot access _OBJECT_HEADER Name at {objt.vol.offset:#x}") continue type_map[i] = type_name @@ -227,10 +227,12 @@ def _make_handle_array(self, offset, level, depth = 0): """Parse a process' handle table and yield valid handle table entries, going as deep into the table "levels" as necessary.""" - virtual = self.config["primary"] + kernel = self.context.modules[self.config['kernel']] + + virtual = kernel.layer_name kvo = self.context.layers[virtual].config['kernel_virtual_offset'] - ntkrnlmp = self.context.module(self.config["nt_symbols"], layer_name = virtual, offset = kvo) + ntkrnlmp = self.context.module(kernel.symbol_table_name, layer_name = virtual, offset = kvo) if level > 0: subtype = ntkrnlmp.get_type("pointer") @@ -291,21 +293,22 @@ def handles(self, handle_table): yield handle_table_entry def _generator(self, procs): + kernel = self.context.modules[self.config['kernel']] type_map = self.get_type_map(context = self.context, - layer_name = self.config["primary"], - symbol_table = self.config["nt_symbols"]) + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name) cookie = self.find_cookie(context = self.context, - layer_name = self.config["primary"], - symbol_table = self.config["nt_symbols"]) + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name) for proc in procs: try: object_table = proc.ObjectTable except exceptions.InvalidAddressException: vollog.log(constants.LOGLEVEL_VVV, - "Cannot access _EPROCESS.ObjectType at {0:#x}".format(proc.vol.offset)) + f"Cannot access _EPROCESS.ObjectType at {proc.vol.offset:#x}") continue process_name = utility.array_to_string(proc.ImageFileName) @@ -320,10 +323,10 @@ def _generator(self, procs): obj_name = item.file_name_with_device() elif obj_type == "Process": item = entry.Body.cast("_EPROCESS") - obj_name = "{} Pid {}".format(utility.array_to_string(proc.ImageFileName), item.UniqueProcessId) + obj_name = f"{utility.array_to_string(proc.ImageFileName)} Pid {item.UniqueProcessId}" elif obj_type == "Thread": item = entry.Body.cast("_ETHREAD") - obj_name = "Tid {} Pid {}".format(item.Cid.UniqueThread, item.Cid.UniqueProcess) + obj_name = f"Tid {item.Cid.UniqueThread} Pid {item.Cid.UniqueProcess}" elif obj_type == "Key": item = entry.Body.cast("_CM_KEY_BODY") obj_name = item.get_full_key_name() @@ -335,7 +338,7 @@ def _generator(self, procs): except (exceptions.InvalidAddressException): vollog.log(constants.LOGLEVEL_VVV, - "Cannot access _OBJECT_HEADER at {0:#x}".format(entry.vol.offset)) + f"Cannot access _OBJECT_HEADER at {entry.vol.offset:#x}") continue yield (0, (proc.UniqueProcessId, process_name, format_hints.Hex(entry.Body.vol.offset), @@ -345,12 +348,13 @@ def _generator(self, procs): def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config['kernel']] return renderers.TreeGrid([("PID", int), ("Process", str), ("Offset", format_hints.Hex), ("HandleValue", format_hints.Hex), ("Type", str), ("GrantedAccess", format_hints.Hex), ("Name", str)], self._generator( pslist.PsList.list_processes(self.context, - self.config['primary'], - self.config['nt_symbols'], + kernel.layer_name, + kernel.symbol_table_name, filter_func = filter_func))) diff --git a/volatility3/framework/plugins/windows/hashdump.py b/volatility3/framework/plugins/windows/hashdump.py index ed2f1988a8..4d0b25b5c3 100644 --- a/volatility3/framework/plugins/windows/hashdump.py +++ b/volatility3/framework/plugins/windows/hashdump.py @@ -21,15 +21,14 @@ class Hashdump(interfaces.plugins.PluginInterface): """Dumps user hashes from memory""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) + _version = (1, 1, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) ] @@ -60,11 +59,22 @@ def get_requirements(cls): empty_lm = b"\xaa\xd3\xb4\x35\xb5\x14\x04\xee\xaa\xd3\xb4\x35\xb5\x14\x04\xee" empty_nt = b"\x31\xd6\xcf\xe0\xd1\x6a\xe9\x31\xb7\x3c\x59\xd7\xe0\xc0\x89\xc0" + @classmethod + def get_hive_key(cls, hive: registry.RegistryHive, key: str): + result = None + try: + result = hive.get_key(key) + except KeyError: + vollog.info( + f"Unable to load the required registry key {hive.get_name()}\\{key} from this memory image") + return result + @classmethod def get_user_keys(cls, samhive: registry.RegistryHive) -> List[interfaces.objects.ObjectInterface]: user_key_path = "SAM\\Domains\\Account\\Users" - user_key = samhive.get_key(user_key_path) + user_key = cls.get_hive_key(samhive, user_key_path) + if not user_key: return [] return [k for k in user_key.get_subkeys() if k.Name != "Names"] @@ -72,10 +82,10 @@ def get_user_keys(cls, samhive: registry.RegistryHive) -> List[interfaces.object @classmethod def get_bootkey(cls, syshive: registry.RegistryHive) -> Optional[bytes]: cs = 1 - lsa_base = "ControlSet{0:03}".format(cs) + "\\Control\\Lsa" + lsa_base = f"ControlSet{cs:03}" + "\\Control\\Lsa" lsa_keys = ["JD", "Skew1", "GBG", "Data"] - lsa = syshive.get_key(lsa_base) + lsa = cls.get_hive_key(syshive, lsa_base) if not lsa: return None @@ -83,9 +93,10 @@ def get_bootkey(cls, syshive: registry.RegistryHive) -> Optional[bytes]: bootkey = '' for lk in lsa_keys: - key = syshive.get_key(lsa_base + '\\' + lk) - - class_data = syshive.read(key.Class + 4, key.ClassLength) + key = cls.get_hive_key(syshive, lsa_base + '\\' + lk) + class_data = None + if key: + class_data = syshive.read(key.Class + 4, key.ClassLength) if class_data is None: return None @@ -102,7 +113,7 @@ def get_hbootkey(cls, samhive: registry.RegistryHive, bootkey: bytes) -> Optiona if not bootkey: return None - sam_account_key = samhive.get_key(sam_account_path) + sam_account_key = cls.get_hive_key(samhive, sam_account_path) if not sam_account_key: return None @@ -121,7 +132,7 @@ def get_hbootkey(cls, samhive: registry.RegistryHive, bootkey: bytes) -> Optiona rc4_key = md5.digest() rc4 = ARC4.new(rc4_key) - hbootkey = rc4.encrypt(sam_data[0x80:0xA0]) + hbootkey = rc4.encrypt(sam_data[0x80:0xA0]) # lgtm [py/weak-cryptographic-algorithm] return hbootkey elif revision == 3: # AES encrypted @@ -133,17 +144,18 @@ def get_hbootkey(cls, samhive: registry.RegistryHive, bootkey: bytes) -> Optiona return None @classmethod - def decrypt_single_salted_hash(cls, rid, hbootkey: bytes, enc_hash: bytes, lmntstr, salt: bytes) -> Optional[bytes]: + def decrypt_single_salted_hash(cls, rid, hbootkey: bytes, enc_hash: bytes, _lmntstr, + salt: bytes) -> Optional[bytes]: (des_k1, des_k2) = cls.sid_to_key(rid) des1 = DES.new(des_k1, DES.MODE_ECB) des2 = DES.new(des_k2, DES.MODE_ECB) cipher = AES.new(hbootkey[:16], AES.MODE_CBC, salt) obfkey = cipher.decrypt(enc_hash) - return des1.decrypt(obfkey[:8]) + des2.decrypt(obfkey[8:16]) + return des1.decrypt(obfkey[:8]) + des2.decrypt(obfkey[8:16]) # lgtm [py/weak-cryptographic-algorithm] @classmethod def get_user_hashes(cls, user: registry.CM_KEY_NODE, samhive: registry.RegistryHive, - hbootkey: bytes) -> Tuple[bytes, bytes]: + hbootkey: bytes) -> Optional[Tuple[bytes, bytes]]: ## Will sometimes find extra user with rid = NAMES, returns empty strings right now try: rid = int(str(user.get_name()), 16) @@ -199,22 +211,16 @@ def sid_to_key(cls, sid: int) -> Tuple[bytes, bytes]: @classmethod def sidbytes_to_key(cls, s: bytes) -> bytes: """Builds final DES key from the strings generated in sid_to_key""" - key = [] - key.append(s[0] >> 1) - key.append(((s[0] & 0x01) << 6) | (s[1] >> 2)) - key.append(((s[1] & 0x03) << 5) | (s[2] >> 3)) - key.append(((s[2] & 0x07) << 4) | (s[3] >> 4)) - key.append(((s[3] & 0x0F) << 3) | (s[4] >> 5)) - key.append(((s[4] & 0x1F) << 2) | (s[5] >> 6)) - key.append(((s[5] & 0x3F) << 1) | (s[6] >> 7)) - key.append(s[6] & 0x7F) + key = [s[0] >> 1, ((s[0] & 0x01) << 6) | (s[1] >> 2), ((s[1] & 0x03) << 5) | (s[2] >> 3), + ((s[2] & 0x07) << 4) | (s[3] >> 4), ((s[3] & 0x0F) << 3) | (s[4] >> 5), + ((s[4] & 0x1F) << 2) | (s[5] >> 6), ((s[5] & 0x3F) << 1) | (s[6] >> 7), s[6] & 0x7F] for i in range(8): key[i] = (key[i] << 1) key[i] = cls.odd_parity[key[i]] return bytes(key) @classmethod - def decrypt_single_hash(cls, rid, hbootkey, enc_hash: bytes, lmntstr): + def decrypt_single_hash(cls, rid: int, hbootkey: bytes, enc_hash: bytes, lmntstr: bytes): (des_k1, des_k2) = cls.sid_to_key(rid) des1 = DES.new(des_k1, DES.MODE_ECB) des2 = DES.new(des_k2, DES.MODE_ECB) @@ -223,26 +229,25 @@ def decrypt_single_hash(cls, rid, hbootkey, enc_hash: bytes, lmntstr): md5.update(hbootkey[:0x10] + pack(" Optional[bytes]: - V = None + def get_user_name(cls, user: registry.CM_KEY_NODE, samhive: registry.RegistryHive) -> Optional[bytes]: + value = None for v in user.get_values(): if v.get_name() == 'V': - V = samhive.read(v.Data + 4, v.DataLength) - if not V: + value = samhive.read(v.Data + 4, v.DataLength) + if not value: return None - name_offset = unpack(" len(V): + name_offset = unpack(" len(value): return None - username = V[name_offset:name_offset + name_length] + username = value[name_offset:name_offset + name_length] return username # replaces the dump_hashes method in vol2 @@ -276,16 +281,17 @@ def _generator(self, syshive: registry.RegistryHive, samhive: registry.RegistryH rid = int(str(user.get_name()), 16) yield (0, (name, rid, lmout, ntout)) else: - raise ValueError("Hbootkey is not valid") + vollog.warning("Hbootkey is not valid") def run(self): offset = self.config.get('offset', None) syshive = None samhive = None + kernel = self.context.modules[self.config['kernel']] for hive in hivelist.HiveList.list_hives(self.context, self.config_path, - self.config['primary'], - self.config['nt_symbols'], + kernel.layer_name, + kernel.symbol_table_name, hive_offsets = None if offset is None else [offset]): if hive.get_name().split('\\')[-1].upper() == 'SYSTEM': diff --git a/volatility3/framework/plugins/windows/info.py b/volatility3/framework/plugins/windows/info.py index 76b81eb00d..172664aef8 100644 --- a/volatility3/framework/plugins/windows/info.py +++ b/volatility3/framework/plugins/windows/info.py @@ -16,16 +16,14 @@ class Info(plugins.PluginInterface): """Show OS & kernel details of the memory sample being analyzed.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols") ] @classmethod @@ -150,19 +148,23 @@ def get_ntheader_structure(cls, context: interfaces.context.ContextInterface, co def _generator(self): - layer_name = self.config['primary'] - symbol_table = self.config['nt_symbols'] + kernel = self.context.modules[self.config['kernel']] + + layer_name = kernel.layer_name + symbol_table = kernel.symbol_table_name + layer = self.context.layers[layer_name] + table = self.context.symbol_space[symbol_table] kdbg = self.get_kdbg_structure(self.context, self.config_path, layer_name, symbol_table) - yield (0, ("Kernel Base", hex(self.config["primary.kernel_virtual_offset"]))) - yield (0, ("DTB", hex(self.config["primary.page_map_offset"]))) - yield (0, ("Symbols", self.config["nt_symbols.isf_url"])) + yield (0, ("Kernel Base", hex(layer.config["kernel_virtual_offset"]))) + yield (0, ("DTB", hex(layer.config["page_map_offset"]))) + yield (0, ("Symbols", table.config["isf_url"])) yield (0, ("Is64Bit", str(symbols.symbol_table_is_64bit(self.context, symbol_table)))) yield (0, ("IsPAE", str(self.context.layers[layer_name].metadata.get("pae", False)))) - for i, layer in self.get_depends(self.context, "primary"): - yield (0, (layer.name, "{} {}".format(i, layer.__class__.__name__))) + for i, layer in self.get_depends(self.context, layer_name): + yield (0, (layer.name, f"{i} {layer.__class__.__name__}")) if kdbg.Header.OwnerTag == 0x4742444B: @@ -173,7 +175,7 @@ def _generator(self): vers = self.get_version_structure(self.context, layer_name, symbol_table) yield (0, ("KdVersionBlock", hex(vers.vol.offset))) - yield (0, ("Major/Minor", "{0}.{1}".format(vers.MajorVersion, vers.MinorVersion))) + yield (0, ("Major/Minor", f"{vers.MajorVersion}.{vers.MinorVersion}")) yield (0, ("MachineType", str(vers.MachineType))) ntkrnlmp = self.get_kernel_module(self.context, layer_name, symbol_table) diff --git a/volatility3/framework/plugins/windows/lsadump.py b/volatility3/framework/plugins/windows/lsadump.py index b9f8df9fe0..edf16416c6 100644 --- a/volatility3/framework/plugins/windows/lsadump.py +++ b/volatility3/framework/plugins/windows/lsadump.py @@ -3,12 +3,14 @@ # import logging from struct import unpack +from typing import Optional from Crypto.Cipher import ARC4, DES, AES from Crypto.Hash import MD5, SHA256 from volatility3.framework import interfaces, renderers from volatility3.framework.configuration import requirements +from volatility3.framework.layers import registry from volatility3.framework.symbols.windows import versions from volatility3.plugins.windows import hashdump from volatility3.plugins.windows.registry import hivelist @@ -19,21 +21,20 @@ class Lsadump(interfaces.plugins.PluginInterface): """Dumps lsa secrets from memory""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), - requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) + requirements.VersionRequirement(name = 'hashdump', component = hashdump.Hashdump, version = (1, 1, 0)), + requirements.VersionRequirement(name = 'hivelist', component = hivelist.HiveList, version = (1, 0, 0)) ] @classmethod - def decrypt_aes(cls, secret, key): + def decrypt_aes(cls, secret: bytes, key: bytes) -> bytes: """ Based on code from http://lab.mediaservice.net/code/cachedump.rb """ @@ -54,7 +55,7 @@ def decrypt_aes(cls, secret, key): return data @classmethod - def get_lsa_key(cls, sechive, bootkey, vista_or_later): + def get_lsa_key(cls, sechive: registry.RegistryHive, bootkey: bytes, vista_or_later: bool) -> Optional[bytes]: if not bootkey: return None @@ -63,7 +64,7 @@ def get_lsa_key(cls, sechive, bootkey, vista_or_later): else: policy_key = 'PolSecretEncryptionKey' - enc_reg_key = sechive.get_key("Policy\\" + policy_key) + enc_reg_key = hashdump.Hashdump.get_hive_key(sechive, "Policy\\" + policy_key) if not enc_reg_key: return None enc_reg_value = next(enc_reg_key.get_values()) @@ -83,7 +84,7 @@ def get_lsa_key(cls, sechive, bootkey, vista_or_later): rc4key = md5.digest() rc4 = ARC4.new(rc4key) - lsa_key = rc4.decrypt(obf_lsa_key[12:60]) + lsa_key = rc4.decrypt(obf_lsa_key[12:60]) # lgtm [py/weak-cryptographic-algorithm] lsa_key = lsa_key[0x10:0x20] else: lsa_key = cls.decrypt_aes(obf_lsa_key, bootkey) @@ -91,28 +92,26 @@ def get_lsa_key(cls, sechive, bootkey, vista_or_later): return lsa_key @classmethod - def get_secret_by_name(cls, sechive, name, lsakey, is_vista_or_later): - try: - enc_secret_key = sechive.get_key("Policy\\Secrets\\" + name + "\\CurrVal") - except KeyError: - raise ValueError("Unable to read cache from memory") - - enc_secret_value = next(enc_secret_key.get_values()) - if not enc_secret_value: - return None + def get_secret_by_name(cls, sechive: registry.RegistryHive, name: str, lsakey: bytes, is_vista_or_later: bool): + enc_secret_key = hashdump.Hashdump.get_hive_key(sechive, "Policy\\Secrets\\" + name + "\\CurrVal") - enc_secret = sechive.read(enc_secret_value.Data + 4, enc_secret_value.DataLength) - if not enc_secret: - return None + secret = None + if enc_secret_key: + enc_secret_value = next(enc_secret_key.get_values()) + if enc_secret_value: + + enc_secret = sechive.read(enc_secret_value.Data + 4, enc_secret_value.DataLength) + if enc_secret: + + if not is_vista_or_later: + secret = cls.decrypt_secret(enc_secret[0xC:], lsakey) + else: + secret = cls.decrypt_aes(enc_secret, lsakey) - if not is_vista_or_later: - secret = cls.decrypt_secret(enc_secret[0xC:], lsakey) - else: - secret = cls.decrypt_aes(enc_secret, lsakey) return secret @classmethod - def decrypt_secret(cls, secret, key): + def decrypt_secret(cls, secret: bytes, key: bytes): """Python implementation of SystemFunction005. Decrypts a block of data with DES using given key. @@ -126,34 +125,42 @@ def decrypt_secret(cls, secret, key): des_key = hashdump.Hashdump.sidbytes_to_key(block_key) des = DES.new(des_key, DES.MODE_ECB) enc_block = enc_block + b"\x00" * int(abs(8 - len(enc_block)) % 8) - decrypted_data += des.decrypt(enc_block) + decrypted_data += des.decrypt(enc_block) # lgtm [py/weak-cryptographic-algorithm] j += 7 if len(key[j:j + 7]) < 7: j = len(key[j:j + 7]) - (dec_data_len, ) = unpack(" List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.IntRequirement(name = 'pid', description = "Process ID to include (all other processes are excluded)", @@ -35,6 +34,7 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] optional = True) ] + def _generator(self, procs): for proc in procs: pid = "Unknown" @@ -48,9 +48,13 @@ def _generator(self, procs): excp.layer_name)) continue - file_handle = self.open("pid.{}.dmp".format(pid)) + if self.config['dump']: + file_handle = self.open(f"pid.{pid}.dmp") + else: + # Ensure the file isn't actually created if not needed + file_handle = contextlib.ExitStack() with file_handle as file_data: - + file_offset = 0 for mapval in proc_layer.mapping(0x0, proc_layer.maximum_address, ignore_errors = True): offset, size, mapped_offset, mapped_size, maplayer = mapval @@ -65,17 +69,22 @@ def _generator(self, procs): vollog.debug("Unable to write {}'s address {} to {}".format( proc_layer_name, offset, file_handle.preferred_filename)) - yield (0, (format_hints.Hex(offset), format_hints.Hex(mapped_offset), format_hints.Hex(mapped_size), - format_hints.Hex(offset), file_output)) + yield (0, (format_hints.Hex(offset), format_hints.Hex(mapped_offset), + format_hints.Hex(mapped_size), + format_hints.Hex(file_offset), file_output)) + + file_offset += mapped_size offset += mapped_size def run(self): filter_func = pslist.PsList.create_pid_filter([self.config.get('pid', None)]) + kernel = self.context.modules[self.config['kernel']] return renderers.TreeGrid([("Virtual", format_hints.Hex), ("Physical", format_hints.Hex), - ("Size", format_hints.Hex), ("Offset", format_hints.Hex), ("File output", str)], + ("Size", format_hints.Hex), ("Offset in File", format_hints.Hex), + ("File output", str)], self._generator( pslist.PsList.list_processes(context = self.context, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, filter_func = filter_func))) diff --git a/volatility3/framework/plugins/windows/modscan.py b/volatility3/framework/plugins/windows/modscan.py index 13b23577fb..b661d71d72 100644 --- a/volatility3/framework/plugins/windows/modscan.py +++ b/volatility3/framework/plugins/windows/modscan.py @@ -17,16 +17,14 @@ class ModScan(interfaces.plugins.PluginInterface): """Scans for modules present in a particular windows memory image.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.VersionRequirement(name = 'poolerscanner', component = poolscanner.PoolScanner, version = (1, 0, 0)), @@ -81,7 +79,7 @@ def get_session_layers(cls, Returns: A list of session layer names """ - seen_ids = [] # type: List[interfaces.objects.ObjectInterface] + seen_ids: List[interfaces.objects.ObjectInterface] = [] filter_func = pslist.PsList.create_pid_filter(pids or []) for proc in pslist.PsList.list_processes(context = context, @@ -137,14 +135,16 @@ def find_session_layer(cls, context: interfaces.context.ContextInterface, sessio return None def _generator(self): - session_layers = list(self.get_session_layers(self.context, self.config['primary'], self.config['nt_symbols'])) + kernel = self.context.modules[self.config['kernel']] + + session_layers = list(self.get_session_layers(self.context, kernel.layer_name, kernel.symbol_table_name)) pe_table_name = intermed.IntermediateSymbolTable.create(self.context, self.config_path, "windows", "pe", class_types = pe.class_types) - for mod in self.scan_modules(self.context, self.config['primary'], self.config['nt_symbols']): + for mod in self.scan_modules(self.context, kernel.layer_name, kernel.symbol_table_name): try: BaseDllName = mod.BaseDllName.get_string() @@ -160,7 +160,7 @@ def _generator(self): if self.config['dump']: session_layer_name = self.find_session_layer(self.context, session_layers, mod.DllBase) - file_output = "Cannot find a viable session layer for {0:#x}".format(mod.DllBase) + file_output = f"Cannot find a viable session layer for {mod.DllBase:#x}" if session_layer_name: file_handle = dlllist.DllList.dump_pe(self.context, pe_table_name, diff --git a/volatility3/framework/plugins/windows/modules.py b/volatility3/framework/plugins/windows/modules.py index b030066db7..7b488a8eb5 100644 --- a/volatility3/framework/plugins/windows/modules.py +++ b/volatility3/framework/plugins/windows/modules.py @@ -19,16 +19,14 @@ class Modules(interfaces.plugins.PluginInterface): """Lists the loaded kernel modules.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 1, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), requirements.VersionRequirement(name = 'dlllist', component = dlllist.DllList, version = (2, 0, 0)), requirements.BooleanRequirement(name = 'dump', @@ -38,13 +36,14 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] ] def _generator(self): + kernel = self.context.modules[self.config['kernel']] pe_table_name = intermed.IntermediateSymbolTable.create(self.context, self.config_path, "windows", "pe", class_types = pe.class_types) - for mod in self.list_modules(self.context, self.config['primary'], self.config['nt_symbols']): + for mod in self.list_modules(self.context, kernel.layer_name, kernel.symbol_table_name): try: BaseDllName = mod.BaseDllName.get_string() @@ -86,7 +85,7 @@ def get_session_layers(cls, Returns: A list of session layer names """ - seen_ids = [] # type: List[interfaces.objects.ObjectInterface] + seen_ids: List[interfaces.objects.ObjectInterface] = [] filter_func = pslist.PsList.create_pid_filter(pids or []) for proc in pslist.PsList.list_processes(context = context, diff --git a/volatility3/framework/plugins/windows/mutantscan.py b/volatility3/framework/plugins/windows/mutantscan.py index c0a47d38e1..29e27c9b10 100644 --- a/volatility3/framework/plugins/windows/mutantscan.py +++ b/volatility3/framework/plugins/windows/mutantscan.py @@ -13,15 +13,13 @@ class MutantScan(interfaces.plugins.PluginInterface): """Scans for mutexes present in a particular windows memory image.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'poolscanner', plugin = poolscanner.PoolScanner, version = (1, 0, 0)), ] @@ -50,7 +48,9 @@ def scan_mutants(cls, yield mem_object def _generator(self): - for mutant in self.scan_mutants(self.context, self.config['primary'], self.config['nt_symbols']): + kernel = self.context.modules[self.config['kernel']] + + for mutant in self.scan_mutants(self.context, kernel.layer_name, kernel.symbol_table_name): try: name = mutant.get_name() diff --git a/volatility3/framework/plugins/windows/netscan.py b/volatility3/framework/plugins/windows/netscan.py index 0f58c97f57..33b5a7fbc3 100644 --- a/volatility3/framework/plugins/windows/netscan.py +++ b/volatility3/framework/plugins/windows/netscan.py @@ -4,7 +4,8 @@ import datetime import logging -from typing import Iterable, List, Optional +import os +from typing import Iterable, List, Optional, Tuple, Type from volatility3.framework import constants, exceptions, interfaces, renderers, symbols from volatility3.framework.configuration import requirements @@ -21,16 +22,14 @@ class NetScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for network objects present in a particular windows memory image.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.VersionRequirement(name = 'poolscanner', component = poolscanner.PoolScanner, version = (1, 0, 0)), @@ -82,7 +81,7 @@ def create_netscan_constraints(context: interfaces.context.ContextInterface, @classmethod def determine_tcpip_version(cls, context: interfaces.context.ContextInterface, layer_name: str, - nt_symbol_table: str) -> str: + nt_symbol_table: str) -> Tuple[str, Type]: """Tries to determine which symbol filename to use for the image's tcpip driver. The logic is partially taken from the info plugin. Args: @@ -151,8 +150,8 @@ def determine_tcpip_version(cls, context: interfaces.context.ContextInterface, l (6, 1, 8400): "netscan-win7-x86", (6, 2, 9200): "netscan-win8-x86", (6, 3, 9600): "netscan-win81-x86", - (10, 0, 10240): "netscan-win10-x86", - (10, 0, 10586): "netscan-win10-x86", + (10, 0, 10240): "netscan-win10-10240-x86", + (10, 0, 10586): "netscan-win10-10586-x86", (10, 0, 14393): "netscan-win10-14393-x86", (10, 0, 15063): "netscan-win10-15063-x86", (10, 0, 16299): "netscan-win10-15063-x86", @@ -179,7 +178,7 @@ def determine_tcpip_version(cls, context: interfaces.context.ContextInterface, l (10, 0, 16299): "netscan-win10-16299-x64", (10, 0, 17134): "netscan-win10-17134-x64", (10, 0, 17763): "netscan-win10-17763-x64", - (10, 0, 18362): "netscan-win10-17763-x64", + (10, 0, 18362): "netscan-win10-18362-x64", (10, 0, 18363): "netscan-win10-18363-x64", (10, 0, 19041): "netscan-win10-19041-x64" } @@ -211,12 +210,12 @@ def determine_tcpip_version(cls, context: interfaces.context.ContextInterface, l latest_version = current_versions[-1] filename = version_dict.get(latest_version) - vollog.debug("Unable to find exact matching symbol file, going with latest: {}".format(filename)) + vollog.debug(f"Unable to find exact matching symbol file, going with latest: {filename}") else: raise NotImplementedError("This version of Windows is not supported: {}.{} {}.{}!".format( nt_major_version, nt_minor_version, vers.MajorVersion, vers_minor_version)) - vollog.debug("Determined symbol filename: {}".format(filename)) + vollog.debug(f"Determined symbol filename: {filename}") return filename, class_types @@ -244,7 +243,7 @@ def create_netscan_symbol_table(cls, context: interfaces.context.ContextInterfac return intermed.IntermediateSymbolTable.create(context, config_path, - "windows", + os.path.join("windows", "netscan"), symbol_filename, class_types = class_types, table_mapping = table_mapping) @@ -278,19 +277,21 @@ def scan(cls, def _generator(self, show_corrupt_results: Optional[bool] = None): """ Generates the network objects for use in rendering. """ - netscan_symbol_table = self.create_netscan_symbol_table(self.context, self.config["primary"], - self.config["nt_symbols"], self.config_path) + kernel = self.context.modules[self.config['kernel']] - for netw_obj in self.scan(self.context, self.config['primary'], self.config['nt_symbols'], - netscan_symbol_table): + netscan_symbol_table = self.create_netscan_symbol_table(self.context, kernel.layer_name, + kernel.symbol_table_name, + self.config_path) - vollog.debug("Found netw obj @ 0x{:2x} of assumed type {}".format(netw_obj.vol.offset, type(netw_obj))) + for netw_obj in self.scan(self.context, kernel.layer_name, kernel.symbol_table_name, netscan_symbol_table): + + vollog.debug(f"Found netw obj @ 0x{netw_obj.vol.offset:2x} of assumed type {type(netw_obj)}") # objects passed pool header constraints. check for additional constraints if strict flag is set. if not show_corrupt_results and not netw_obj.is_valid(): continue if isinstance(netw_obj, network._UDP_ENDPOINT): - vollog.debug("Found UDP_ENDPOINT @ 0x{:2x}".format(netw_obj.vol.offset)) + vollog.debug(f"Found UDP_ENDPOINT @ 0x{netw_obj.vol.offset:2x}") # For UdpA, the state is always blank and the remote end is asterisks for ver, laddr, _ in netw_obj.dual_stack_sockets(): @@ -300,7 +301,7 @@ def _generator(self, show_corrupt_results: Optional[bool] = None): or renderers.UnreadableValue())) elif isinstance(netw_obj, network._TCP_ENDPOINT): - vollog.debug("Found _TCP_ENDPOINT @ 0x{:2x}".format(netw_obj.vol.offset)) + vollog.debug(f"Found _TCP_ENDPOINT @ 0x{netw_obj.vol.offset:2x}") if netw_obj.get_address_family() == network.AF_INET: proto = "TCPv4" elif netw_obj.get_address_family() == network.AF_INET6: @@ -321,7 +322,7 @@ def _generator(self, show_corrupt_results: Optional[bool] = None): # check for isinstance of tcp listener last, because all other objects are inherited from here elif isinstance(netw_obj, network._TCP_LISTENER): - vollog.debug("Found _TCP_LISTENER @ 0x{:2x}".format(netw_obj.vol.offset)) + vollog.debug(f"Found _TCP_LISTENER @ 0x{netw_obj.vol.offset:2x}") # For TcpL, the state is always listening and the remote port is zero for ver, laddr, raddr in netw_obj.dual_stack_sockets(): @@ -331,7 +332,7 @@ def _generator(self, show_corrupt_results: Optional[bool] = None): or renderers.UnreadableValue())) else: # this should not happen therefore we log it. - vollog.debug("Found network object unsure of its type: {} of type {}".format(netw_obj, type(netw_obj))) + vollog.debug(f"Found network object unsure of its type: {netw_obj} of type {type(netw_obj)}") def generate_timeline(self): for row in self._generator(): diff --git a/volatility3/framework/plugins/windows/netstat.py b/volatility3/framework/plugins/windows/netstat.py new file mode 100644 index 0000000000..e71683c10e --- /dev/null +++ b/volatility3/framework/plugins/windows/netstat.py @@ -0,0 +1,524 @@ +# This file is Copyright 2020 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +import datetime +import logging +from typing import Iterable, Optional, Generator, Tuple + +from volatility3.framework import constants, exceptions, interfaces, renderers, symbols +from volatility3.framework.configuration import requirements +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols.windows import pdbutil +from volatility3.framework.symbols.windows.extensions import network +from volatility3.plugins import timeliner +from volatility3.plugins.windows import netscan, modules + +vollog = logging.getLogger(__name__) + + +class NetStat(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): + """Traverses network tracking structures present in a particular windows memory image.""" + + _required_framework_version = (2, 0, 0) + _version = (1, 0, 0) + + @classmethod + def get_requirements(cls): + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), + requirements.VersionRequirement(name = 'netscan', component = netscan.NetScan, version = (1, 0, 0)), + requirements.VersionRequirement(name = 'modules', component = modules.Modules, version = (1, 0, 0)), + requirements.VersionRequirement(name = 'pdbutil', component = pdbutil.PDBUtility, version = (1, 0, 0)), + requirements.BooleanRequirement( + name = 'include-corrupt', + description = + "Radically eases result validation. This will show partially overwritten data. WARNING: the results are likely to include garbage and/or corrupt data. Be cautious!", + default = False, + optional = True), + ] + + @classmethod + def _decode_pointer(self, value): + """Copied from `windows.handles`. + + Windows encodes pointers to objects and decodes them on the fly + before using them. + + This function mimics the decoding routine so we can generate the + proper pointer values as well. + """ + + value = value & 0xFFFFFFFFFFFFFFFC + + return value + + @classmethod + def read_pointer(cls, context: interfaces.context.ContextInterface, layer_name: str, offset: int, + length: int) -> int: + """Reads a pointer at a given offset and returns the address it points to. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + offset: Offset of pointer + length: Pointer length + + Returns: + The value the pointer points to. + """ + + return int.from_bytes(context.layers[layer_name].read(offset, length), "little") + + @classmethod + def parse_bitmap(cls, context: interfaces.context.ContextInterface, layer_name: str, bitmap_offset: int, + bitmap_size_in_byte: int) -> list: + """Parses a given bitmap and looks for each occurrence of a 1. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + bitmap_offset: Start address of bitmap + bitmap_size_in_byte: Bitmap size in Byte, not in bit. + + Returns: + The list of indices at which a 1 was found. + """ + ret = [] + for idx in range(bitmap_size_in_byte): + current_byte = context.layers[layer_name].read(bitmap_offset + idx, 1)[0] + current_offs = idx * 8 + for bit in range(8): + if current_byte & (1 << bit) != 0: + ret.append(bit + current_offs) + return ret + + @classmethod + def enumerate_structures_by_port(cls, + context: interfaces.context.ContextInterface, + layer_name: str, + net_symbol_table: str, + port: int, + port_pool_addr: int, + proto = "tcp") -> \ + Iterable[interfaces.objects.ObjectInterface]: + """Lists all UDP Endpoints and TCP Listeners by parsing UdpPortPool and TcpPortPool. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + net_symbol_table: The name of the table containing the tcpip types + port: Current port as integer to lookup the associated object. + port_pool_addr: Address of port pool object + proto: Either "tcp" or "udp" to decide which types to use. + + Returns: + The list of network objects from this image's TCP and UDP `PortPools` + """ + if proto == "tcp": + obj_name = net_symbol_table + constants.BANG + "_TCP_LISTENER" + ptr_offset = context.symbol_space.get_type(obj_name).relative_child_offset("Next") + elif proto == "udp": + obj_name = net_symbol_table + constants.BANG + "_UDP_ENDPOINT" + ptr_offset = context.symbol_space.get_type(obj_name).relative_child_offset("Next") + else: + # invalid argument. + return + + vollog.debug(f"Current Port: {port}") + # the given port serves as a shifted index into the port pool lists + list_index = port >> 8 + truncated_port = port & 0xff + + # constructing port_pool object here so callers don't have to + port_pool = context.object(net_symbol_table + constants.BANG + "_INET_PORT_POOL", + layer_name = layer_name, + offset = port_pool_addr) + + # first, grab the given port's PortAssignment (`_PORT_ASSIGNMENT`) + inpa = port_pool.PortAssignments[list_index] + + # then parse the port assignment list (`_PORT_ASSIGNMENT_LIST`) and grab the correct entry + assignment = inpa.InPaBigPoolBase.Assignments[truncated_port] + + if not assignment: + return + + # the value within assignment.Entry is a) masked and b) points inside of the network object + # first decode the pointer + netw_inside = cls._decode_pointer(assignment.Entry) + + if netw_inside: + # if the value is valid, calculate the actual object address by subtracting the offset + curr_obj = context.object(obj_name, layer_name = layer_name, offset = netw_inside - ptr_offset) + yield curr_obj + + # if the same port is used on different interfaces multiple objects are created + # those can be found by following the pointer within the object's `Next` field until it is empty + while curr_obj.Next: + curr_obj = context.object(obj_name, + layer_name = layer_name, + offset = cls._decode_pointer(curr_obj.Next) - ptr_offset) + yield curr_obj + + @classmethod + def get_tcpip_module(cls, context: interfaces.context.ContextInterface, layer_name: str, + nt_symbols: str) -> Optional[interfaces.objects.ObjectInterface]: + """Uses `windows.modules` to find tcpip.sys in memory. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + nt_symbols: The name of the table containing the kernel symbols + + Returns: + The constructed tcpip.sys module object. + """ + for mod in modules.Modules.list_modules(context, layer_name, nt_symbols): + if mod.BaseDllName.get_string() == "tcpip.sys": + vollog.debug(f"Found tcpip.sys image base @ 0x{mod.DllBase:x}") + return mod + return None + + @classmethod + def parse_hashtable(cls, context: interfaces.context.ContextInterface, layer_name: str, ht_offset: int, + ht_length: int, alignment: int, + net_symbol_table: str) -> Generator[interfaces.objects.ObjectInterface, None, None]: + """Parses a hashtable quick and dirty. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + ht_offset: Beginning of the hash table + ht_length: Length of the hash table + + Returns: + The hash table entries which are _not_ empty + """ + # we are looking for entries whose values are not their own address + for index in range(ht_length): + current_addr = ht_offset + index * alignment + current_pointer = context.object(net_symbol_table + constants.BANG + "pointer", + layer_name = layer_name, + offset = current_addr) + # check if addr of pointer is equal to the value pointed to + if current_pointer.vol.offset == current_pointer: + continue + yield current_pointer + + @classmethod + def parse_partitions(cls, context: interfaces.context.ContextInterface, layer_name: str, net_symbol_table: str, + tcpip_symbol_table: str, + tcpip_module_offset: int) -> Iterable[interfaces.objects.ObjectInterface]: + """Parses tcpip.sys's PartitionTable containing established TCP connections. + The amount of Partition depends on the value of the symbol `PartitionCount` and correlates with + the maximum processor count (refer to Art of Memory Forensics, chapter 11). + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + net_symbol_table: The name of the table containing the tcpip types + tcpip_symbol_table: The name of the table containing the tcpip driver symbols + tcpip_module_offset: The offset of the tcpip module + + Returns: + The list of TCP endpoint objects from the `layer_name` layer's `PartitionTable` + """ + if symbols.symbol_table_is_64bit(context, net_symbol_table): + alignment = 0x10 + else: + alignment = 8 + + obj_name = net_symbol_table + constants.BANG + "_TCP_ENDPOINT" + # part_table_symbol is the offset within tcpip.sys which contains the address of the partition table itself + part_table_symbol = context.symbol_space.get_symbol(tcpip_symbol_table + constants.BANG + + "PartitionTable").address + part_count_symbol = context.symbol_space.get_symbol(tcpip_symbol_table + constants.BANG + + "PartitionCount").address + + part_table_addr = context.object(net_symbol_table + constants.BANG + "pointer", + layer_name = layer_name, + offset = tcpip_module_offset + part_table_symbol) + + # part_table is the actual partition table offset and consists out of a dynamic amount of _PARTITION objects + part_table = context.object(net_symbol_table + constants.BANG + "_PARTITION_TABLE", + layer_name = layer_name, + offset = part_table_addr) + part_count = int.from_bytes(context.layers[layer_name].read(tcpip_module_offset + part_count_symbol, 1), + "little") + part_table.Partitions.count = part_count + + vollog.debug("Found TCP connection PartitionTable @ 0x{:x} (partition count: {})".format( + part_table_addr, part_count)) + entry_offset = context.symbol_space.get_type(obj_name).relative_child_offset("ListEntry") + for ctr, partition in enumerate(part_table.Partitions): + vollog.debug(f"Parsing partition {ctr}") + if partition.Endpoints.NumEntries > 0: + for endpoint_entry in cls.parse_hashtable(context, layer_name, partition.Endpoints.Directory, + partition.Endpoints.TableSize, alignment, net_symbol_table): + + endpoint = context.object(obj_name, layer_name = layer_name, offset = endpoint_entry - entry_offset) + yield endpoint + + @classmethod + def create_tcpip_symbol_table(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, + tcpip_module_offset: int, tcpip_module_size: int) -> str: + """DEPRECATED: Use PDBUtility.symbol_table_from_pdb instead + + Creates symbol table for the current image's tcpip.sys driver. + + Searches the memory section of the loaded tcpip.sys module for its PDB GUID + and loads the associated symbol table into the symbol space. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + config_path: The config path where to find symbol files + layer_name: The name of the layer on which to operate + tcpip_module_offset: This memory dump's tcpip.sys image offset + tcpip_module_size: The size of `tcpip.sys` for this dump + + Returns: + The name of the constructed and loaded symbol table + """ + vollog.debug( + "Deprecation: This plugin uses netstat.create_tcpip_symbol_table instead of PDBUtility.symbol_table_from_pdb" + ) + return pdbutil.PDBUtility.symbol_table_from_pdb(context, + interfaces.configuration.path_join(config_path, 'tcpip'), + layer_name, "tcpip.pdb", tcpip_module_offset, tcpip_module_size) + + @classmethod + def find_port_pools(cls, context: interfaces.context.ContextInterface, layer_name: str, net_symbol_table: str, + tcpip_symbol_table: str, tcpip_module_offset: int) -> Tuple[int, int]: + """Finds the given image's port pools. Older Windows versions (presumably < Win10 build 14251) use driver + symbols called `UdpPortPool` and `TcpPortPool` which point towards the pools. + Newer Windows versions use `UdpCompartmentSet` and `TcpCompartmentSet`, which we first have to translate into + the port pool address. See also: http://redplait.blogspot.com/2016/06/tcpip-port-pools-in-fresh-windows-10.html + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + net_symbol_table: The name of the table containing the tcpip types + tcpip_module_offset: This memory dump's tcpip.sys image offset + tcpip_symbol_table: The name of the table containing the tcpip driver symbols + + Returns: + The tuple containing the address of the UDP and TCP port pool respectively. + """ + + if "UdpPortPool" in context.symbol_space[tcpip_symbol_table].symbols: + # older Windows versions + upp_symbol = context.symbol_space.get_symbol(tcpip_symbol_table + constants.BANG + "UdpPortPool").address + upp_addr = context.object(net_symbol_table + constants.BANG + "pointer", + layer_name = layer_name, + offset = tcpip_module_offset + upp_symbol) + + tpp_symbol = context.symbol_space.get_symbol(tcpip_symbol_table + constants.BANG + "TcpPortPool").address + tpp_addr = context.object(net_symbol_table + constants.BANG + "pointer", + layer_name = layer_name, + offset = tcpip_module_offset + tpp_symbol) + + elif "UdpCompartmentSet" in context.symbol_space[tcpip_symbol_table].symbols: + # newer Windows versions since 10.14xxx + ucs = context.symbol_space.get_symbol(tcpip_symbol_table + constants.BANG + "UdpCompartmentSet").address + tcs = context.symbol_space.get_symbol(tcpip_symbol_table + constants.BANG + "TcpCompartmentSet").address + + ucs_offset = context.object(net_symbol_table + constants.BANG + "pointer", + layer_name = layer_name, + offset = tcpip_module_offset + ucs) + tcs_offset = context.object(net_symbol_table + constants.BANG + "pointer", + layer_name = layer_name, + offset = tcpip_module_offset + tcs) + + ucs_obj = context.object(net_symbol_table + constants.BANG + "_INET_COMPARTMENT_SET", + layer_name = layer_name, + offset = ucs_offset) + upp_addr = ucs_obj.InetCompartment.ProtocolCompartment.PortPool + + tcs_obj = context.object(net_symbol_table + constants.BANG + "_INET_COMPARTMENT_SET", + layer_name = layer_name, + offset = tcs_offset) + tpp_addr = tcs_obj.InetCompartment.ProtocolCompartment.PortPool + + else: + # this branch should not be reached. + raise exceptions.SymbolError( + "UdpPortPool", tcpip_symbol_table, + f"Neither UdpPortPool nor UdpCompartmentSet found in {tcpip_symbol_table} table") + + vollog.debug(f"Found PortPools @ 0x{upp_addr:x} (UDP) && 0x{tpp_addr:x} (TCP)") + return upp_addr, tpp_addr + + @classmethod + def list_sockets(cls, + context: interfaces.context.ContextInterface, + layer_name: str, + nt_symbols: str, + net_symbol_table: str, + tcpip_module_offset: int, + tcpip_symbol_table: str) -> \ + Iterable[interfaces.objects.ObjectInterface]: + """Lists all UDP Endpoints, TCP Listeners and TCP Endpoints in the primary layer that + are in tcpip.sys's UdpPortPool, TcpPortPool and TCP Endpoint partition table, respectively. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + layer_name: The name of the layer on which to operate + nt_symbols: The name of the table containing the kernel symbols + net_symbol_table: The name of the table containing the tcpip types + tcpip_module_offset: Offset of `tcpip.sys`'s PE image in memory + tcpip_symbol_table: The name of the table containing the tcpip driver symbols + + Returns: + The list of network objects from the `layer_name` layer's `PartitionTable` and `PortPools` + """ + + # first, TCP endpoints by parsing the partition table + for endpoint in cls.parse_partitions(context, layer_name, net_symbol_table, tcpip_symbol_table, + tcpip_module_offset): + yield endpoint + + # then, towards the UDP and TCP port pools + # first, find their addresses + upp_addr, tpp_addr = cls.find_port_pools(context, layer_name, net_symbol_table, tcpip_symbol_table, + tcpip_module_offset) + + # create port pool objects at the detected address and parse the port bitmap + upp_obj = context.object(net_symbol_table + constants.BANG + "_INET_PORT_POOL", + layer_name = layer_name, + offset = upp_addr) + udpa_ports = cls.parse_bitmap(context, layer_name, upp_obj.PortBitMap.Buffer, + upp_obj.PortBitMap.SizeOfBitMap // 8) + + tpp_obj = context.object(net_symbol_table + constants.BANG + "_INET_PORT_POOL", + layer_name = layer_name, + offset = tpp_addr) + tcpl_ports = cls.parse_bitmap(context, layer_name, tpp_obj.PortBitMap.Buffer, + tpp_obj.PortBitMap.SizeOfBitMap // 8) + + vollog.debug(f"Found TCP Ports: {tcpl_ports}") + vollog.debug(f"Found UDP Ports: {udpa_ports}") + # given the list of TCP / UDP ports, calculate the address of their respective objects and yield them. + for port in tcpl_ports: + # port value can be 0, which we can skip + if not port: + continue + for obj in cls.enumerate_structures_by_port(context, layer_name, net_symbol_table, port, tpp_addr, "tcp"): + yield obj + + for port in udpa_ports: + # same as above, skip port 0 + if not port: + continue + for obj in cls.enumerate_structures_by_port(context, layer_name, net_symbol_table, port, upp_addr, "udp"): + yield obj + + def _generator(self, show_corrupt_results: Optional[bool] = None): + """ Generates the network objects for use in rendering. """ + + kernel = self.context.modules[self.config['kernel']] + + netscan_symbol_table = netscan.NetScan.create_netscan_symbol_table(self.context, + kernel.layer_name, + kernel.symbol_table_name, + self.config_path) + + tcpip_module = self.get_tcpip_module(self.context, kernel.layer_name, kernel.symbol_table_name) + + try: + tcpip_symbol_table = pdbutil.PDBUtility.symbol_table_from_pdb( + self.context, interfaces.configuration.path_join(self.config_path, 'tcpip'), + kernel.layer_name, "tcpip.pdb", tcpip_module.DllBase, tcpip_module.SizeOfImage) + except exceptions.VolatilityException: + vollog.warning("Unable to locate symbols for the memory image's tcpip module") + + for netw_obj in self.list_sockets(self.context, kernel.layer_name, kernel.symbol_table_name, + netscan_symbol_table, tcpip_module.DllBase, tcpip_symbol_table): + + # objects passed pool header constraints. check for additional constraints if strict flag is set. + if not show_corrupt_results and not netw_obj.is_valid(): + continue + + if isinstance(netw_obj, network._UDP_ENDPOINT): + vollog.debug(f"Found UDP_ENDPOINT @ 0x{netw_obj.vol.offset:2x}") + + # For UdpA, the state is always blank and the remote end is asterisks + for ver, laddr, _ in netw_obj.dual_stack_sockets(): + yield (0, (format_hints.Hex(netw_obj.vol.offset), "UDP" + ver, laddr, netw_obj.Port, "*", 0, "", + netw_obj.get_owner_pid() or renderers.UnreadableValue(), netw_obj.get_owner_procname() + or renderers.UnreadableValue(), netw_obj.get_create_time() + or renderers.UnreadableValue())) + + elif isinstance(netw_obj, network._TCP_ENDPOINT): + vollog.debug(f"Found _TCP_ENDPOINT @ 0x{netw_obj.vol.offset:2x}") + if netw_obj.get_address_family() == network.AF_INET: + proto = "TCPv4" + elif netw_obj.get_address_family() == network.AF_INET6: + proto = "TCPv6" + else: + vollog.debug("TCP Endpoint @ 0x{:2x} has unknown address family 0x{:x}".format( + netw_obj.vol.offset, netw_obj.get_address_family())) + proto = "TCPv?" + + try: + state = netw_obj.State.description + except ValueError: + state = renderers.UnreadableValue() + + yield (0, (format_hints.Hex(netw_obj.vol.offset), proto, netw_obj.get_local_address() + or renderers.UnreadableValue(), netw_obj.LocalPort, netw_obj.get_remote_address() + or renderers.UnreadableValue(), netw_obj.RemotePort, state, netw_obj.get_owner_pid() + or renderers.UnreadableValue(), netw_obj.get_owner_procname() or renderers.UnreadableValue(), + netw_obj.get_create_time() or renderers.UnreadableValue())) + + # check for isinstance of tcp listener last, because all other objects are inherited from here + elif isinstance(netw_obj, network._TCP_LISTENER): + vollog.debug(f"Found _TCP_LISTENER @ 0x{netw_obj.vol.offset:2x}") + + # For TcpL, the state is always listening and the remote port is zero + for ver, laddr, raddr in netw_obj.dual_stack_sockets(): + yield (0, (format_hints.Hex(netw_obj.vol.offset), "TCP" + ver, laddr, netw_obj.Port, raddr, 0, + "LISTENING", netw_obj.get_owner_pid() or renderers.UnreadableValue(), + netw_obj.get_owner_procname() or renderers.UnreadableValue(), netw_obj.get_create_time() + or renderers.UnreadableValue())) + else: + # this should not happen therefore we log it. + vollog.debug(f"Found network object unsure of its type: {netw_obj} of type {type(netw_obj)}") + + def generate_timeline(self): + for row in self._generator(): + _depth, row_data = row + row_dict = {} + row_dict["Offset"], row_dict["Proto"], row_dict["LocalAddr"], row_dict["LocalPort"], \ + row_dict["ForeignAddr"], row_dict["ForeignPort"], row_dict["State"], \ + row_dict["PID"], row_dict["Owner"], row_dict["Created"] = row_data + + # Skip network connections without creation time + if not isinstance(row_dict["Created"], datetime.datetime): + continue + description = "Network connection: Process {} {} Local Address {}:{} " \ + "Remote Address {}:{} State {} Protocol {} ".format(row_dict["PID"], row_dict["Owner"], + row_dict["LocalAddr"], + row_dict["LocalPort"], + row_dict["ForeignAddr"], + row_dict["ForeignPort"], + row_dict["State"], row_dict["Proto"]) + + yield (description, timeliner.TimeLinerType.CREATED, row_dict["Created"]) + + def run(self): + show_corrupt_results = self.config.get('include-corrupt', None) + + return renderers.TreeGrid([ + ("Offset", format_hints.Hex), + ("Proto", str), + ("LocalAddr", str), + ("LocalPort", int), + ("ForeignAddr", str), + ("ForeignPort", int), + ("State", str), + ("PID", int), + ("Owner", str), + ("Created", datetime.datetime), + ], self._generator(show_corrupt_results = show_corrupt_results)) diff --git a/volatility3/framework/plugins/windows/poolscanner.py b/volatility3/framework/plugins/windows/poolscanner.py index b2ea449ead..c5d60ce03d 100644 --- a/volatility3/framework/plugins/windows/poolscanner.py +++ b/volatility3/framework/plugins/windows/poolscanner.py @@ -18,9 +18,7 @@ vollog = logging.getLogger(__name__) -# TODO: When python3.5 is no longer supported, make this enum.IntFlag -# Revisit the page_type signature of PoolConstraint once using enum.IntFlag -class PoolType(enum.IntEnum): +class PoolType(enum.IntFlag): """Class to maintain the different possible PoolTypes The values must be integer powers of 2.""" @@ -37,11 +35,12 @@ def __init__(self, tag: bytes, type_name: str, object_type: Optional[str] = None, - page_type: Optional[int] = None, + page_type: Optional[PoolType] = None, size: Optional[Tuple[Optional[int], Optional[int]]] = None, index: Optional[Tuple[Optional[int], Optional[int]]] = None, alignment: Optional[int] = 1, - skip_type_test: bool = False) -> None: + skip_type_test: bool = False, + additional_structures: Optional[List[str]] = None) -> None: self.tag = tag self.type_name = type_name self.object_type = object_type @@ -50,6 +49,7 @@ def __init__(self, self.index = index self.alignment = alignment self.skip_type_test = skip_type_test + self.additional_structures = additional_structures class PoolHeaderScanner(interfaces.layers.ScannerInterface): @@ -115,24 +115,24 @@ class PoolScanner(plugins.PluginInterface): """A generic pool scanner plugin.""" _version = (1, 0, 0) - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'handles', plugin = handles.Handles, version = (1, 0, 0)), ] def _generator(self): - symbol_table = self.config["nt_symbols"] + kernel = self.context.modules[self.config['kernel']] + + symbol_table = kernel.symbol_table_name constraints = self.builtin_constraints(symbol_table) - for constraint, mem_object, header in self.generate_pool_scan(self.context, self.config["primary"], + for constraint, mem_object, header in self.generate_pool_scan(self.context, kernel.layer_name, symbol_table, constraints): # generate some type-specific info for sanity checking if constraint.object_type == "Process": @@ -143,7 +143,7 @@ def _generator(self): try: name = mem_object.FileName.String except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, "Skipping file at {0:#x}".format(mem_object.vol.offset)) + vollog.log(constants.LOGLEVEL_VVV, f"Skipping file at {mem_object.vol.offset:#x}") continue else: name = renderers.NotApplicableValue() @@ -214,7 +214,8 @@ def builtin_constraints(symbol_table: str, tags_filter: List[bytes] = None) -> L type_name = symbol_table + constants.BANG + "_DRIVER_OBJECT", object_type = "Driver", size = (248, None), - page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE), + page_type = PoolType.PAGED | PoolType.NONPAGED | PoolType.FREE, + additional_structures = ["_DRIVER_EXTENSION"]), # drivers on windows starting with windows 8 PoolConstraint(b'Driv', type_name = symbol_table + constants.BANG + "_DRIVER_OBJECT", @@ -293,26 +294,26 @@ def generate_pool_scan(cls, for constraint, header in cls.pool_scan(context, scan_layer, symbol_table, constraints, alignment = alignment): - mem_object = header.get_object(type_name = constraint.type_name, + mem_objects = header.get_object(constraint = constraint, use_top_down = is_windows_8_or_later, - executive = constraint.object_type is not None, - native_layer_name = 'primary', + native_layer_name = layer_name, kernel_symbol_table = symbol_table) - if mem_object is None: - vollog.log(constants.LOGLEVEL_VVV, "Cannot create an instance of {}".format(constraint.type_name)) - continue + for mem_object in mem_objects: + if mem_object is None: + vollog.log(constants.LOGLEVEL_VVV, f"Cannot create an instance of {constraint.type_name}") + continue - if constraint.object_type is not None and not constraint.skip_type_test: - try: - if mem_object.get_object_header().get_object_type(type_map, cookie) != constraint.object_type: + if constraint.object_type is not None and not constraint.skip_type_test: + try: + if mem_object.get_object_header().get_object_type(type_map, cookie) != constraint.object_type: + continue + except exceptions.InvalidAddressException: + vollog.log(constants.LOGLEVEL_VVV, + f"Cannot test instance type check for {constraint.type_name}") continue - except exceptions.InvalidAddressException: - vollog.log(constants.LOGLEVEL_VVV, - "Cannot test instance type check for {}".format(constraint.type_name)) - continue - yield constraint, mem_object, header + yield constraint, mem_object, header @classmethod def pool_scan(cls, @@ -340,10 +341,10 @@ def pool_scan(cls, An Iterable of pool constraints and the pool headers associated with them """ # Setup the pattern - constraint_lookup = {} # type: Dict[bytes, PoolConstraint] + constraint_lookup: Dict[bytes, PoolConstraint] = {} for constraint in pool_constraints: if constraint.tag in constraint_lookup: - raise ValueError("Constraint tag is used for more than one constraint: {}".format(repr(constraint.tag))) + raise ValueError(f"Constraint tag is used for more than one constraint: {repr(constraint.tag)}") constraint_lookup[constraint.tag] = constraint pool_header_table_name = cls.get_pool_header_table(context, symbol_table) diff --git a/volatility3/framework/plugins/windows/privileges.py b/volatility3/framework/plugins/windows/privileges.py index 8c652daa64..eaafbeae6e 100644 --- a/volatility3/framework/plugins/windows/privileges.py +++ b/volatility3/framework/plugins/windows/privileges.py @@ -16,8 +16,8 @@ class Privs(interfaces.plugins.PluginInterface): """Lists process token privileges""" - _version = (1, 0, 0) - _required_framework_version = (1, 0, 0) + _version = (1, 2, 0) + _required_framework_version = (2, 0, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -40,10 +40,8 @@ def __init__(self, *args, **kwargs): def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.ListRequirement(name = 'pid', description = 'Filter on specific process IDs', element_type = int, @@ -64,7 +62,7 @@ def _generator(self, procs): # Skip privileges whose bit positions cannot be # translated to a privilege name if not self.privilege_info.get(int(value)): - vollog.log(constants.LOGLEVEL_VVV, 'Skeep invalid privilege ({}).'.format(value)) + vollog.log(constants.LOGLEVEL_VVV, f'Skeep invalid privilege ({value}).') continue name, desc = self.privilege_info.get(int(value)) @@ -89,11 +87,12 @@ def _generator(self, procs): def run(self): filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) + kernel = self.context.modules[self.config['kernel']] return renderers.TreeGrid([("PID", int), ("Process", str), ("Value", int), ("Privilege", str), ("Attributes", str), ("Description", str)], self._generator( pslist.PsList.list_processes(context = self.context, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, filter_func = filter_func))) diff --git a/volatility3/framework/plugins/windows/pslist.py b/volatility3/framework/plugins/windows/pslist.py index f9fb20aaf8..cadccc5f1d 100644 --- a/volatility3/framework/plugins/windows/pslist.py +++ b/volatility3/framework/plugins/windows/pslist.py @@ -6,7 +6,7 @@ import logging from typing import Callable, Iterable, List, Type -from volatility3.framework import renderers, interfaces, layers, constants +from volatility3.framework import renderers, interfaces, layers, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.framework.objects import utility from volatility3.framework.renderers import format_hints @@ -20,17 +20,15 @@ class PsList(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Lists the processes present in a particular windows memory image.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (2, 0, 0) PHYSICAL_DEFAULT = False @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.BooleanRequirement(name = 'physical', description = 'Display physical offsets instead of virtual', default = cls.PHYSICAL_DEFAULT, @@ -73,22 +71,24 @@ def process_dump( dos_header = context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", offset = peb.ImageBaseAddress, layer_name = proc_layer_name) - file_handle = open_method("pid.{0}.{1:#x}.dmp".format(proc.UniqueProcessId, peb.ImageBaseAddress)) + file_handle = open_method(f"pid.{proc.UniqueProcessId}.{peb.ImageBaseAddress:#x}.dmp") for offset, data in dos_header.reconstruct(): file_handle.seek(offset) file_handle.write(data) except Exception as excp: - vollog.debug("Unable to dump PE with pid {}: {}".format(proc.UniqueProcessId, excp)) + vollog.debug(f"Unable to dump PE with pid {proc.UniqueProcessId}: {excp}") return file_handle @classmethod - def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[interfaces.objects.ObjectInterface], bool]: + def create_pid_filter(cls, pid_list: List[int] = None, exclude: bool = False) -> Callable[ + [interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process IDs. Args: pid_list: A list of process IDs that are acceptable, all other processes will be filtered out + exclude: Accept only tasks that are not in pid_list Returns: Filter function for passing to the `list_processes` method @@ -98,17 +98,21 @@ def create_pid_filter(cls, pid_list: List[int] = None) -> Callable[[interfaces.o pid_list = pid_list or [] filter_list = [x for x in pid_list if x is not None] if filter_list: - filter_func = lambda x: x.UniqueProcessId not in filter_list + if exclude: + filter_func = lambda x: x.UniqueProcessId in filter_list + else: + filter_func = lambda x: x.UniqueProcessId not in filter_list return filter_func @classmethod - def create_name_filter(cls, name_list: List[str] = None) -> Callable[[interfaces.objects.ObjectInterface], bool]: + def create_name_filter(cls, name_list: List[str] = None, exclude: bool = False) -> Callable[ + [interfaces.objects.ObjectInterface], bool]: """A factory for producing filter functions that filter based on a list of process names. Args: name_list: A list of process names that are acceptable, all other processes will be filtered out - + exclude: Accept only tasks that are not in name_list Returns: Filter function for passing to the `list_processes` method """ @@ -117,7 +121,10 @@ def create_name_filter(cls, name_list: List[str] = None) -> Callable[[interfaces name_list = name_list or [] filter_list = [x for x in name_list if x is not None] if filter_list: - filter_func = lambda x: utility.array_to_string(x.ImageFileName) not in filter_list + if exclude: + filter_func = lambda x: utility.array_to_string(x.ImageFileName) in filter_list + else: + filter_func = lambda x: utility.array_to_string(x.ImageFileName) not in filter_list return filter_func @classmethod @@ -166,19 +173,21 @@ def list_processes(cls, yield proc def _generator(self): + kernel = self.context.modules[self.config['kernel']] + pe_table_name = intermed.IntermediateSymbolTable.create(self.context, self.config_path, "windows", "pe", class_types = pe.class_types) - memory = self.context.layers[self.config['primary']] + memory = self.context.layers[kernel.layer_name] if not isinstance(memory, layers.intel.Intel): raise TypeError("Primary layer is not an intel layer") for proc in self.list_processes(self.context, - self.config['primary'], - self.config['nt_symbols'], + kernel.layer_name, + kernel.symbol_table_name, filter_func = self.create_pid_filter(self.config.get('pid', None))): if not self.config.get('physical', self.PHYSICAL_DEFAULT): @@ -187,22 +196,29 @@ def _generator(self): (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] file_output = "Disabled" - if self.config['dump']: - file_handle = self.process_dump(self.context, self.config['nt_symbols'], pe_table_name, proc, self.open) - file_output = "Error outputting file" - if file_handle: - file_handle.close() - file_output = str(file_handle.preferred_filename) - - yield (0, (proc.UniqueProcessId, proc.InheritedFromUniqueProcessId, - proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, errors = 'replace'), - format_hints.Hex(offset), proc.ActiveThreads, proc.get_handle_count(), proc.get_session_id(), - proc.get_is_wow64(), proc.get_create_time(), proc.get_exit_time(), file_output)) + + try: + if self.config['dump']: + file_handle = self.process_dump(self.context, kernel.symbol_table_name, + pe_table_name, proc, self.open) + file_output = "Error outputting file" + if file_handle: + file_handle.close() + file_output = str(file_handle.preferred_filename) + + yield (0, (proc.UniqueProcessId, proc.InheritedFromUniqueProcessId, + proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, + errors = 'replace'), + format_hints.Hex(offset), proc.ActiveThreads, proc.get_handle_count(), proc.get_session_id(), + proc.get_is_wow64(), proc.get_create_time(), proc.get_exit_time(), file_output)) + + except exceptions.InvalidAddressException: + vollog.info(f"Invalid process found at address: {proc.vol.offset:x}. Skipping") def generate_timeline(self): for row in self._generator(): _depth, row_data = row - description = "Process: {} {} ({})".format(row_data[0], row_data[2], row_data[3]) + description = f"Process: {row_data[0]} {row_data[2]} ({row_data[3]})" yield (description, timeliner.TimeLinerType.CREATED, row_data[8]) yield (description, timeliner.TimeLinerType.MODIFIED, row_data[9]) @@ -210,7 +226,7 @@ def run(self): offsettype = "(V)" if not self.config.get('physical', self.PHYSICAL_DEFAULT) else "(P)" return renderers.TreeGrid([("PID", int), ("PPID", int), ("ImageFileName", str), - ("Offset{0}".format(offsettype), format_hints.Hex), ("Threads", int), + (f"Offset{offsettype}", format_hints.Hex), ("Threads", int), ("Handles", int), ("SessionId", int), ("Wow64", bool), ("CreateTime", datetime.datetime), ("ExitTime", datetime.datetime), ("File output", str)], self._generator()) diff --git a/volatility3/framework/plugins/windows/psscan.py b/volatility3/framework/plugins/windows/psscan.py index a5e20ef62e..237c0edcd2 100644 --- a/volatility3/framework/plugins/windows/psscan.py +++ b/volatility3/framework/plugins/windows/psscan.py @@ -6,7 +6,7 @@ import logging from typing import Iterable, Callable, Tuple -from volatility3.framework import renderers, interfaces +from volatility3.framework import renderers, interfaces, layers, exceptions from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed @@ -22,16 +22,14 @@ class PsScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for processes present in a particular windows memory image.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 1, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.VersionRequirement(name = 'info', component = info.Info, version = (1, 0, 0)), requirements.ListRequirement(name = 'pid', @@ -41,6 +39,10 @@ def get_requirements(cls): requirements.BooleanRequirement(name = 'dump', description = "Extract listed processes", default = False, + optional = True), + requirements.BooleanRequirement(name = 'physical', + description = "Display physical offset instead of virtual", + default = False, optional = True) ] @@ -144,46 +146,62 @@ def get_osversion(cls, context: interfaces.context.ContextInterface, layer_name: return (nt_major_version, nt_minor_version, build) def _generator(self): + kernel = self.context.modules[self.config['kernel']] pe_table_name = intermed.IntermediateSymbolTable.create(self.context, self.config_path, "windows", "pe", class_types = pe.class_types) + memory = self.context.layers[kernel.layer_name] + if not isinstance(memory, layers.intel.Intel): + raise TypeError("Primary layer is not an intel layer") + for proc in self.scan_processes(self.context, - self.config['primary'], - self.config['nt_symbols'], + kernel.layer_name, + kernel.symbol_table_name, filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None))): file_output = "Disabled" if self.config['dump']: # windows 10 objects (maybe others in the future) are already in virtual memory - if proc.vol.layer_name == self.config['primary']: + if proc.vol.layer_name == kernel.layer_name: vproc = proc else: - vproc = self.virtual_process_from_physical(self.context, self.config['primary'], - self.config['nt_symbols'], proc) + vproc = self.virtual_process_from_physical(self.context, kernel.layer_name, + kernel.symbol_table_name, proc) - file_handle = pslist.PsList.process_dump(self.context, self.config['nt_symbols'], pe_table_name, vproc, + file_handle = pslist.PsList.process_dump(self.context, kernel.symbol_table_name, + pe_table_name, vproc, self.open) file_output = "Error outputting file" if file_handle: file_output = file_handle.preferred_filename - yield (0, (proc.UniqueProcessId, proc.InheritedFromUniqueProcessId, + if not self.config['physical']: + offset = proc.vol.offset + else: + (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] + + try: + yield (0, (proc.UniqueProcessId, proc.InheritedFromUniqueProcessId, proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, - errors = 'replace'), format_hints.Hex(proc.vol.offset), + errors = 'replace'), format_hints.Hex(offset), proc.ActiveThreads, proc.get_handle_count(), proc.get_session_id(), proc.get_is_wow64(), proc.get_create_time(), proc.get_exit_time(), file_output)) + except exceptions.InvalidAddressException: + vollog.info(f"Invalid process found at address: {proc.vol.offset:x}. Skipping") def generate_timeline(self): for row in self._generator(): _depth, row_data = row - description = "Process: {} {} ({})".format(row_data[0], row_data[2], row_data[3]) + description = f"Process: {row_data[0]} {row_data[2]} ({row_data[3]})" yield (description, timeliner.TimeLinerType.CREATED, row_data[8]) yield (description, timeliner.TimeLinerType.MODIFIED, row_data[9]) def run(self): - return renderers.TreeGrid([("PID", int), ("PPID", int), ("ImageFileName", str), ("Offset", format_hints.Hex), - ("Threads", int), ("Handles", int), ("SessionId", int), ("Wow64", bool), + offsettype = "(V)" if not self.config['physical'] else "(P)" + return renderers.TreeGrid([("PID", int), ("PPID", int), ("ImageFileName", str), + (f"Offset{offsettype}", format_hints.Hex), ("Threads", int), + ("Handles", int), ("SessionId", int), ("Wow64", bool), ("CreateTime", datetime.datetime), ("ExitTime", datetime.datetime), ("File output", str)], self._generator()) diff --git a/volatility3/framework/plugins/windows/pstree.py b/volatility3/framework/plugins/windows/pstree.py index 3b695ea70e..fbb883839f 100644 --- a/volatility3/framework/plugins/windows/pstree.py +++ b/volatility3/framework/plugins/windows/pstree.py @@ -2,33 +2,33 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import datetime -from typing import Dict, Set +import logging +from typing import Dict, Set, Tuple from volatility3.framework import objects, interfaces, renderers from volatility3.framework.configuration import requirements from volatility3.framework.renderers import format_hints from volatility3.plugins.windows import pslist +vollog = logging.getLogger(__name__) class PsTree(interfaces.plugins.PluginInterface): """Plugin for listing processes in a tree based on their parent process ID.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) - self._processes = {} # type: Dict[int, interfaces.objects.ObjectInterface] - self._levels = {} # type: Dict[int, int] - self._children = {} # type: Dict[int, Set[int]] + self._processes: Dict[int, Tuple[interfaces.objects.ObjectInterface, int]] = {} + self._levels: Dict[int, int] = {} + self._children: Dict[int, Set[int]] = {} @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.BooleanRequirement(name = 'physical', description = 'Display physical offsets instead of virtual', default = pslist.PsList.PHYSICAL_DEFAULT, @@ -45,34 +45,42 @@ def find_level(self, pid: objects.Pointer) -> None: seen = set([]) seen.add(pid) level = 0 - proc = self._processes.get(pid, None) + proc, _ = self._processes.get(pid, None) while proc is not None and proc.InheritedFromUniqueProcessId not in seen: child_list = self._children.get(proc.InheritedFromUniqueProcessId, set([])) child_list.add(proc.UniqueProcessId) self._children[proc.InheritedFromUniqueProcessId] = child_list - proc = self._processes.get(proc.InheritedFromUniqueProcessId, None) + seen.add(proc.InheritedFromUniqueProcessId) + proc, _ = self._processes.get(proc.InheritedFromUniqueProcessId, (None, None)) level += 1 self._levels[pid] = level def _generator(self): """Generates the Tree of processes.""" - for proc in pslist.PsList.list_processes(self.context, self.config['primary'], self.config['nt_symbols']): + kernel = self.context.modules[self.config['kernel']] + for proc in pslist.PsList.list_processes(self.context, kernel.layer_name, + kernel.symbol_table_name): if not self.config.get('physical', pslist.PsList.PHYSICAL_DEFAULT): offset = proc.vol.offset else: - layer_name = self.config['primary'] + layer_name = kernel.layer_name memory = self.context.layers[layer_name] (_, _, offset, _, _) = list(memory.mapping(offset = proc.vol.offset, length = 0))[0] - self._processes[proc.UniqueProcessId] = proc + self._processes[proc.UniqueProcessId] = proc, offset # Build the child/level maps for pid in self._processes: self.find_level(pid) + process_pids = set([]) def yield_processes(pid): - proc = self._processes[pid] + if pid in process_pids: + vollog.debug(f"Pid cycle: already processed pid {pid}") + return + process_pids.add(pid) + proc, offset = self._processes[pid] row = (proc.UniqueProcessId, proc.InheritedFromUniqueProcessId, proc.ImageFileName.cast("string", max_length = proc.ImageFileName.vol.count, errors = 'replace'), format_hints.Hex(offset), proc.ActiveThreads, proc.get_handle_count(), proc.get_session_id(), @@ -90,7 +98,7 @@ def run(self): offsettype = "(V)" if not self.config.get('physical', pslist.PsList.PHYSICAL_DEFAULT) else "(P)" return renderers.TreeGrid([("PID", int), ("PPID", int), ("ImageFileName", str), - ("Offset{0}".format(offsettype), format_hints.Hex), ("Threads", int), + (f"Offset{offsettype}", format_hints.Hex), ("Threads", int), ("Handles", int), ("SessionId", int), ("Wow64", bool), ("CreateTime", datetime.datetime), ("ExitTime", datetime.datetime)], self._generator()) diff --git a/volatility3/framework/plugins/windows/registry/hivelist.py b/volatility3/framework/plugins/windows/registry/hivelist.py index 249118c8ed..ac30560c6e 100644 --- a/volatility3/framework/plugins/windows/registry/hivelist.py +++ b/volatility3/framework/plugins/windows/registry/hivelist.py @@ -17,7 +17,7 @@ class HiveGenerator: """Walks the registry HiveList linked list in a given direction and stores an invalid offset if it's unable to fully walk the list""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) def __init__(self, cmhive, forward = True): self._cmhive = cmhive @@ -40,15 +40,13 @@ class HiveList(interfaces.plugins.PluginInterface): """Lists the registry hives present in a particular memory image.""" _version = (1, 0, 0) - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.StringRequirement(name = 'filter', description = "String to filter hive names returned", optional = True, @@ -65,10 +63,11 @@ def _sanitize_hive_name(self, name: str) -> str: def _generator(self) -> Iterator[Tuple[int, Tuple[int, str]]]: chunk_size = 0x500000 + kernel = self.context.modules[self.config['kernel']] for hive_object in self.list_hive_objects(context = self.context, - layer_name = self.config["primary"], - symbol_table = self.config["nt_symbols"], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, filter_string = self.config.get('filter', None)): file_output = "Disabled" @@ -77,13 +76,13 @@ def _generator(self) -> Iterator[Tuple[int, Tuple[int, str]]]: hive = next( self.list_hives(self.context, self.config_path, - layer_name = self.config["primary"], - symbol_table = self.config["nt_symbols"], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, hive_offsets = [hive_object.vol.offset])) maxaddr = hive.hive.Storage[0].Length hive_name = self._sanitize_hive_name(hive.get_name()) - file_handle = self.open('registry.{}.{}.hive'.format(hive_name, hex(hive.hive_offset))) + file_handle = self.open(f'registry.{hive_name}.{hex(hive.hive_offset)}.hive') with file_handle as file_data: if hive._base_block: hive_data = self.context.layers[hive.dependencies[0]].read(hive.hive.BaseBlock, 1 << 12) @@ -143,7 +142,7 @@ def list_hives(cls, try: hive = registry.RegistryHive(context, reg_config_path, name = 'hive' + hex(hive_offset)) except exceptions.InvalidAddressException: - vollog.warning("Couldn't create RegistryHive layer at offset {}, skipping".format(hex(hive_offset))) + vollog.warning(f"Couldn't create RegistryHive layer at offset {hex(hive_offset)}, skipping") continue context.layers.add_layer(hive) yield hive diff --git a/volatility3/framework/plugins/windows/registry/hivescan.py b/volatility3/framework/plugins/windows/registry/hivescan.py index ac04fcc025..ab15e56acd 100644 --- a/volatility3/framework/plugins/windows/registry/hivescan.py +++ b/volatility3/framework/plugins/windows/registry/hivescan.py @@ -15,16 +15,14 @@ class HiveScan(interfaces.plugins.PluginInterface): """Scans for registry hives present in a particular windows memory image.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'poolscanner', plugin = poolscanner.PoolScanner, version = (1, 0, 0)), requirements.PluginRequirement(name = 'bigpools', plugin = bigpools.BigPools, version = (1, 0, 0)), ] @@ -68,7 +66,9 @@ def scan_hives(cls, yield mem_object def _generator(self): - for hive in self.scan_hives(self.context, self.config['primary'], self.config['nt_symbols']): + kernel = self.context.modules[self.config['kernel']] + + for hive in self.scan_hives(self.context, kernel.layer_name, kernel.symbol_table_name): yield (0, (format_hints.Hex(hive.vol.offset), )) diff --git a/volatility3/framework/plugins/windows/registry/printkey.py b/volatility3/framework/plugins/windows/registry/printkey.py index 5b0a909fc4..2082c339e5 100644 --- a/volatility3/framework/plugins/windows/registry/printkey.py +++ b/volatility3/framework/plugins/windows/registry/printkey.py @@ -19,16 +19,14 @@ class PrintKey(interfaces.plugins.PluginInterface): """Lists the registry keys under a hive or specific key value.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)), requirements.IntRequirement(name = 'offset', description = "Hive Offset", default = None, optional = True), requirements.StringRequirement(name = 'key', @@ -123,23 +121,23 @@ def _printkey_iterator(self, value_node_name = renderers.UnreadableValue() try: - value_type = RegValueTypes.get(node.Type).name + value_type = RegValueTypes(node.Type).name except (exceptions.InvalidAddressException, RegistryFormatException) as excp: vollog.debug(excp) value_type = renderers.UnreadableValue() if isinstance(value_type, renderers.UnreadableValue): vollog.debug("Couldn't read registry value type, so data is unreadable") - value_data = renderers.UnreadableValue() # type: Union[interfaces.renderers.BaseAbsentValue, bytes] + value_data: Union[interfaces.renderers.BaseAbsentValue, bytes] = renderers.UnreadableValue() else: try: value_data = node.decode_data() if isinstance(value_data, int): value_data = format_hints.MultiTypeData(value_data, encoding = 'utf-8') - elif RegValueTypes.get(node.Type) == RegValueTypes.REG_BINARY: + elif RegValueTypes(node.Type) == RegValueTypes.REG_BINARY: value_data = format_hints.MultiTypeData(value_data, show_hex = True) - elif RegValueTypes.get(node.Type) == RegValueTypes.REG_MULTI_SZ: + elif RegValueTypes(node.Type) == RegValueTypes.REG_MULTI_SZ: value_data = format_hints.MultiTypeData(value_data, encoding = 'utf-16-le', split_nulls = True) @@ -176,11 +174,11 @@ def _registry_walker(self, yield (x - len(node_path), y) except (exceptions.InvalidAddressException, KeyError, RegistryFormatException) as excp: if isinstance(excp, KeyError): - vollog.debug("Key '{}' not found in Hive at offset {}.".format(key, hex(hive.hive_offset))) + vollog.debug(f"Key '{key}' not found in Hive at offset {hex(hive.hive_offset)}.") elif isinstance(excp, RegistryFormatException): vollog.debug(excp) elif isinstance(excp, exceptions.InvalidAddressException): - vollog.debug("Invalid address identified in Hive: {}".format(hex(excp.invalid_address))) + vollog.debug(f"Invalid address identified in Hive: {hex(excp.invalid_address)}") result = (0, (renderers.UnreadableValue(), format_hints.Hex(hive.hive_offset), "Key", '?\\' + (key or ''), renderers.UnreadableValue(), renderers.UnreadableValue(), renderers.UnreadableValue())) @@ -188,12 +186,13 @@ def _registry_walker(self, def run(self): offset = self.config.get('offset', None) + kernel = self.context.modules[self.config['kernel']] return TreeGrid(columns = [('Last Write Time', datetime.datetime), ('Hive Offset', format_hints.Hex), ('Type', str), ('Key', str), ('Name', str), ('Data', format_hints.MultiTypeData), ('Volatile', bool)], - generator = self._registry_walker(self.config['primary'], - self.config['nt_symbols'], + generator = self._registry_walker(kernel.layer_name, + kernel.symbol_table_name, hive_offsets = None if offset is None else [offset], key = self.config.get('key', None), recurse = self.config.get('recurse', None))) diff --git a/volatility3/framework/plugins/windows/registry/userassist.py b/volatility3/framework/plugins/windows/registry/userassist.py index b2ce971bff..a788f058f5 100644 --- a/volatility3/framework/plugins/windows/registry/userassist.py +++ b/volatility3/framework/plugins/windows/registry/userassist.py @@ -23,7 +23,7 @@ class UserAssist(interfaces.plugins.PluginInterface): """Print userassist registry keys and information.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -37,10 +37,8 @@ def __init__(self, *args, **kwargs): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.IntRequirement(name = 'offset', description = "Hive Offset", default = None, optional = True), requirements.PluginRequirement(name = 'hivelist', plugin = hivelist.HiveList, version = (1, 0, 0)) ] @@ -115,13 +113,17 @@ def _determine_userassist_type(self) -> None: def _win7_or_later(self) -> bool: # TODO: change this if there is a better way of determining the OS version # _KUSER_SHARED_DATA.CookiePad is in Windows 6.1 (Win7) and later - return self.context.symbol_space.get_type(self.config['nt_symbols'] + constants.BANG + + kernel = self.context.modules[self.config['kernel']] + + return self.context.symbol_space.get_type(kernel.symbol_table_name + constants.BANG + "_KUSER_SHARED_DATA").has_member('CookiePad') def list_userassist(self, hive: RegistryHive) -> Generator[Tuple[int, Tuple], None, None]: """Generate userassist data for a registry hive.""" - hive_name = hive.hive.cast(self.config["nt_symbols"] + constants.BANG + "_CMHIVE").get_name() + kernel = self.context.modules[self.config['kernel']] + + hive_name = hive.hive.cast(kernel.symbol_table_name + constants.BANG + "_CMHIVE").get_name() if self._win7 is None: try: @@ -151,12 +153,12 @@ def list_userassist(self, hive: RegistryHive) -> Generator[Tuple[int, Tuple], No countkey_last_write_time = conversion.wintime_to_datetime(countkey.LastWriteTime.QuadPart) # output the parent Count key - result = ( + result: Tuple[int, Tuple[format_hints.Hex, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] = ( 0, (renderers.format_hints.Hex(hive.hive_offset), hive_name, countkey_path, countkey_last_write_time, "Key", renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue(), renderers.NotApplicableValue()) - ) # type: Tuple[int, Tuple[format_hints.Hex, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any]] + ) yield result # output any subkeys under Count @@ -215,19 +217,20 @@ def _generator(self): hive_offsets = None if self.config.get('offset', None) is not None: hive_offsets = [self.config.get('offset', None)] + kernel = self.context.modules[self.config['kernel']] # get all the user hive offsets or use the one specified for hive in hivelist.HiveList.list_hives(context = self.context, base_config_path = self.config_path, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, filter_string = 'ntuser.dat', hive_offsets = hive_offsets): try: yield from self.list_userassist(hive) continue except exceptions.PagedInvalidAddressException as excp: - vollog.debug("Invalid address identified in Hive: {}".format(hex(excp.invalid_address))) + vollog.debug(f"Invalid address identified in Hive: {hex(excp.invalid_address)}") except exceptions.InvalidAddressException as excp: vollog.debug("Invalid address identified in lower layer {}: {}".format( excp.layer_name, excp.invalid_address)) diff --git a/volatility3/framework/plugins/windows/skeleton_key_check.py b/volatility3/framework/plugins/windows/skeleton_key_check.py new file mode 100644 index 0000000000..cd4a5baecf --- /dev/null +++ b/volatility3/framework/plugins/windows/skeleton_key_check.py @@ -0,0 +1,602 @@ +# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +# This module attempts to locate skeleton-key like function hooks. +# It does this by locating the CSystems array through a variety of methods, +# and then validating the entry for RC4 HMAC (0x17 / 23) +# +# For a thorough walkthrough on how the R&D was performed to develop this plugin, +# please see our blogpost here: +# +# + +import io +import logging +from typing import Iterable, Tuple, List, Optional + +import pefile + +from volatility3.framework import interfaces, symbols, exceptions +from volatility3.framework import renderers, constants +from volatility3.framework.configuration import requirements +from volatility3.framework.layers import scanners +from volatility3.framework.objects import utility +from volatility3.framework.renderers import format_hints +from volatility3.framework.symbols import intermed +from volatility3.framework.symbols.windows import pdbutil +from volatility3.framework.symbols.windows.extensions import pe +from volatility3.plugins.windows import pslist, vadinfo + +try: + import capstone + + has_capstone = True +except ImportError: + has_capstone = False + +vollog = logging.getLogger(__name__) + + +class Skeleton_Key_Check(interfaces.plugins.PluginInterface): + """ Looks for signs of Skeleton Key malware """ + + _required_framework_version = (2, 0, 0) + + @classmethod + def get_requirements(cls): + # Since we're calling the plugin, make sure we have the plugin's requirements + return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), + requirements.VersionRequirement(name = 'pslist', component = pslist.PsList, version = (2, 0, 0)), + requirements.VersionRequirement(name = 'vadinfo', component = vadinfo.VadInfo, version = (2, 0, 0)), + requirements.VersionRequirement(name = 'pdbutil', component = pdbutil.PDBUtility, version = (1, 0, 0)), + ] + + def _get_pefile_obj(self, pe_table_name: str, layer_name: str, base_address: int) -> pefile.PE: + """ + Attempts to pefile object from the bytes of the PE file + + Args: + pe_table_name: name of the pe types table + layer_name: name of the lsass.exe process layer + base_address: base address of cryptdll.dll in lsass.exe + + Returns: + the constructed pefile object + """ + pe_data = io.BytesIO() + + try: + dos_header = self.context.object(pe_table_name + constants.BANG + "_IMAGE_DOS_HEADER", + offset = base_address, + layer_name = layer_name) + + for offset, data in dos_header.reconstruct(): + pe_data.seek(offset) + pe_data.write(data) + + pe_ret = pefile.PE(data = pe_data.getvalue(), fast_load = True) + + except exceptions.InvalidAddressException: + vollog.debug("Unable to reconstruct cryptdll.dll in memory") + pe_ret = None + + return pe_ret + + def _check_for_skeleton_key_vad(self, csystem: interfaces.objects.ObjectInterface, + cryptdll_base: int, + cryptdll_size: int) -> bool: + """ + Checks if Initialize and/or Decrypt is hooked by determining if + these function pointers reference addresses inside of the cryptdll VAD + + Args: + csystem: The RC4HMAC KERB_ECRYPT instance + cryptdll_base: Base address of the cryptdll.dll VAD + cryptdll_size: Size of the VAD + Returns: + bool: if a skeleton key hook is present + """ + return not ((cryptdll_base <= csystem.Initialize <= cryptdll_base + cryptdll_size) and \ + (cryptdll_base <= csystem.Decrypt <= cryptdll_base + cryptdll_size)) + + def _check_for_skeleton_key_symbols(self, csystem: interfaces.objects.ObjectInterface, + rc4HmacInitialize: int, + rc4HmacDecrypt: int) -> bool: + """ + Uses the PDB information to specifically check if the csystem for RC4HMAC + has an initialization pointer to rc4HmacInitialize and a decryption pointer + to rc4HmacDecrypt. + + Args: + csystem: The RC4HMAC KERB_ECRYPT instance + rc4HmacInitialize: The expected address of csystem Initialization function + rc4HmacDecrypt: The expected address of the csystem Decryption function + + Returns: + bool: if a skeleton key hook was found + """ + return csystem.Initialize != rc4HmacInitialize or csystem.Decrypt != rc4HmacDecrypt + + def _construct_ecrypt_array(self, array_start: int, count: int, \ + cryptdll_types: interfaces.context.ModuleInterface) -> interfaces.context.ModuleInterface: + """ + Attempts to construct an array of _KERB_ECRYPT structures + + Args: + array_start: starting virtual address of the array + count: how many elements are in the array + cryptdll_types: the reverse engineered types + + Returns: + The instantiated array + """ + + try: + array = cryptdll_types.object(object_type = "array", + offset = array_start, + subtype = cryptdll_types.get_type("_KERB_ECRYPT"), + count = count, + absolute = True) + + except exceptions.InvalidAddressException: + vollog.debug("Unable to construct cSystems array at given offset: {:x}".format(array_start)) + array = None + + return array + + def _find_array_with_pdb_symbols(self, cryptdll_symbols: str, + cryptdll_types: interfaces.context.ModuleInterface, + proc_layer_name: str, + cryptdll_base: int) -> Tuple[interfaces.objects.ObjectInterface, int, int, int]: + + """ + Finds the CSystems array through use of PDB symbols + + Args: + cryptdll_symbols: The symbols table from the PDB file + cryptdll_types: The types from cryptdll binary analysis + proc_layer_name: The lsass.exe process layer name + cryptdll_base: Base address of cryptdll.dll inside of lsass.exe + + Returns: + Tuple of: + array: The cSystems array + rc4HmacInitialize: The runtime address of the expected initialization function + rc4HmacDecrypt: The runtime address of the expected decryption function + """ + cryptdll_module = self.context.module(cryptdll_symbols, layer_name = proc_layer_name, offset = cryptdll_base) + + rc4HmacInitialize = cryptdll_module.get_absolute_symbol_address("rc4HmacInitialize") + + rc4HmacDecrypt = cryptdll_module.get_absolute_symbol_address("rc4HmacDecrypt") + + count_address = cryptdll_module.get_symbol("cCSystems").address + + # we do not want to fail just because the count is not in memory + # 16 was the size on samples I tested, so I chose it as the default + try: + count = cryptdll_types.object(object_type = "unsigned long", offset = count_address) + except exceptions.InvalidAddressException: + count = 16 + + array_start = cryptdll_module.get_absolute_symbol_address("CSystems") + + array = self._construct_ecrypt_array(array_start, count, cryptdll_types) + + if array is None: + vollog.debug("The CSystem array is not present in memory. Stopping PDB based analysis.") + + return array, rc4HmacInitialize, rc4HmacDecrypt + + def _get_cryptdll_types(self, context: interfaces.context.ContextInterface, + config, + config_path: str, + proc_layer_name: str, + cryptdll_base: int): + """ + Builds a symbol table from the cryptdll types generated after binary analysis + + Args: + context: the context to operate upon + config: + config_path: + proc_layer_name: name of the lsass.exe process layer + cryptdll_base: base address of cryptdll.dll inside of lsass.exe + """ + kernel = self.context.modules[self.config['kernel']] + table_mapping = {"nt_symbols": kernel.symbol_table_name} + + cryptdll_symbol_table = intermed.IntermediateSymbolTable.create(context = context, + config_path = config_path, + sub_path = "windows", + filename = "kerb_ecrypt", + table_mapping = table_mapping) + + return context.module(cryptdll_symbol_table, proc_layer_name, offset = cryptdll_base) + + def _find_lsass_proc(self, proc_list: Iterable) -> \ + Tuple[interfaces.context.ContextInterface, str]: + """ + Walks the process list and returns the first valid lsass instances. + There should be only one lsass process, but malware will often use the + process name to try and blend in. + + Args: + proc_list: The process list generator + + Return: + The process object for lsass + """ + + for proc in proc_list: + try: + proc_id = proc.UniqueProcessId + proc_layer_name = proc.add_process_layer() + + return proc, proc_layer_name + + except exceptions.InvalidAddressException as excp: + vollog.debug("Process {}: invalid address {} in layer {}".format(proc_id, excp.invalid_address, + excp.layer_name)) + + return None, None + + def _find_cryptdll(self, lsass_proc: interfaces.context.ContextInterface) -> \ + Tuple[int, int]: + """ + Finds the base address of cryptdll.dll inside of lsass.exe + + Args: + lsass_proc: the process object for lsass.exe + + Returns: + A tuple of: + cryptdll_base: the base address of cryptdll.dll + crytpdll_size: the size of the VAD for cryptdll.dll + """ + for vad in lsass_proc.get_vad_root().traverse(): + filename = vad.get_file_name() + + if isinstance(filename, str) and filename.lower().endswith("cryptdll.dll"): + base = vad.get_start() + return base, vad.get_end() - base + + return None, None + + def _find_csystems_with_symbols(self, proc_layer_name: str, + cryptdll_types: interfaces.context.ModuleInterface, + cryptdll_base: int, + cryptdll_size: int) -> \ + Tuple[interfaces.objects.ObjectInterface, int, int]: + """ + Attempts to find CSystems and the expected address of the handlers. + Relies on downloading and parsing of the cryptdll PDB file. + + Args: + proc_layer_name: the name of the lsass.exe process layer + cryptdll_types: The types from cryptdll binary analysis + cryptdll_base: the base address of cryptdll.dll + crytpdll_size: the size of the VAD for cryptdll.dll + + Returns: + A tuple of: + array: An initialized Volatility array of _KERB_ECRYPT structures + rc4HmacInitialize: The expected address of csystem Initialization function + rc4HmacDecrypt: The expected address of the csystem Decryption function + """ + try: + cryptdll_symbols = pdbutil.PDBUtility.symbol_table_from_pdb(self.context, + interfaces.configuration.path_join( + self.config_path, 'cryptdll'), + proc_layer_name, + "cryptdll.pdb", + cryptdll_base, + cryptdll_size) + except exceptions.VolatilityException: + vollog.debug("Unable to use the cryptdll PDB. Stopping PDB symbols based analysis.") + return None, None, None + + array, rc4HmacInitialize, rc4HmacDecrypt = \ + self._find_array_with_pdb_symbols(cryptdll_symbols, cryptdll_types, proc_layer_name, cryptdll_base) + + if array is None: + vollog.debug("The CSystem array is not present in memory. Stopping PDB symbols based analysis.") + + return array, rc4HmacInitialize, rc4HmacDecrypt + + def _get_rip_relative_target(self, inst) -> int: + """ + Returns the target address of a RIP-relative instruction. + + These instructions contain the offset of a target address + relative to the current instruction pointer. + + Args: + inst: A capstone instruction instance + + Returns: + None or the target address of the instruction + """ + try: + opnd = inst.operands[1] + except capstone.CsError: + return None + + if opnd.type != capstone.x86.X86_OP_MEM: + return None + + if inst.reg_name(opnd.mem.base) != "rip": + return None + + return inst.address + inst.size + opnd.mem.disp + + def _analyze_cdlocatecsystem(self, function_bytes: bytes, + function_start: int, + cryptdll_types: interfaces.context.ModuleInterface, + proc_layer_name: str) -> Optional[interfaces.objects.ObjectInterface]: + """ + Performs static analysis on CDLocateCSystem to find the instructions that + reference CSystems as well as cCsystems + + Args: + function_bytes: the instruction bytes of CDLocateCSystem + function_start: the address of CDLocateCSystem + proc_layer_name: the name of the lsass.exe process layer + + Return: + The cSystems array of ecrypt instances + """ + found_count = False + array_start = None + count = None + + ## we only support 64bit disassembly analysis + md = capstone.Cs(capstone.CS_ARCH_X86, capstone.CS_MODE_64) + md.detail = True + + for inst in md.disasm(function_bytes, function_start): + # we should not reach debug traps + if inst.mnemonic == "int3": + break + + # cCsystems is referenced by a mov instruction + elif inst.mnemonic == "mov": + if not found_count: + target_address = self._get_rip_relative_target(inst) + + # we do not want to fail just because the count is not in memory + # 16 was the size on samples I tested, so I chose it as the default + if target_address: + count = int.from_bytes(self.context.layers[proc_layer_name].read(target_address, 4), "little") + else: + count = 16 + + found_count = True + + elif inst.mnemonic == "lea": + target_address = self._get_rip_relative_target(inst) + + if target_address: + array_start = target_address + + # we find the count before, so we can terminate the static analysis here + break + + if array_start and count: + array = self._construct_ecrypt_array(array_start, count, cryptdll_types) + else: + array = None + + return array + + def _find_csystems_with_export(self, proc_layer_name: str, + cryptdll_types: interfaces.context.ModuleInterface, + cryptdll_base: int, + _) -> Optional[interfaces.objects.ObjectInterface]: + """ + Uses export table analysis to locate CDLocateCsystem + This function references CSystems and cCsystems + + Args: + proc_layer_name: The lsass.exe process layer name + cryptdll_types: The types from cryptdll binary analysis + cryptdll_base: Base address of cryptdll.dll inside of lsass.exe + _: unused in this source + Returns: + The cSystems array + """ + + if not has_capstone: + vollog.debug("capstone is not installed so cannot fall back to export table analysis.") + return None + + vollog.debug("Unable to perform analysis using PDB symbols, falling back to export table analysis.") + + pe_table_name = intermed.IntermediateSymbolTable.create(self.context, + self.config_path, + "windows", + "pe", + class_types = pe.class_types) + + cryptdll = self._get_pefile_obj(pe_table_name, proc_layer_name, cryptdll_base) + if not cryptdll: + return None + + cryptdll.parse_data_directories(directories = [pefile.DIRECTORY_ENTRY["IMAGE_DIRECTORY_ENTRY_EXPORT"]]) + if not hasattr(cryptdll, 'DIRECTORY_ENTRY_EXPORT'): + return None + + # find the location of CDLocateCSystem and then perform static analysis + for export in cryptdll.DIRECTORY_ENTRY_EXPORT.symbols: + if export.name != b"CDLocateCSystem": + continue + + function_start = cryptdll_base + export.address + + try: + function_bytes = self.context.layers[proc_layer_name].read(function_start, 0x50) + except exceptions.InvalidAddressException: + vollog.debug( + "The CDLocateCSystem function is not present in the lsass address space. Stopping export based analysis.") + break + + array = self._analyze_cdlocatecsystem(function_bytes, function_start, cryptdll_types, proc_layer_name) + if array is None: + vollog.debug("The CSystem array is not present in memory. Stopping export based analysis.") + + return array + + return None + + def _find_csystems_with_scanning(self, proc_layer_name: str, + cryptdll_types: interfaces.context.ModuleInterface, + cryptdll_base: int, + cryptdll_size: int) -> List[interfaces.context.ModuleInterface]: + """ + Performs scanning to find potential RC4 HMAC csystem instances + + This function may return several values as it cannot validate which is the active one + + Args: + proc_layer_name: the lsass.exe process layer name + cryptdll_types: the types from cryptdll binary analysis + cryptdll_base: base address of cryptdll.dll inside of lsass.exe + cryptdll_size: size of the VAD + Returns: + A list of csystem instances + """ + + csystems = [] + + cryptdll_end = cryptdll_base + cryptdll_size + + proc_layer = self.context.layers[proc_layer_name] + + ecrypt_size = cryptdll_types.get_type("_KERB_ECRYPT").size + + # scan for potential instances of RC4 HMAC + # the signature is based on the type being 0x17 + # and the block size member being 1 in all test samples + for address in proc_layer.scan(self.context, + scanners.BytesScanner(b"\x17\x00\x00\x00\x01\x00\x00\x00"), + sections = [(cryptdll_base, cryptdll_size)]): + + # this occurs across page boundaries + if not proc_layer.is_valid(address, ecrypt_size): + continue + + kerb = cryptdll_types.object("_KERB_ECRYPT", + offset = address, + absolute = True) + + # ensure the Encrypt and Finish pointers are inside the VAD + # these are not manipulated in the attack + if (cryptdll_base < kerb.Encrypt < cryptdll_end) and \ + (cryptdll_base < kerb.Finish < cryptdll_end): + csystems.append(kerb) + + return csystems + + def _generator(self, procs): + """ + Finds instances of the RC4 HMAC CSystem structure + + Returns whether the instances are hooked as well as the function handler addresses + + Args: + procs: the process list filtered to lsass.exe instances + """ + kernel = self.context.modules[self.config['kernel']] + + if not symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name): + vollog.info("This plugin only supports 64bit Windows memory samples") + return + + lsass_proc, proc_layer_name = self._find_lsass_proc(procs) + if not lsass_proc: + vollog.info( + "Unable to find a valid lsass.exe process in the process list. This should never happen. Analysis cannot proceed.") + return + + cryptdll_base, cryptdll_size = self._find_cryptdll(lsass_proc) + if not cryptdll_base: + vollog.info("Unable to find the location of cryptdll.dll inside of lsass.exe. Analysis cannot proceed.") + return + + # the custom type information from binary analysis + cryptdll_types = self._get_cryptdll_types(self.context, + self.config, + self.config_path, + proc_layer_name, + cryptdll_base) + + # attempt to find the array and symbols directly from the PDB + csystems, rc4HmacInitialize, rc4HmacDecrypt = \ + self._find_csystems_with_symbols(proc_layer_name, + cryptdll_types, + cryptdll_base, + cryptdll_size) + + # if we can't find cSystems through the PDB then + # we fall back to export analysis and scanning + # we keep the address of the rc4 functions from the PDB + # though as its our only source to get them + if csystems is None: + fallback_sources = [self._find_csystems_with_export, + self._find_csystems_with_scanning] + + for source in fallback_sources: + csystems = source(proc_layer_name, + cryptdll_types, + cryptdll_base, + cryptdll_size) + + if csystems is not None: + break + + if csystems is None: + vollog.info("Unable to find CSystems inside of cryptdll.dll. Analysis cannot proceed.") + return + + for csystem in csystems: + if not self.context.layers[proc_layer_name].is_valid(csystem.vol.offset, csystem.vol.size): + continue + + # filter for RC4 HMAC + if csystem.EncryptionType != 0x17: + continue + + # use the specific symbols if present, otherwise use the vad start and size + if rc4HmacInitialize and rc4HmacDecrypt: + skeleton_key_present = self._check_for_skeleton_key_symbols(csystem, rc4HmacInitialize, rc4HmacDecrypt) + else: + skeleton_key_present = self._check_for_skeleton_key_vad(csystem, cryptdll_base, cryptdll_size) + + yield 0, (lsass_proc.UniqueProcessId, "lsass.exe", skeleton_key_present, \ + format_hints.Hex(csystem.Initialize), format_hints.Hex(csystem.Decrypt)) + + def _lsass_proc_filter(self, proc): + """ + Used to filter to only lsass.exe processes + + There should only be one of these, but malware can/does make lsass.exe + named processes to blend in or uses lsass.exe as a process hollowing target + """ + process_name = utility.array_to_string(proc.ImageFileName) + + return process_name != "lsass.exe" + + def run(self): + kernel = self.context.modules[self.config['kernel']] + + return renderers.TreeGrid( + [("PID", int), ("Process", str), ("Skeleton Key Found", bool), ("rc4HmacInitialize", format_hints.Hex), + ("rc4HmacDecrypt", format_hints.Hex)], + self._generator( + pslist.PsList.list_processes(context = self.context, + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, + filter_func = self._lsass_proc_filter))) diff --git a/volatility3/framework/plugins/windows/ssdt.py b/volatility3/framework/plugins/windows/ssdt.py index 8092beb2f9..0d921535d9 100644 --- a/volatility3/framework/plugins/windows/ssdt.py +++ b/volatility3/framework/plugins/windows/ssdt.py @@ -18,16 +18,14 @@ class SSDT(plugins.PluginInterface): """Lists the system call table.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'modules', plugin = modules.Modules, version = (1, 0, 0)), ] @@ -62,12 +60,12 @@ def build_module_collection(cls, context: interfaces.context.ContextInterface, l if module_name in constants.windows.KERNEL_MODULE_NAMES: symbol_table_name = symbol_table - context_module = contexts.SizedModule(context, - module_name, - layer_name, - mod.DllBase, - mod.SizeOfImage, - symbol_table_name = symbol_table_name) + context_module = contexts.SizedModule.create(context = context, + module_name = module_name, + layer_name = layer_name, + offset = mod.DllBase, + size = mod.SizeOfImage, + symbol_table_name = symbol_table_name) context_modules.append(context_module) @@ -75,11 +73,13 @@ def build_module_collection(cls, context: interfaces.context.ContextInterface, l def _generator(self) -> Iterator[Tuple[int, Tuple[int, int, Any, Any]]]: - layer_name = self.config['primary'] - collection = self.build_module_collection(self.context, self.config["primary"], self.config["nt_symbols"]) + kernel = self.context.modules[self.config['kernel']] + + layer_name = kernel.layer_name + collection = self.build_module_collection(self.context, layer_name, kernel.symbol_table_name) kvo = self.context.layers[layer_name].config['kernel_virtual_offset'] - ntkrnlmp = self.context.module(self.config["nt_symbols"], layer_name = layer_name, offset = kvo) + ntkrnlmp = self.context.module(kernel.symbol_table_name, layer_name = layer_name, offset = kvo) # this is just one way to enumerate the native (NT) service table. # to do the same thing for the Win32K service table, we would need Win32K.sys symbol support @@ -91,7 +91,7 @@ def _generator(self) -> Iterator[Tuple[int, Tuple[int, int, Any, Any]]]: # on 32-bit systems the table indexes are 32-bits and contain pointers (unsigned) # on 64-bit systems the indexes are also 32-bits but they're offsets from the # base address of the table and can be negative, so we need a signed data type - is_kernel_64 = symbols.symbol_table_is_64bit(self.context, self.config["nt_symbols"]) + is_kernel_64 = symbols.symbol_table_is_64bit(self.context, kernel.symbol_table_name) if is_kernel_64: array_subtype = "long" diff --git a/volatility3/framework/plugins/windows/strings.py b/volatility3/framework/plugins/windows/strings.py index 3e9fdfb855..e79f64741f 100644 --- a/volatility3/framework/plugins/windows/strings.py +++ b/volatility3/framework/plugins/windows/strings.py @@ -4,10 +4,9 @@ import logging import re -from os import path -from typing import Dict, Generator, List, Set, Tuple +from typing import Dict, Generator, List, Set, Tuple, Optional -from volatility3.framework import interfaces, renderers, exceptions +from volatility3.framework import interfaces, renderers, exceptions, constants from volatility3.framework.configuration import requirements from volatility3.framework.layers import intel, resources, linear from volatility3.framework.renderers import format_hints @@ -19,17 +18,20 @@ class Strings(interfaces.plugins.PluginInterface): """Reads output from the strings command and indicates which process(es) each string belongs to.""" - _required_framework_version = (1, 0, 0) - strings_pattern = re.compile(rb"(?:\W*)([0-9]+)(?:\W*)(\w[\w\W]+)\n?") + _version = (1, 2, 0) + _required_framework_version = (2, 0, 0) + strings_pattern = re.compile(rb"^(?:\W*)([0-9]+)(?:\W*)(\w[\w\W]+)\n?") @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + requirements.ListRequirement(name = 'pid', + element_type = int, + description = "Process ID to include (all other processes are excluded)", + optional = True), requirements.URIRequirement(name = "strings_file", description = "Strings file") ] # TODO: Make URLRequirement that can accept a file address which the framework can open @@ -40,27 +42,40 @@ def run(self): def _generator(self) -> Generator[Tuple, None, None]: """Generates results from a strings file.""" - revmap = self.generate_mapping(self.config['primary']) + string_list: List[Tuple[int,bytes]] = [] + # Test strings file format is accurate accessor = resources.ResourceAccessor() strings_fp = accessor.open(self.config['strings_file'], "rb") - strings_size = path.getsize(strings_fp.file.name) - line = strings_fp.readline() - last_prog = 0 + count: float = 0 while line: + count += 1 try: offset, string = self._parse_line(line) - try: - revmap_list = [name + ":" + hex(offset) for (name, offset) in revmap[offset >> 12]] - except (IndexError, KeyError): - revmap_list = ["FREE MEMORY"] - yield (0, (str(string, 'latin-1'), format_hints.Hex(offset), ", ".join(revmap_list))) + string_list.append((offset, string)) except ValueError: - vollog.error("Strings file is in the wrong format") - return + vollog.error(f"Line in unrecognized format: line {count}") line = strings_fp.readline() - prog = strings_fp.tell() / strings_size * 100 + kernel = self.context.modules[self.config['kernel']] + + revmap = self.generate_mapping(self.context, + kernel.layer_name, + kernel.symbol_table_name, + progress_callback = self._progress_callback, + pid_list = self.config['pid']) + + last_prog: float = 0 + line_count: float = 0 + num_strings = len(string_list) + for offset, string in string_list: + line_count += 1 + try: + revmap_list = [name + ":" + hex(offset) for (name, offset) in revmap[offset >> 12]] + except (IndexError, KeyError): + revmap_list = ["FREE MEMORY"] + yield (0, (str(string, 'latin-1'), format_hints.Hex(offset), ", ".join(revmap_list))) + prog = line_count / num_strings * 100 if round(prog, 1) > last_prog: last_prog = round(prog, 1) self._progress_callback(prog, "Matching strings in memory") @@ -81,18 +96,30 @@ def _parse_line(self, line: bytes) -> Tuple[int, bytes]: offset, string = match.group(1, 2) return int(offset), string - def generate_mapping(self, layer_name: str) -> Dict[int, Set[Tuple[str, int]]]: + @classmethod + def generate_mapping(cls, + context: interfaces.context.ContextInterface, + layer_name: str, + symbol_table: str, + progress_callback: constants.ProgressCallback = None, + pid_list: Optional[List[int]] = None) -> Dict[int, Set[Tuple[str, int]]]: """Creates a reverse mapping between virtual addresses and physical addresses. Args: + context: the context for the method to run against layer_name: the layer to map against the string lines + symbol_table: the name of the symbol table for the provided layer + progress_callback: an optional callable to display progress + pid_list: a lit of process IDs to consider when generating the reverse map Returns: A mapping of virtual offsets to strings and physical offsets """ - layer = self._context.layers[layer_name] - reverse_map = dict() # type: Dict[int, Set[Tuple[str, int]]] + filter = pslist.PsList.create_pid_filter(pid_list) + + layer = context.layers[layer_name] + reverse_map: Dict[int, Set[Tuple[str, int]]] = dict() if isinstance(layer, intel.Intel): # We don't care about errors, we just wanted chunks that map correctly for mapval in layer.mapping(0x0, layer.maximum_address, ignore_errors = True): @@ -101,31 +128,33 @@ def generate_mapping(self, layer_name: str) -> Dict[int, Set[Tuple[str, int]]]: cur_set = reverse_map.get(mapped_offset >> 12, set()) cur_set.add(("kernel", offset)) reverse_map[mapped_offset >> 12] = cur_set - self._progress_callback((offset * 100) / layer.maximum_address, "Creating reverse kernel map") + if progress_callback: + progress_callback((offset * 100) / layer.maximum_address, "Creating reverse kernel map") # TODO: Include kernel modules - for process in pslist.PsList.list_processes(self.context, self.config['primary'], - self.config['nt_symbols']): - proc_id = "Unknown" - try: - proc_id = process.UniqueProcessId - proc_layer_name = process.add_process_layer() - except exceptions.InvalidAddressException as excp: - vollog.debug("Process {}: invalid address {} in layer {}".format( - proc_id, excp.invalid_address, excp.layer_name)) - continue - - proc_layer = self.context.layers[proc_layer_name] - if isinstance(proc_layer, linear.LinearlyMappedLayer): - for mapval in proc_layer.mapping(0x0, proc_layer.maximum_address, ignore_errors = True): - mapped_offset, _, offset, mapped_size, maplayer = mapval - for val in range(mapped_offset, mapped_offset + mapped_size, 0x1000): - cur_set = reverse_map.get(mapped_offset >> 12, set()) - cur_set.add(("Process {}".format(process.UniqueProcessId), offset)) - reverse_map[mapped_offset >> 12] = cur_set - # FIXME: make the progress for all processes, rather than per-process - self._progress_callback((offset * 100) / layer.maximum_address, - "Creating mapping for task {}".format(process.UniqueProcessId)) + for process in pslist.PsList.list_processes(context, layer_name, symbol_table): + if not filter(process): + proc_id = "Unknown" + try: + proc_id = process.UniqueProcessId + proc_layer_name = process.add_process_layer() + except exceptions.InvalidAddressException as excp: + vollog.debug("Process {}: invalid address {} in layer {}".format( + proc_id, excp.invalid_address, excp.layer_name)) + continue + + proc_layer = context.layers[proc_layer_name] + if isinstance(proc_layer, linear.LinearlyMappedLayer): + for mapval in proc_layer.mapping(0x0, proc_layer.maximum_address, ignore_errors = True): + mapped_offset, _, offset, mapped_size, maplayer = mapval + for val in range(mapped_offset, mapped_offset + mapped_size, 0x1000): + cur_set = reverse_map.get(mapped_offset >> 12, set()) + cur_set.add((f"Process {process.UniqueProcessId}", offset)) + reverse_map[mapped_offset >> 12] = cur_set + # FIXME: make the progress for all processes, rather than per-process + if progress_callback: + progress_callback((offset * 100) / layer.maximum_address, + f"Creating mapping for task {process.UniqueProcessId}") return reverse_map diff --git a/volatility3/framework/plugins/windows/svcscan.py b/volatility3/framework/plugins/windows/svcscan.py index 4820e5735b..b6adbb0b09 100644 --- a/volatility3/framework/plugins/windows/svcscan.py +++ b/volatility3/framework/plugins/windows/svcscan.py @@ -3,6 +3,7 @@ # import logging +import os from typing import List from volatility3.framework import interfaces, renderers, constants, symbols, exceptions @@ -20,17 +21,15 @@ class SvcScan(interfaces.plugins.PluginInterface): """Scans for windows services.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.PluginRequirement(name = 'poolscanner', plugin = poolscanner.PoolScanner, version = (1, 0, 0)), requirements.PluginRequirement(name = 'vadyarascan', plugin = vadyarascan.VadYaraScan, version = (1, 0, 0)) @@ -87,21 +86,24 @@ def create_service_table(context: interfaces.context.ContextInterface, symbol_ta return intermed.IntermediateSymbolTable.create(context, config_path, - "windows", + os.path.join("windows", "services"), symbol_filename, class_types = services.class_types, native_types = native_types) def _generator(self): + kernel = self.context.modules[self.config['kernel']] - service_table_name = self.create_service_table(self.context, self.config["nt_symbols"], self.config_path) + service_table_name = self.create_service_table(self.context, kernel.symbol_table_name, + self.config_path) relative_tag_offset = self.context.symbol_space.get_type(service_table_name + constants.BANG + "_SERVICE_RECORD").relative_child_offset("Tag") filter_func = pslist.PsList.create_name_filter(["services.exe"]) - is_vista_or_later = versions.is_vista_or_later(context = self.context, symbol_table = self.config["nt_symbols"]) + is_vista_or_later = versions.is_vista_or_later(context = self.context, + symbol_table = kernel.symbol_table_name) if is_vista_or_later: service_tag = b"serH" @@ -111,8 +113,8 @@ def _generator(self): seen = [] for task in pslist.PsList.list_processes(context = self.context, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, filter_func = filter_func): proc_id = "Unknown" @@ -160,7 +162,7 @@ def run(self): return renderers.TreeGrid([ ('Offset', format_hints.Hex), ('Order', int), - ('Pid', int), + ('PID', int), ('Start', str), ('State', str), ('Type', str), diff --git a/volatility3/framework/plugins/windows/symlinkscan.py b/volatility3/framework/plugins/windows/symlinkscan.py index abfbd1d6d9..ef970b2968 100644 --- a/volatility3/framework/plugins/windows/symlinkscan.py +++ b/volatility3/framework/plugins/windows/symlinkscan.py @@ -15,15 +15,13 @@ class SymlinkScan(interfaces.plugins.PluginInterface, timeliner.TimeLinerInterface): """Scans for links present in a particular windows memory image.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls): return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), ] @classmethod @@ -51,7 +49,9 @@ def scan_symlinks(cls, yield mem_object def _generator(self): - for link in self.scan_symlinks(self.context, self.config['primary'], self.config['nt_symbols']): + kernel = self.context.modules[self.config['kernel']] + + for link in self.scan_symlinks(self.context, kernel.layer_name, kernel.symbol_table_name): try: from_name = link.get_link_name() @@ -68,7 +68,7 @@ def _generator(self): def generate_timeline(self): for row in self._generator(): _depth, row_data = row - description = "Symlink: {} -> {}".format(row_data[2], row_data[3]) + description = f"Symlink: {row_data[2]} -> {row_data[3]}" yield (description, timeliner.TimeLinerType.CREATED, row_data[1]) def run(self): diff --git a/volatility3/framework/plugins/windows/vadinfo.py b/volatility3/framework/plugins/windows/vadinfo.py index d50f596aa0..9fa1458d17 100644 --- a/volatility3/framework/plugins/windows/vadinfo.py +++ b/volatility3/framework/plugins/windows/vadinfo.py @@ -33,9 +33,9 @@ class VadInfo(interfaces.plugins.PluginInterface): """Lists process memory ranges.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (2, 0, 0) - MAXSIZE_DEFAULT = 0 + MAXSIZE_DEFAULT = 1024 * 1024 * 1024 # 1 Gb def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -44,10 +44,8 @@ def __init__(self, *args, **kwargs): @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements - return [requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', + return [requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), # TODO: Convert this to a ListRequirement so that people can filter on sets of ranges requirements.IntRequirement(name = 'address', description = "Process virtual memory address to include " \ @@ -132,11 +130,11 @@ def vad_dump(cls, vad_end = vad.get_end() except AttributeError: vollog.debug("Unable to find the starting/ending VPN member") - return + return None if maxsize > 0 and (vad_end - vad_start) > maxsize: - vollog.debug("Skip VAD dump {0:#x}-{1:#x} due to maxsize limit".format(vad_start, vad_end)) - return + vollog.debug(f"Skip VAD dump {vad_start:#x}-{vad_end:#x} due to maxsize limit") + return None proc_id = "Unknown" try: @@ -148,7 +146,7 @@ def vad_dump(cls, return None proc_layer = context.layers[proc_layer_name] - file_name = "pid.{0}.vad.{1:#x}-{2:#x}.dmp".format(proc_id, vad_start, vad_end) + file_name = f"pid.{proc_id}.vad.{vad_start:#x}-{vad_end:#x}.dmp" try: file_handle = open_method(file_name) chunk_size = 1024 * 1024 * 10 @@ -162,12 +160,13 @@ def vad_dump(cls, offset += to_read except Exception as excp: - vollog.debug("Unable to dump VAD {}: {}".format(file_name, excp)) - return + vollog.debug(f"Unable to dump VAD {file_name}: {excp}") + return None return file_handle def _generator(self, procs): + kernel = self.context.modules[self.config['kernel']] def passthrough(_: interfaces.objects.ObjectInterface) -> bool: return False @@ -196,11 +195,12 @@ def filter_function(x: interfaces.objects.ObjectInterface) -> bool: yield (0, (proc.UniqueProcessId, process_name, format_hints.Hex(vad.vol.offset), format_hints.Hex(vad.get_start()), format_hints.Hex(vad.get_end()), vad.get_tag(), vad.get_protection( - self.protect_values(self.context, self.config['primary'], self.config['nt_symbols']), + self.protect_values(self.context, kernel.layer_name, kernel.symbol_table_name), winnt_protections), vad.get_commit_charge(), vad.get_private_memory(), format_hints.Hex(vad.get_parent()), vad.get_file_name(), file_output)) def run(self): + kernel = self.context.modules[self.config['kernel']] filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) @@ -210,6 +210,6 @@ def run(self): ("Parent", format_hints.Hex), ("File", str), ("File output", str)], self._generator( pslist.PsList.list_processes(context = self.context, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, filter_func = filter_func))) diff --git a/volatility3/framework/plugins/windows/vadyarascan.py b/volatility3/framework/plugins/windows/vadyarascan.py index cdb349cf8e..06a87d0033 100644 --- a/volatility3/framework/plugins/windows/vadyarascan.py +++ b/volatility3/framework/plugins/windows/vadyarascan.py @@ -17,16 +17,14 @@ class VadYaraScan(interfaces.plugins.PluginInterface): """Scans all the Virtual Address Descriptor memory maps using yara.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = "Memory layer for the kernel", + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), requirements.BooleanRequirement(name = "wide", description = "Match wide (unicode) strings", default = False, @@ -35,6 +33,11 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] description = "Yara rules (as a string)", optional = True), requirements.URIRequirement(name = "yara_file", description = "Yara rules (as a file)", optional = True), + # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code + # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later + requirements.URIRequirement(name = "yara_compiled_file", + description = "Yara compiled rules (as a file)", + optional = True), requirements.IntRequirement(name = "max_size", default = 0x40000000, description = "Set the maximum size (default is 1GB)", @@ -49,14 +52,15 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] ] def _generator(self): + kernel = self.context.modules[self.config['kernel']] rules = yarascan.YaraScan.process_yara_options(dict(self.config)) filter_func = pslist.PsList.create_pid_filter(self.config.get('pid', None)) for task in pslist.PsList.list_processes(context = self.context, - layer_name = self.config['primary'], - symbol_table = self.config['nt_symbols'], + layer_name = kernel.layer_name, + symbol_table = kernel.symbol_table_name, filter_func = filter_func): layer_name = task.add_process_layer() layer = self.context.layers[layer_name] @@ -83,5 +87,5 @@ def get_vad_maps(task: interfaces.objects.ObjectInterface) -> Iterable[Tuple[int yield (start, end - start) def run(self): - return renderers.TreeGrid([('Offset', format_hints.Hex), ('Pid', int), ('Rule', str), ('Component', str), + return renderers.TreeGrid([('Offset', format_hints.Hex), ('PID', int), ('Rule', str), ('Component', str), ('Value', bytes)], self._generator()) diff --git a/volatility3/framework/plugins/windows/verinfo.py b/volatility3/framework/plugins/windows/verinfo.py index d65402a6ad..762832e396 100644 --- a/volatility3/framework/plugins/windows/verinfo.py +++ b/volatility3/framework/plugins/windows/verinfo.py @@ -4,10 +4,12 @@ import io import logging -from typing import Generator, List, Tuple +import struct +from typing import Generator, List, Tuple, Optional from volatility3.framework import exceptions, renderers, constants, interfaces from volatility3.framework.configuration import requirements +from volatility3.framework.layers import scanners from volatility3.framework.renderers import format_hints from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows.extensions import pe @@ -25,22 +27,45 @@ class VerInfo(interfaces.plugins.PluginInterface): """Lists version information from PE files.""" - _required_framework_version = (1, 0, 0) + _version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: ## TODO: we might add a regex option on the name later, but otherwise we're good ## TODO: and we don't want any CLI options from pslist, modules, or moddump return [ + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]), requirements.PluginRequirement(name = 'pslist', plugin = pslist.PsList, version = (2, 0, 0)), requirements.PluginRequirement(name = 'modules', plugin = modules.Modules, version = (1, 0, 0)), requirements.VersionRequirement(name = 'dlllist', component = dlllist.DllList, version = (2, 0, 0)), - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols"), + requirements.BooleanRequirement(name = "extensive", + description = "Search physical layer for version information", + optional = True, + default = False), ] + @classmethod + def find_version_info(cls, context: interfaces.context.ContextInterface, layer_name: str, + filename: str) -> Optional[Tuple[int, int, int, int]]: + """Searches for an original filename, then tracks back to find the VS_VERSION_INFO and read the fixed + version information structure""" + premable_max_distance = 0x500 + filename = "OriginalFilename\x00" + filename + iterator = context.layers[layer_name].scan(context = context, + scanner = scanners.BytesScanner(bytes(filename, 'utf-16be'))) + for offset in iterator: + data = context.layers[layer_name].read(offset - premable_max_distance, premable_max_distance) + vs_ver_info = b"\xbd\x04\xef\xfe" + verinfo_offset = data.find(vs_ver_info) + len(vs_ver_info) + if verinfo_offset >= 0: + structure = ' Tuple[int, int, int, int]: @@ -96,6 +121,7 @@ def _generator(self, procs: Generator[interfaces.objects.ObjectInterface, None, mods: of modules session_layers: of layers in the session to be checked """ + kernel = self.context.modules[self.config['kernel']] pe_table_name = intermed.IntermediateSymbolTable.create(self.context, self.config_path, @@ -103,6 +129,9 @@ def _generator(self, procs: Generator[interfaces.objects.ObjectInterface, None, "pe", class_types = pe.class_types) + # TODO: Fix this so it works with more than just intel layers + physical_layer_name = self.context.layers[kernel.layer_name].config.get('memory_layer', None) + for mod in mods: try: BaseDllName = mod.BaseDllName.get_string() @@ -115,6 +144,11 @@ def _generator(self, procs: Generator[interfaces.objects.ObjectInterface, None, session_layer_name, mod.DllBase) except (exceptions.InvalidAddressException, TypeError, AttributeError): (major, minor, product, build) = [renderers.UnreadableValue()] * 4 + if (not isinstance(BaseDllName, renderers.UnreadableValue) and physical_layer_name is not None + and self.config['extensive']): + result = self.find_version_info(self._context, physical_layer_name, BaseDllName) + if result is not None: + (major, minor, product, build) = result # the pid and process are not applicable for kernel modules yield (0, (renderers.NotApplicableValue(), renderers.NotApplicableValue(), format_hints.Hex(mod.DllBase), @@ -156,13 +190,14 @@ def _generator(self, procs: Generator[interfaces.objects.ObjectInterface, None, build)) def run(self): - procs = pslist.PsList.list_processes(self.context, self.config["primary"], self.config["nt_symbols"]) + kernel = self.context.modules[self.config['kernel']] + + procs = pslist.PsList.list_processes(self.context, kernel.layer_name, kernel.symbol_table_name) - mods = modules.Modules.list_modules(self.context, self.config["primary"], self.config["nt_symbols"]) + mods = modules.Modules.list_modules(self.context, kernel.layer_name, kernel.symbol_table_name) # populate the session layers for kernel modules - session_layers = modules.Modules.get_session_layers(self.context, self.config['primary'], - self.config['nt_symbols']) + session_layers = modules.Modules.get_session_layers(self.context, kernel.layer_name, kernel.symbol_table_name) return renderers.TreeGrid([("PID", int), ("Process", str), ("Base", format_hints.Hex), ("Name", str), ("Major", int), ("Minor", int), ("Product", int), ("Build", int)], diff --git a/volatility3/framework/plugins/windows/virtmap.py b/volatility3/framework/plugins/windows/virtmap.py index 552564cad3..9a241d3d84 100644 --- a/volatility3/framework/plugins/windows/virtmap.py +++ b/volatility3/framework/plugins/windows/virtmap.py @@ -16,16 +16,14 @@ class VirtMap(interfaces.plugins.PluginInterface): """Lists virtual mapped sections.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: # Since we're calling the plugin, make sure we have the plugin's requirements return [ - requirements.TranslationLayerRequirement(name = 'primary', - description = 'Memory layer for the kernel', - architectures = ["Intel32", "Intel64"]), - requirements.SymbolTableRequirement(name = "nt_symbols", description = "Windows kernel symbols") + requirements.ModuleRequirement(name = 'kernel', description = 'Windows kernel', + architectures = ["Intel32", "Intel64"]) ] def _generator(self, map): @@ -41,7 +39,7 @@ def determine_map(cls, module: interfaces.context.ModuleInterface) -> \ if not isinstance(layer, intel.Intel): raise - result = {} # type: Dict[str, List[Tuple[int, int]]] + result: Dict[str, List[Tuple[int, int]]] = {} system_va_type = module.get_enumeration('_MI_SYSTEM_VA_TYPE') large_page_size = (layer.page_size ** 2) // module.get_type("_MMPTE").size @@ -84,7 +82,7 @@ def determine_map(cls, module: interfaces.context.ModuleInterface) -> \ def _enumerate_system_va_type(cls, large_page_size: int, system_range_start: int, module: interfaces.context.ModuleInterface, type_array: interfaces.objects.ObjectInterface) -> Dict[str, List[Tuple[int, int]]]: - result = {} # type: Dict[str, List[Tuple[int, int]]] + result: Dict[str, List[Tuple[int, int]]] = {} system_va_type = module.get_enumeration('_MI_SYSTEM_VA_TYPE') start = system_range_start prev_entry = -1 @@ -112,8 +110,10 @@ def scannable_sections(cls, module: interfaces.context.ModuleInterface) -> Gener yield value def run(self): - layer = self.context.layers[self.config['primary']] - module = self.context.module(self.config['nt_symbols'], + kernel = self.context.modules[self.config['kernel']] + + layer = self.context.layers[kernel.layer_name] + module = self.context.module(kernel.symbol_table_name, layer_name = layer.name, offset = layer.config['kernel_virtual_offset']) diff --git a/volatility3/framework/plugins/yarascan.py b/volatility3/framework/plugins/yarascan.py index 1c04e23110..94b3cba452 100644 --- a/volatility3/framework/plugins/yarascan.py +++ b/volatility3/framework/plugins/yarascan.py @@ -26,6 +26,8 @@ class YaraScanner(interfaces.layers.ScannerInterface): # yara.Rules isn't exposed, so we can't type this properly def __init__(self, rules) -> None: super().__init__() + if rules is None: + raise ValueError("No rules provided to YaraScanner") self._rules = rules def __call__(self, data: bytes, data_offset: int) -> Iterable[Tuple[int, str, str, bytes]]: @@ -37,7 +39,7 @@ def __call__(self, data: bytes, data_offset: int) -> Iterable[Tuple[int, str, st class YaraScan(plugins.PluginInterface): """Scans kernel memory using yara rules (string or file).""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) _version = (1, 0, 0) @classmethod @@ -58,6 +60,11 @@ def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface] description = "Yara rules (as a string)", optional = True), requirements.URIRequirement(name = "yara_file", description = "Yara rules (as a file)", optional = True), + # This additional requirement is to follow suit with upstream, who feel that compiled rules could potentially be used to execute malicious code + # As such, there's a separate option to run compiled files, as happened with yara-3.9 and later + requirements.URIRequirement(name = "yara_compiled_file", + description = "Yara compiled rules (as a file)", + optional = True), requirements.IntRequirement(name = "max_size", default = 0x40000000, description = "Set the maximum size (default is 1GB)", @@ -70,14 +77,16 @@ def process_yara_options(cls, config: Dict[str, Any]): if config.get('yara_rules', None) is not None: rule = config['yara_rules'] if rule[0] not in ["{", "/"]: - rule = '"{}"'.format(rule) + rule = f'"{rule}"' if config.get('case', False): rule += " nocase" if config.get('wide', False): rule += " wide ascii" - rules = yara.compile(sources = {'n': 'rule r1 {{strings: $a = {} condition: $a}}'.format(rule)}) + rules = yara.compile(sources = {'n': f'rule r1 {{strings: $a = {rule} condition: $a}}'}) elif config.get('yara_file', None) is not None: rules = yara.compile(file = resources.ResourceAccessor().open(config['yara_file'], "rb")) + elif config.get('yara_compiled_file', None) is not None: + rules = yara.load(file = resources.ResourceAccessor().open(config['yara_compiled_file'], "rb")) else: vollog.error("No yara rules, nor yara rules file were specified") return rules diff --git a/volatility3/framework/renderers/__init__.py b/volatility3/framework/renderers/__init__.py index c7dc126b21..5773861d91 100644 --- a/volatility3/framework/renderers/__init__.py +++ b/volatility3/framework/renderers/__init__.py @@ -7,9 +7,10 @@ or file or graphical output """ import collections +import collections.abc import datetime import logging -from typing import Any, Callable, Iterable, List, Optional, Sequence, Tuple, TypeVar, Union +from typing import Any, Callable, Iterable, List, Optional, Tuple, TypeVar, Union from volatility3.framework import interfaces from volatility3.framework.interfaces import renderers @@ -48,7 +49,7 @@ class NotAvailableValue(interfaces.renderers.BaseAbsentValue): class TreeNode(interfaces.renderers.TreeNode): """Class representing a particular node in a tree grid.""" - def __init__(self, path: str, treegrid: 'TreeGrid', parent: Optional['TreeNode'], + def __init__(self, path: str, treegrid: 'TreeGrid', parent: Optional[interfaces.renderers.TreeNode], values: List[interfaces.renderers.BaseTypes]) -> None: if not isinstance(treegrid, TreeGrid): raise TypeError("Treegrid must be an instance of TreeGrid") @@ -59,7 +60,7 @@ def __init__(self, path: str, treegrid: 'TreeGrid', parent: Optional['TreeNode'] self._values = treegrid.RowStructure(*values) # type: ignore def __repr__(self) -> str: - return "".format(self.path, self._values) + return f"" def __getitem__(self, item: Union[int, slice]) -> Any: return self._treegrid.children(self).__getitem__(item) @@ -85,10 +86,10 @@ def _validate_values(self, values: List[interfaces.renderers.BaseTypes]) -> None # tznaive = val.tzinfo is None or val.tzinfo.utcoffset(val) is None @property - def values(self) -> Sequence[interfaces.renderers.BaseTypes]: + def values(self) -> List[interfaces.renderers.BaseTypes]: """Returns the list of values from the particular node, based on column index.""" - return self._values + return list(self._values) @property def path(self) -> str: @@ -101,7 +102,7 @@ def path(self) -> str: return self._path @property - def parent(self) -> Optional['TreeNode']: + def parent(self) -> Optional[interfaces.renderers.TreeNode]: """Returns the parent node of this node or None.""" return self._parent @@ -158,8 +159,8 @@ def __init__(self, columns: List[Tuple[str, interfaces.renderers.BaseTypes]], """ self._populated = False self._row_count = 0 - self._children = [] # type: List[interfaces.renderers.TreeNode] - converted_columns = [] # type: List[interfaces.renderers.Column] + self._children: List[interfaces.renderers.TreeNode] = [] + converted_columns: List[interfaces.renderers.Column] = [] if len(columns) < 1: raise ValueError("Columns must be a list containing at least one column") for (name, column_type) in columns: @@ -207,7 +208,7 @@ def function(_x: interfaces.renderers.TreeNode, _y: Any) -> Any: if not self.populated: try: - prev_nodes = [] # type: List[interfaces.renderers.TreeNode] + prev_nodes: List[interfaces.renderers.TreeNode] = [] for (level, item) in self._generator: parent_index = min(len(prev_nodes), level) parent = prev_nodes[parent_index - 1] if parent_index > 0 else None @@ -219,7 +220,7 @@ def function(_x: interfaces.renderers.TreeNode, _y: Any) -> Any: except Exception as excp: if fail_on_errors: raise - vollog.debug("Exception during population: {}".format(excp)) + vollog.debug(f"Exception during population: {excp}") self._populated = True return excp self._populated = True @@ -271,20 +272,26 @@ def values(self, node): def _append(self, parent: Optional[interfaces.renderers.TreeNode], values: Any) -> TreeNode: """Adds a new node at the top level if parent is None, or under the parent node otherwise, after all other children.""" - children = self.children(parent) - return self._insert(parent, len(children), values) + return self._insert(parent, None, values) - def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: int, values: Any) -> TreeNode: + def _insert(self, parent: Optional[interfaces.renderers.TreeNode], position: Optional[int], values: Any) -> TreeNode: """Inserts an element into the tree at a specific position.""" parent_path = "" children = self._find_children(parent) if parent is not None: parent_path = parent.path + self.path_sep - newpath = parent_path + str(position) + if position is None: + newpath = parent_path + str(len(children)) + else: + newpath = parent_path + str(position) + for node, _ in children[position:]: + self.visit(node, lambda child, _: child.path_changed(newpath, True), None) + tree_item = TreeNode(newpath, self, parent, values) - for node, _ in children[position:]: - self.visit(node, lambda child, _: child.path_changed(newpath, True), None) - children.insert(position, (tree_item, [])) + if position is None: + children.append((tree_item, [])) + else: + children.insert(position, (tree_item, [])) return tree_item def is_ancestor(self, node, descendant): @@ -363,7 +370,7 @@ def __init__(self, treegrid: TreeGrid, column_name: str, ascending: bool = True) _index = i self._type = column.type if _index is None: - raise ValueError("Column not found in TreeGrid columns: {}".format(column_name)) + raise ValueError(f"Column not found in TreeGrid columns: {column_name}") self._index = _index def __call__(self, values: List[Any]) -> Any: diff --git a/volatility3/framework/renderers/conversion.py b/volatility3/framework/renderers/conversion.py index b60b47411c..996cf03a50 100644 --- a/volatility3/framework/renderers/conversion.py +++ b/volatility3/framework/renderers/conversion.py @@ -24,7 +24,7 @@ def wintime_to_datetime(wintime: int) -> Union[interfaces.renderers.BaseAbsentVa def unixtime_to_datetime(unixtime: int) -> Union[interfaces.renderers.BaseAbsentValue, datetime.datetime]: - ret = renderers.UnparsableValue() # type: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] + ret: Union[interfaces.renderers.BaseAbsentValue, datetime.datetime] = renderers.UnparsableValue() if unixtime > 0: try: @@ -96,7 +96,7 @@ def convert_network_four_tuple(family, four_tuple): dest port) into their string equivalents. IP addresses are expected as a tuple - of unsigned shorts Ports are converted to proper endianess as well + of unsigned shorts Ports are converted to proper endianness as well """ if family == socket.AF_INET: diff --git a/volatility3/framework/renderers/format_hints.py b/volatility3/framework/renderers/format_hints.py index 029049df32..f386d8e9df 100644 --- a/volatility3/framework/renderers/format_hints.py +++ b/volatility3/framework/renderers/format_hints.py @@ -8,7 +8,7 @@ Text renderers should attempt to honour all hints provided in this module where possible """ -from typing import Type +from typing import Type, Union class Bin(int): @@ -18,7 +18,7 @@ class Bin(int): class Hex(int): """A class to indicate that the integer value should be represented as a - hexidecimal value.""" + hexadecimal value.""" class HexBytes(bytes): @@ -30,20 +30,23 @@ class MultiTypeData(bytes): """The contents are supposed to be a string, but may contain binary data.""" def __new__(cls: Type['MultiTypeData'], - original: int, + original: Union[int, bytes], encoding: str = 'utf-16-le', split_nulls: bool = False, show_hex: bool = False) -> 'MultiTypeData': + if isinstance(original, int): - original = str(original).encode(encoding) - return super().__new__(cls, original) + data = str(original).encode(encoding) + else: + data = original + return super().__new__(cls, data) def __init__(self, original: bytes, encoding: str = 'utf-16-le', split_nulls: bool = False, show_hex: bool = False) -> None: - self.converted_int = False # type: bool + self.converted_int: bool = False if isinstance(original, int): self.converted_int = True self.encoding = encoding diff --git a/volatility3/framework/symbols/__init__.py b/volatility3/framework/symbols/__init__.py index c2444682c1..d6ecb5393a 100644 --- a/volatility3/framework/symbols/__init__.py +++ b/volatility3/framework/symbols/__init__.py @@ -31,15 +31,15 @@ class SymbolSpace(interfaces.symbols.SymbolSpaceInterface): def __init__(self) -> None: super().__init__() - self._dict = collections.OrderedDict() # type: Dict[str, interfaces.symbols.BaseSymbolTableInterface] + self._dict: Dict[str, interfaces.symbols.BaseSymbolTableInterface] = collections.OrderedDict() # Permanently cache all resolved symbols - self._resolved = {} # type: Dict[str, interfaces.objects.Template] - self._resolved_symbols = {} # type: Dict[str, interfaces.objects.Template] + self._resolved: Dict[str, interfaces.objects.Template] = {} + self._resolved_symbols: Dict[str, interfaces.objects.Template] = {} def clear_symbol_cache(self, table_name: str = None) -> None: """Clears the symbol cache for the specified table name. If no table name is specified, the caches of all symbol tables are cleared.""" - table_list = list() # type: List[interfaces.symbols.BaseSymbolTableInterface] + table_list: List[interfaces.symbols.BaseSymbolTableInterface] = list() if table_name is None: table_list = list(self._dict.values()) else: @@ -65,7 +65,7 @@ def get_symbols_by_type(self, type_name: str) -> Iterable[str]: def get_symbols_by_location(self, offset: int, size: int = 0, table_name: str = None) -> Iterable[str]: """Returns all symbols that exist at a specific relative address.""" - table_list = self._dict.values() # type: Iterable[interfaces.symbols.BaseSymbolTableInterface] + table_list: Iterable[interfaces.symbols.BaseSymbolTableInterface] = self._dict.values() if table_name is not None: if table_name in self._dict: table_list = [self._dict[table_name]] @@ -117,7 +117,7 @@ class UnresolvedTemplate(objects.templates.ReferenceTemplate): """ def __init__(self, type_name: str, **kwargs) -> None: - vollog.debug("Unresolved reference: {}".format(type_name)) + vollog.debug(f"Unresolved reference: {type_name}") super().__init__(type_name = type_name, **kwargs) def _weak_resolve(self, resolve_type: SymbolType, name: str) -> SymbolSpaceReturnType: @@ -139,8 +139,8 @@ def _weak_resolve(self, resolve_type: SymbolType, name: str) -> SymbolSpaceRetur return getattr(self._dict[table_name], get_function)(component_name) except KeyError as e: raise exceptions.SymbolError(component_name, table_name, - 'Type {} references missing Type/Symbol/Enum: {}'.format(name, e)) - raise exceptions.SymbolError(name, None, "Malformed name: {}".format(name)) + f'Type {name} references missing Type/Symbol/Enum: {e}') + raise exceptions.SymbolError(name, None, f"Malformed name: {name}") def _iterative_resolve(self, traverse_list): """Iteratively resolves a type, populating linked child @@ -185,7 +185,7 @@ def get_type(self, type_name: str) -> interfaces.objects.Template: index = type_name.find(constants.BANG) if index > 0: table_name, type_name = type_name[:index], type_name[index + 1:] - raise exceptions.SymbolError(type_name, table_name, "Unresolvable symbol requested: {}".format(type_name)) + raise exceptions.SymbolError(type_name, table_name, f"Unresolvable symbol requested: {type_name}") return self._resolved[type_name] def get_symbol(self, symbol_name: str) -> interfaces.symbols.SymbolInterface: @@ -198,7 +198,7 @@ def get_symbol(self, symbol_name: str) -> interfaces.symbols.SymbolInterface: index = symbol_name.find(constants.BANG) if index > 0: table_name, symbol_name = symbol_name[:index], symbol_name[index + 1:] - raise exceptions.SymbolError(symbol_name, table_name, "Unresolvable Symbol: {}".format(symbol_name)) + raise exceptions.SymbolError(symbol_name, table_name, f"Unresolvable Symbol: {symbol_name}") return retval def _subresolve(self, object_template: interfaces.objects.Template) -> interfaces.objects.Template: @@ -220,7 +220,7 @@ def get_enumeration(self, enum_name: str) -> interfaces.objects.Template: index = enum_name.find(constants.BANG) if index > 0: table_name, enum_name = enum_name[:index], enum_name[index + 1:] - raise exceptions.SymbolError(enum_name, table_name, "Unresolvable Enumeration: {}".format(enum_name)) + raise exceptions.SymbolError(enum_name, table_name, f"Unresolvable Enumeration: {enum_name}") return retval def _membership(self, member_type: SymbolType, name: str) -> bool: diff --git a/volatility3/framework/symbols/intermed.py b/volatility3/framework/symbols/intermed.py index 0ff03cbcab..c48fcc8a7a 100644 --- a/volatility3/framework/symbols/intermed.py +++ b/volatility3/framework/symbols/intermed.py @@ -13,14 +13,15 @@ from abc import ABCMeta from typing import Any, Dict, Generator, Iterable, List, Optional, Type, Tuple, Mapping -from volatility3.framework.layers import resources from volatility3 import schemas, symbols from volatility3.framework import class_subclasses, constants, exceptions, interfaces, objects from volatility3.framework.configuration import requirements +from volatility3.framework.layers import resources from volatility3.framework.symbols import native, metadata vollog = logging.getLogger(__name__) + # ## TODO # # All symbol tables should take a label to an object template @@ -47,7 +48,6 @@ def _construct_delegate_function(name: str, is_property: bool = False) -> Any: - def _delegate_function(self, *args, **kwargs): if is_property: return getattr(self._delegate, name) @@ -83,7 +83,6 @@ def __init__(self, table_mapping: Optional[Dict[str, str]] = None, validate: bool = True, class_types: Optional[Mapping[str, Type[interfaces.objects.ObjectInterface]]] = None, - symbol_shift: int = 0, symbol_mask: int = 0) -> None: """Instantiates a SymbolTable based on an IntermediateSymbolFormat JSON file. This is validated against the appropriate schema. The validation can be disabled by passing validate = False, but this should almost never be @@ -98,7 +97,6 @@ def __init__(self, table_mapping: A dictionary linking names referenced in the file with symbol tables in the context validate: Determines whether the ISF file will be validated against the appropriate schema class_types: A dictionary of type names and classes that override StructType when they are instantiated - symbol_shift: An offset by which to alter all returned symbols for this table symbol_mask: An address mask used for all returned symbol offsets from this table (a mask of 0 disables masking) """ # Check there are no obvious errors @@ -111,7 +109,7 @@ def __init__(self, # Validation is expensive, but we cache to store the hashes of successfully validated json objects if validate and not schemas.validate(json_object): - raise exceptions.SymbolSpaceError("File does not pass version validation: {}".format(isf_url)) + raise exceptions.SymbolSpaceError(f"File does not pass version validation: {isf_url}") metadata = json_object.get('metadata', None) @@ -123,7 +121,7 @@ def __init__(self, raise RuntimeError("ISF version {} is no longer supported: {}".format(metadata.get('format', "0.0.0"), isf_url)) elif self._delegate.version < constants.ISF_MINIMUM_DEPRECATED: - vollog.warning("ISF version {} has been deprecated: {}".format(metadata.get('format', "0.0.0"), isf_url)) + vollog.warning(f"ISF version {metadata.get('format', '0.0.0')} has been deprecated: {isf_url}") # Inherit super().__init__(context, @@ -135,7 +133,6 @@ def __init__(self, # Since we've been created with parameters, ensure our config is populated likewise self.config['isf_url'] = isf_url - self.config['symbol_shift'] = symbol_shift self.config['symbol_mask'] = symbol_mask @staticmethod @@ -154,7 +151,7 @@ def _closest_version(version: str, versions: Dict[Tuple[int, int, int], Type['IS supported_versions = [x for x in versions if x[0] == major and x[1] >= minor] if not supported_versions: raise ValueError( - "No Intermediate Format interface versions support file interface version: {}".format(version)) + f"No Intermediate Format interface versions support file interface version: {version}") return versions[max(supported_versions)] symbols = _construct_delegate_function('symbols', True) @@ -188,7 +185,7 @@ def file_symbol_url(cls, sub_path: str, filename: Optional[str] = None) -> Gener zip_match = "/".join(os.path.split(filename)) # Check user symbol directory first, then fallback to the framework's library to allow for overloading - vollog.log(constants.LOGLEVEL_VVVV, "Searching for symbols in {}".format(", ".join(symbols.__path__))) + vollog.log(constants.LOGLEVEL_VVVV, f"Searching for symbols in {', '.join(symbols.__path__)}") for path in symbols.__path__: if not os.path.isabs(path): path = os.path.abspath(os.path.join(__file__, path)) @@ -221,7 +218,6 @@ def create(cls, native_types: Optional[interfaces.symbols.NativeTableInterface] = None, table_mapping: Optional[Dict[str, str]] = None, class_types: Optional[Mapping[str, Type[interfaces.objects.ObjectInterface]]] = None, - symbol_shift: int = 0, symbol_mask: int = 0) -> str: """Takes a context and loads an intermediate symbol table based on a filename. @@ -233,7 +229,6 @@ def create(cls, filename: Basename of the file to find under the sub_path native_types: Set of native types, defaults to native types read from the intermediate symbol format file table_mapping: a dictionary of table names mentioned within the ISF file, and the tables within the context which they map to - symbol_shift: An offset by which to alter all returned symbols for this table symbol_mask: An address mask used for all returned symbol offsets from this table (a mask of 0 disables masking) Returns: @@ -250,7 +245,6 @@ def create(cls, native_types = native_types, table_mapping = table_mapping, class_types = class_types, - symbol_shift = symbol_shift, symbol_mask = symbol_mask) context.symbol_space.append(table) return table_name @@ -282,8 +276,8 @@ def __init__(self, raise TypeError("Native table not provided") nt.name = name + "_natives" super().__init__(context, config_path, name, nt, table_mapping = table_mapping) - self._overrides = {} # type: Dict[str, Type[interfaces.objects.ObjectInterface]] - self._symbol_cache = {} # type: Dict[str, interfaces.symbols.SymbolInterface] + self._overrides: Dict[str, Type[interfaces.objects.ObjectInterface]] = {} + self._symbol_cache: Dict[str, interfaces.symbols.SymbolInterface] = {} def _get_natives(self) -> Optional[interfaces.symbols.NativeTableInterface]: """Determines the appropriate native_types to use from the JSON @@ -300,7 +294,7 @@ def _get_natives(self) -> Optional[interfaces.symbols.NativeTableInterface]: # TODO: determine whether we should give voids a size - We don't give voids a length, whereas microsoft seemingly do pass else: - vollog.debug("Choosing appropriate natives for symbol library: {}".format(nc)) + vollog.debug(f"Choosing appropriate natives for symbol library: {nc}") return native_class.natives return None @@ -335,10 +329,11 @@ def get_symbol(self, name: str) -> interfaces.symbols.SymbolInterface: return self._symbol_cache[name] symbol = self._json_object['symbols'].get(name, None) if not symbol: - raise exceptions.SymbolError(name, self.name, "Unknown symbol: {}".format(name)) - address = symbol['address'] + self.config.get('symbol_shift', 0) + raise exceptions.SymbolError(name, self.name, f"Unknown symbol: {name}") + address = symbol['address'] if self.config.get('symbol_mask', 0): address = address & self.config['symbol_mask'] + self._symbol_cache[name] = interfaces.symbols.SymbolInterface(name = name, address = address) return self._symbol_cache[name] @@ -362,7 +357,7 @@ def get_type_class(self, name: str) -> Type[interfaces.objects.ObjectInterface]: def set_type_class(self, name: str, clazz: Type[interfaces.objects.ObjectInterface]) -> None: if name not in self.types: - raise ValueError("Symbol type not in {} SymbolTable: {}".format(self.name, name)) + raise ValueError(f"Symbol type not in {self.name} SymbolTable: {name}") self._overrides[name] = clazz def del_type_class(self, name: str) -> None: @@ -372,7 +367,7 @@ def del_type_class(self, name: str) -> None: def _interdict_to_template(self, dictionary: Dict[str, Any]) -> interfaces.objects.Template: """Converts an intermediate format dict into an object template.""" if not dictionary: - raise exceptions.SymbolSpaceError("Invalid intermediate dictionary: {}".format(dictionary)) + raise exceptions.SymbolSpaceError(f"Invalid intermediate dictionary: {dictionary}") type_name = dictionary['kind'] if type_name == 'base': @@ -407,7 +402,7 @@ def _interdict_to_template(self, dictionary: Dict[str, Any]) -> interfaces.objec # Otherwise if dictionary['kind'] not in objects.AggregateTypes.values(): - raise exceptions.SymbolSpaceError("Unknown Intermediate format: {}".format(dictionary)) + raise exceptions.SymbolSpaceError(f"Unknown Intermediate format: {dictionary}") reference_name = dictionary['name'] if constants.BANG not in reference_name: @@ -424,7 +419,7 @@ def _lookup_enum(self, name: str) -> Dict[str, Any]: parameters for an Enum.""" lookup = self._json_object['enums'].get(name, None) if not lookup: - raise exceptions.SymbolSpaceError("Unknown enumeration: {}".format(name)) + raise exceptions.SymbolSpaceError(f"Unknown enumeration: {name}") result = {"choices": copy.deepcopy(lookup['constants']), "base_type": self.natives.get_type(lookup['base'])} return result @@ -432,11 +427,11 @@ def get_enumeration(self, enum_name: str) -> interfaces.objects.Template: """Resolves an individual enumeration.""" if constants.BANG in enum_name: raise exceptions.SymbolError(enum_name, self.name, - "Enumeration for a different table requested: {}".format(enum_name)) + f"Enumeration for a different table requested: {enum_name}") if enum_name not in self._json_object['enums']: # Fall back to the natives table raise exceptions.SymbolError(enum_name, self.name, - "Enumeration not found in {} table: {}".format(self.name, enum_name)) + f"Enumeration not found in {self.name} table: {enum_name}") curdict = self._json_object['enums'][enum_name] base_type = self.natives.get_type(curdict['base']) # The size isn't actually used, the base-type defines it. @@ -452,7 +447,7 @@ def get_type(self, type_name: str) -> interfaces.objects.Template: table_name, type_name = type_name[:index], type_name[index + 1:] raise exceptions.SymbolError( type_name, table_name, - "Symbol for a different table requested: {}".format(table_name + constants.BANG + type_name)) + f"Symbol for a different table requested: {table_name + constants.BANG + type_name}") if type_name not in self._json_object['user_types']: # Fall back to the natives table return self.natives.get_type(self.name + constants.BANG + type_name) @@ -491,7 +486,7 @@ def _get_natives(self) -> Optional[interfaces.symbols.NativeTableInterface]: # TODO: determine whether we should give voids a size - We don't give voids a length, whereas microsoft seemingly do pass else: - vollog.debug("Choosing appropriate natives for symbol library: {}".format(nc)) + vollog.debug(f"Choosing appropriate natives for symbol library: {nc}") return native_class.natives return None @@ -502,13 +497,13 @@ def get_type(self, type_name: str) -> interfaces.objects.Template: table_name, type_name = type_name[:index], type_name[index + 1:] raise exceptions.SymbolError( type_name, table_name, - "Symbol for a different table requested: {}".format(table_name + constants.BANG + type_name)) + f"Symbol for a different table requested: {table_name + constants.BANG + type_name}") if type_name not in self._json_object['user_types']: # Fall back to the natives table if type_name in self.natives.types: return self.natives.get_type(self.name + constants.BANG + type_name) else: - raise exceptions.SymbolError(type_name, self.name, "Unknown symbol: {}".format(type_name)) + raise exceptions.SymbolError(type_name, self.name, f"Unknown symbol: {type_name}") curdict = self._json_object['user_types'][type_name] members = {} for member_name in curdict['fields']: @@ -536,18 +531,16 @@ def get_symbol(self, name: str) -> interfaces.symbols.SymbolInterface: return self._symbol_cache[name] symbol = self._json_object['symbols'].get(name, None) if not symbol: - raise exceptions.SymbolError(name, self.name, "Unknown symbol: {}".format(name)) + raise exceptions.SymbolError(name, self.name, f"Unknown symbol: {name}") + address = symbol['address'] + if self.config.get('symbol_mask', 0): + address = address & self.config['symbol_mask'] + symbol_type = None if 'type' in symbol: symbol_type = self._interdict_to_template(symbol['type']) - # Mask the addresses if necessary - address = symbol['address'] + self.config.get('symbol_shift', 0) - if self.config.get('symbol_mask', 0): - address = address & self.config['symbol_mask'] - self._symbol_cache[name] = interfaces.symbols.SymbolInterface(name = name, - address = address, - type = symbol_type) + self._symbol_cache[name] = interfaces.symbols.SymbolInterface(name = name, address = address, type = symbol_type) return self._symbol_cache[name] @@ -593,7 +586,11 @@ def get_symbol(self, name: str) -> interfaces.symbols.SymbolInterface: return self._symbol_cache[name] symbol = self._json_object['symbols'].get(name, None) if not symbol: - raise exceptions.SymbolError(name, self.name, "Unknown symbol: {}".format(name)) + raise exceptions.SymbolError(name, self.name, f"Unknown symbol: {name}") + address = symbol['address'] + if self.config.get('symbol_mask', 0): + address = address & self.config['symbol_mask'] + symbol_type = None if 'type' in symbol: symbol_type = self._interdict_to_template(symbol['type']) @@ -601,10 +598,6 @@ def get_symbol(self, name: str) -> interfaces.symbols.SymbolInterface: if 'constant_data' in symbol: symbol_constant_data = base64.b64decode(symbol.get('constant_data')) - # Mask the addresses if necessary - address = symbol['address'] + self.config.get('symbol_shift', 0) - if self.config.get('symbol_mask', 0): - address = address & self.config['symbol_mask'] self._symbol_cache[name] = interfaces.symbols.SymbolInterface(name = name, address = address, type = symbol_type, @@ -666,7 +659,7 @@ def get_type(self, type_name: str) -> interfaces.objects.Template: table_name, type_name = type_name[:index], type_name[index + 1:] raise exceptions.SymbolError( type_name, table_name, - "Symbol for a different table requested: {}".format(table_name + constants.BANG + type_name)) + f"Symbol for a different table requested: {table_name + constants.BANG + type_name}") type_definition = self._json_object['user_types'].get(type_name) if type_definition is None: diff --git a/volatility3/framework/symbols/linux/__init__.py b/volatility3/framework/symbols/linux/__init__.py index 0d8e99e420..36e23a35d7 100644 --- a/volatility3/framework/symbols/linux/__init__.py +++ b/volatility3/framework/symbols/linux/__init__.py @@ -3,11 +3,11 @@ # from typing import List, Tuple, Iterator -from volatility3.framework import exceptions, constants, interfaces, objects, contexts +from volatility3 import framework +from volatility3.framework import exceptions, constants, interfaces, objects from volatility3.framework.objects import utility from volatility3.framework.symbols import intermed from volatility3.framework.symbols.linux import extensions -from volatility3.framework.objects import utility class LinuxKernelIntermedSymbols(intermed.IntermediateSymbolTable): @@ -40,13 +40,16 @@ def __init__(self, *args, **kwargs) -> None: class LinuxUtilities(interfaces.configuration.VersionableInterface): """Class with multiple useful linux functions.""" - _version = (1, 0, 0) + _version = (2, 0, 0) + _required_framework_version = (2, 0, 0) + + framework.require_interface_version(*_required_framework_version) # based on __d_path from the Linux kernel @classmethod def _do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> str: - ret_path = [] # type: List[str] + ret_path: List[str] = [] while dentry != rdentry or vfsmnt != rmnt: dname = dentry.path() @@ -83,7 +86,7 @@ def _do_get_path(cls, rdentry, rmnt, dentry, vfsmnt) -> str: except exceptions.InvalidAddressException: ino = 0 - ret_val = ret_val[:-1] + ":[{0}]".format(ino) + ret_val = ret_val[:-1] + f":[{ino}]" else: ret_val = ret_val.replace("/", "") @@ -114,7 +117,13 @@ def _get_new_sock_pipe_path(cls, context, task, filp) -> str: if len(symbol_table_arr) == 2: symbol_table = symbol_table_arr[0] - symbs = list(context.symbol_space.get_symbols_by_location(sym_addr, table_name = symbol_table)) + for module_name in context.modules.get_modules_by_symbol_tables(symbol_table): + kernel_module = context.modules[module_name] + break + else: + raise ValueError(f"No module using the symbol table {symbol_table}") + + symbs = list(kernel_module.get_symbols_by_absolute_location(sym_addr)) if len(symbs) == 1: sym = symbs[0].split(constants.BANG)[1] @@ -132,12 +141,12 @@ def _get_new_sock_pipe_path(cls, context, task, filp) -> str: pre_name = cls._get_path_file(task, filp) else: - pre_name = "".format(sym) + pre_name = f"" - ret = "{0}:[{1:d}]".format(pre_name, dentry.d_inode.i_ino) + ret = f"{pre_name}:[{dentry.d_inode.i_ino:d}]" else: - ret = " {0:x}".format(sym_addr) + ret = f" {sym_addr:x}" return ret @@ -207,15 +216,15 @@ def mask_mods_list(cls, context: interfaces.context.ContextInterface, layer_name @classmethod def generate_kernel_handler_info( - cls, context: interfaces.context.ContextInterface, layer_name: str, kernel_name: str, + cls, context: interfaces.context.ContextInterface, kernel_module_name: str, mods_list: Iterator[interfaces.objects.ObjectInterface]) -> List[Tuple[str, int, int]]: """ A helper function that gets the beginning and end address of the kernel module """ - kernel = contexts.Module(context, kernel_name, layer_name, 0) + kernel = context.modules[kernel_module_name] - mask = context.layers[layer_name].address_mask + mask = context.layers[kernel.layer_name].address_mask start_addr = kernel.object_from_symbol("_text") start_addr = start_addr.vol.offset & mask @@ -224,10 +233,11 @@ def generate_kernel_handler_info( end_addr = end_addr.vol.offset & mask return [(constants.linux.KERNEL_NAME, start_addr, end_addr)] + \ - LinuxUtilities.mask_mods_list(context, layer_name, mods_list) + LinuxUtilities.mask_mods_list(context, kernel.layer_name, mods_list) @classmethod - def lookup_module_address(cls, context: interfaces.context.ContextInterface, handlers: List[Tuple[str, int, int]], + def lookup_module_address(cls, kernel_module: interfaces.context.ModuleInterface, + handlers: List[Tuple[str, int, int]], target_address: int): """ Searches between the start and end address of the kernel module using target_address. @@ -241,7 +251,7 @@ def lookup_module_address(cls, context: interfaces.context.ContextInterface, han if start <= target_address <= end: mod_name = name if name == constants.linux.KERNEL_NAME: - symbols = list(context.symbol_space.get_symbols_by_location(target_address)) + symbols = list(kernel_module.get_symbols_by_absolute_location(target_address)) if len(symbols): symbol_name = symbols[0].split(constants.BANG)[1] if constants.BANG in symbols[0] else \ diff --git a/volatility3/framework/symbols/linux/extensions/__init__.py b/volatility3/framework/symbols/linux/extensions/__init__.py index 5bcfda11ee..0edd60608f 100644 --- a/volatility3/framework/symbols/linux/extensions/__init__.py +++ b/volatility3/framework/symbols/linux/extensions/__init__.py @@ -108,7 +108,7 @@ def get_symbols(self): "linux", "elf", native_types = None, - class_types = extensions.elf.class_types) + class_types = elf.class_types) syms = self._context.object( self.get_symbol_table().name + constants.BANG + "array", @@ -181,7 +181,7 @@ def add_process_layer(self, config_prefix: str = None, preferred_name: str = Non return None if preferred_name is None: - preferred_name = self.vol.layer_name + "_Process{}".format(self.pid) + preferred_name = self.vol.layer_name + f"_Process{self.pid}" # Add the constructed layer and return the name return self._add_process_layer(self._context, dtb, config_prefix, preferred_name) @@ -197,7 +197,7 @@ def get_process_memory_sections(self, heap_only: bool = False) -> Generator[Tupl continue else: # FIXME: Check if this actually needs to be printed out or not - vollog.info("adding vma: {:x} {:x} | {:x} {:x}".format(start, self.mm.brk, end, self.mm.start_brk)) + vollog.info(f"adding vma: {start:x} {self.mm.brk:x} | {end:x} {self.mm.start_brk:x}") yield (start, end - start) @@ -403,7 +403,19 @@ def to_list(self, forward: bool = True, sentinel: bool = True, layer: Optional[str] = None) -> Iterator[interfaces.objects.ObjectInterface]: - """Returns an iterator of the entries in the list.""" + """Returns an iterator of the entries in the list. + + Args: + symbol_type: Type of the list elements + member: Name of the list_head member in the list elements + forward: Set false to go backwards + sentinel: Whether self is a "sentinel node", meaning it is not embedded in a member of the list + Sentinel nodes are NOT yielded. See https://en.wikipedia.org/wiki/Sentinel_node for further reference + layer: Name of layer to read from + Yields: + Objects of the type specified via the "symbol_type" argument. + + """ layer = layer or self.vol.layer_name relative_offset = self._context.symbol_space.get_type(symbol_type).relative_child_offset(member) @@ -496,7 +508,7 @@ def is_valid(self): def _get_real_mnt(self): table_name = self.vol.type_name.split(constants.BANG)[0] - mount_struct = "{0}{1}mount".format(table_name, constants.BANG) + mount_struct = f"{table_name}{constants.BANG}mount" offset = self._context.symbol_space.get_type(mount_struct).relative_child_offset("mnt") return self._context.object(mount_struct, self.vol.layer_name, offset = self.vol.offset - offset) diff --git a/volatility3/framework/symbols/linux/extensions/elf.py b/volatility3/framework/symbols/linux/extensions/elf.py index 0774e937ac..1277afe936 100644 --- a/volatility3/framework/symbols/linux/extensions/elf.py +++ b/volatility3/framework/symbols/linux/extensions/elf.py @@ -45,7 +45,7 @@ def __init__(self, context: interfaces.context.ContextInterface, type_name: str, elif ei_class == 2: self._type_prefix = "Elf64_" else: - raise ValueError("Unsupported ei_class value {}".format(ei_class)) + raise ValueError(f"Unsupported ei_class value {ei_class}") # Construct the full header self._hdr = self._context.object(symbol_table_name + constants.BANG + self._type_prefix + "Ehdr", diff --git a/volatility3/framework/symbols/mac/__init__.py b/volatility3/framework/symbols/mac/__init__.py index 54a5259d0a..7d094ea4f7 100644 --- a/volatility3/framework/symbols/mac/__init__.py +++ b/volatility3/framework/symbols/mac/__init__.py @@ -35,8 +35,10 @@ class MacUtilities(interfaces.configuration.VersionableInterface): Version History: 1.1.0 -> added walk_list_head API 1.2.0 -> added walk_slist API + 1.3.0 -> add parameter to lookup_module_address to pass kernel module name """ - _version = (1, 2, 0) + _version = (1, 3, 0) + _required_framework_version = (2, 0, 0) @classmethod def mask_mods_list(cls, context: interfaces.context.ContextInterface, layer_name: str, @@ -77,15 +79,20 @@ def generate_kernel_handler_info( @classmethod def lookup_module_address(cls, context: interfaces.context.ContextInterface, handlers: Iterator[Any], - target_address): + target_address, kernel_module_name: str = None): mod_name = "UNKNOWN" symbol_name = "N/A" + module_shift = 0 + if kernel_module_name: + module = context.modules[kernel_module_name] + module_shift = module.offset + for name, start, end in handlers: if start <= target_address <= end: mod_name = name if name == "__kernel__": - symbols = list(context.symbol_space.get_symbols_by_location(target_address)) + symbols = list(context.symbol_space.get_symbols_by_location(target_address - module_shift)) if len(symbols) > 0: symbol_name = str(symbols[0].split(constants.BANG)[1]) if constants.BANG in symbols[0] else \ @@ -149,7 +156,7 @@ def files_descriptors_for_process(cls, context: interfaces.context.ContextInterf vnode = f.f_fglob.fg_data.dereference().cast("vnode") path = vnode.full_path() elif ftype: - path = "<{}>".format(ftype.lower()) + path = f"<{ftype.lower()}>" yield f, path, fd_num @@ -160,7 +167,7 @@ def _walk_iterable(cls, list_next_member: str, next_member: str, max_elements: int = 4096) -> Iterable[interfaces.objects.ObjectInterface]: - seen = set() # type: Set[int] + seen: Set[int] = set() try: current = queue.member(attr = list_head_member) diff --git a/volatility3/framework/symbols/mac/extensions/__init__.py b/volatility3/framework/symbols/mac/extensions/__init__.py index 37d733af6d..94045d2e73 100644 --- a/volatility3/framework/symbols/mac/extensions/__init__.py +++ b/volatility3/framework/symbols/mac/extensions/__init__.py @@ -4,12 +4,15 @@ from typing import Generator, Iterable, Optional, Set, Tuple +import logging + from volatility3.framework import constants, objects, renderers from volatility3.framework import exceptions, interfaces from volatility3.framework.objects import utility from volatility3.framework.renderers import conversion from volatility3.framework.symbols import generic +vollog = logging.getLogger(__name__) class proc(generic.GenericIntelProcess): @@ -32,7 +35,7 @@ def add_process_layer(self, config_prefix: str = None, preferred_name: str = Non return None if preferred_name is None: - preferred_name = self.vol.layer_name + "_Process{}".format(self.p_pid) + preferred_name = self.vol.layer_name + f"_Process{self.p_pid}" # Add the constructed layer and return the name return self._add_process_layer(self._context, dtb, config_prefix, preferred_name) @@ -48,10 +51,18 @@ def get_map_iter(self) -> Iterable[interfaces.objects.ObjectInterface]: except exceptions.InvalidAddressException: return - seen = set() # type: Set[int] + seen: Set[int] = set() for i in range(task.map.hdr.nentries): - if not current_map or current_map.vol.offset in seen: + if (not current_map or + current_map.vol.offset in seen or + not self._context.layers[task.vol.native_layer_name].is_valid(current_map.dereference().vol.offset, current_map.dereference().vol.size)): + + vollog.log(constants.LOGLEVEL_VVV, "Breaking process maps iteration due to invalid state.") + break + + # ZP_POISON value used to catch programming errors + if current_map.links.start == 0xdeadbeefdeadbeef or current_map.links.end == 0xdeadbeefdeadbeef: break yield current_map @@ -120,7 +131,10 @@ def _do_calc_path(self, ret, vnodeobj, vname): return if vname: - ret.append(utility.pointer_to_string(vname, 255)) + try: + ret.append(utility.pointer_to_string(vname, 255)) + except exceptions.InvalidAddressException: + return if int(vnodeobj.v_flag) & 0x000001 != 0 and int(vnodeobj.v_mount) != 0: if int(vnodeobj.v_mount.mnt_vnodecovered) != 0: @@ -210,10 +224,21 @@ def get_path(self, context, config_prefix): ret = node elif node: path = [] - while node: - v_name = utility.pointer_to_string(node.v_name, 255) + seen: Set[int] = set() + while node and node.vol.offset not in seen: + try: + v_name = utility.pointer_to_string(node.v_name, 255) + except exceptions.InvalidAddressException: + break + path.append(v_name) + if len(path) > 1024: + break + + seen.add(node.vol.offset) + node = node.v_parent + path.reverse() ret = "/" + "/".join(path) else: @@ -243,9 +268,10 @@ def get_vnode(self, context, config_prefix): # based on find_vnode_object vnode_object = self.get_object().get_map_object() + if vnode_object == 0: + return None found_end = False - while not found_end: try: tmp_vnode_object = vnode_object.shadow.dereference() @@ -257,8 +283,15 @@ def get_vnode(self, context, config_prefix): else: vnode_object = tmp_vnode_object + if vnode_object.vol.offset == 0: + return None + try: - ops = vnode_object.pager.mo_pager_ops.dereference() + pager = vnode_object.pager + if pager == 0: + return None + + ops = pager.mo_pager_ops.dereference() except exceptions.InvalidAddressException: return None @@ -478,7 +511,7 @@ def __str__(self): e = e.cast("unsigned char") - ret = ret + "{:02X}:".format(e) + ret = ret + f"{e:02X}:" if ret and ret[-1] == ":": ret = ret[:-1] diff --git a/volatility3/framework/symbols/native.py b/volatility3/framework/symbols/native.py index fdc161322c..d9833e26dc 100644 --- a/volatility3/framework/symbols/native.py +++ b/volatility3/framework/symbols/native.py @@ -15,7 +15,7 @@ class NativeTable(interfaces.symbols.NativeTableInterface): def __init__(self, name: str, native_dictionary: Dict[str, Any]) -> None: super().__init__(name, self) self._native_dictionary = copy.deepcopy(native_dictionary) - self._overrides = {} # type: Dict[str, interfaces.objects.ObjectInterface] + self._overrides: Dict[str, interfaces.objects.ObjectInterface] = {} for native_type in self._native_dictionary: native_class, _native_struct = self._native_dictionary[native_type] self._overrides[native_type] = native_class @@ -45,12 +45,12 @@ def get_type(self, type_name: str) -> interfaces.objects.Template: if constants.BANG in type_name: name_split = type_name.split(constants.BANG) if len(name_split) > 2: - raise ValueError("SymbolName cannot contain multiple {} separators".format(constants.BANG)) + raise ValueError(f"SymbolName cannot contain multiple {constants.BANG} separators") table_name, type_name = name_split prefix = table_name + constants.BANG - additional = {} # type: Dict[str, Any] - obj = None # type: Optional[Type[interfaces.objects.ObjectInterface]] + additional: Dict[str, Any] = {} + obj: Optional[Type[interfaces.objects.ObjectInterface]] = None if type_name == 'void' or type_name == 'function': obj = objects.Void elif type_name == 'array': diff --git a/volatility3/framework/symbols/windows/bigpools-vista-x64.json b/volatility3/framework/symbols/windows/bigpools/bigpools-vista-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/bigpools-vista-x64.json rename to volatility3/framework/symbols/windows/bigpools/bigpools-vista-x64.json diff --git a/volatility3/framework/symbols/windows/bigpools-vista-x86.json b/volatility3/framework/symbols/windows/bigpools/bigpools-vista-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/bigpools-vista-x86.json rename to volatility3/framework/symbols/windows/bigpools/bigpools-vista-x86.json diff --git a/volatility3/framework/symbols/windows/bigpools-win10-x64.json b/volatility3/framework/symbols/windows/bigpools/bigpools-win10-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/bigpools-win10-x64.json rename to volatility3/framework/symbols/windows/bigpools/bigpools-win10-x64.json diff --git a/volatility3/framework/symbols/windows/bigpools-win10-x86.json b/volatility3/framework/symbols/windows/bigpools/bigpools-win10-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/bigpools-win10-x86.json rename to volatility3/framework/symbols/windows/bigpools/bigpools-win10-x86.json diff --git a/volatility3/framework/symbols/windows/bigpools-x64.json b/volatility3/framework/symbols/windows/bigpools/bigpools-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/bigpools-x64.json rename to volatility3/framework/symbols/windows/bigpools/bigpools-x64.json diff --git a/volatility3/framework/symbols/windows/bigpools-x86.json b/volatility3/framework/symbols/windows/bigpools/bigpools-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/bigpools-x86.json rename to volatility3/framework/symbols/windows/bigpools/bigpools-x86.json diff --git a/volatility3/framework/symbols/windows/crash64.json b/volatility3/framework/symbols/windows/crash64.json index 3a71346340..6106575e26 100644 --- a/volatility3/framework/symbols/windows/crash64.json +++ b/volatility3/framework/symbols/windows/crash64.json @@ -65,25 +65,25 @@ "offset": 40, "type": { "kind": "base", - "name": "unsigned long" + "name": "unsigned long long" } }, "MachineImageType": { - "offset": 44, + "offset": 48, "type": { "kind": "base", "name": "unsigned long" } }, "NumberProcessors": { - "offset": 48, + "offset": 52, "type": { "kind": "base", "name": "unsigned long" } }, "BugCheckCode": { - "offset": 60, + "offset": 56, "type": { "kind": "base", "name": "unsigned long" @@ -157,7 +157,7 @@ "name": "unsigned long long" } }, - "SystemUpTime": { + "SystemTime": { "offset": 4008, "type": { "kind": "base", @@ -175,7 +175,7 @@ } } }, - "SystemTime": { + "SystemUpTime": { "offset": 4144, "type": { "kind": "base", @@ -242,84 +242,6 @@ "kind": "struct", "size": 8192 }, - "_SUMMARY_DUMP64": { - "fields": { - "Signature": { - "offset": 0, - "type": { - "count": 4, - "kind": "array", - "subtype": { - "kind": "base", - "name": "unsigned char" - } - } - }, - "ValidDump": { - "offset": 4, - "type": { - "count": 4, - "kind": "array", - "subtype": { - "kind": "base", - "name": "unsigned char" - } - } - }, - "DumpOptions": { - "offset": 8, - "type": { - "kind": "base", - "name": "unsigned long" - } - }, - "HeaderSize": { - "offset": 32, - "type": { - "kind": "base", - "name": "unsigned long long" - } - }, - "BitmapSize": { - "offset": 40, - "type": { - "kind": "base", - "name": "unsigned long long" - } - }, - "Pages": { - "offset": 48, - "type": { - "kind": "base", - "name": "unsigned long long" - } - }, - "BufferLong": { - "offset": 56, - "type": { - "kind": "array", - "count": 1, - "subtype": { - "kind": "base", - "name": "unsigned long" - } - } - }, - "BufferChar": { - "offset": 56, - "type": { - "kind": "array", - "count": 1, - "subtype": { - "kind": "base", - "name": "unsigned char" - } - } - } - }, - "kind": "struct", - "size": 56 - }, "_EXCEPTION_RECORD64": { "fields": { "ExceptionCode": { diff --git a/volatility3/framework/symbols/windows/crash_common.json b/volatility3/framework/symbols/windows/crash_common.json new file mode 100644 index 0000000000..fe29b20a82 --- /dev/null +++ b/volatility3/framework/symbols/windows/crash_common.json @@ -0,0 +1,138 @@ +{ + "symbols": { + }, + "user_types": { + "_SUMMARY_DUMP": { + "fields": { + "Signature": { + "offset": 0, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "ValidDump": { + "offset": 4, + "type": { + "count": 4, + "kind": "array", + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + }, + "DumpOptions": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "HeaderSize": { + "offset": 32, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "Pages": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "BitmapSize": { + "offset": 48, + "type": { + "kind": "base", + "name": "unsigned long long" + } + }, + "BufferLong": { + "offset": 56, + "type": { + "kind": "array", + "count": 1, + "subtype": { + "kind": "base", + "name": "unsigned long" + } + } + }, + "BufferChar": { + "offset": 56, + "type": { + "kind": "array", + "count": 1, + "subtype": { + "kind": "base", + "name": "unsigned char" + } + } + } + }, + "kind": "struct", + "size": 56 + } + }, + "enums": { + }, + "base_types": { + "unsigned char": { + "endian": "little", + "kind": "char", + "signed": false, + "size": 1 + }, + "unsigned short": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 2 + }, + "long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 4 + }, + "char": { + "endian": "little", + "kind": "char", + "signed": true, + "size": 1 + }, + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "long long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 8 + }, + "unsigned long long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 8 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "ikelos-by-hand", + "datetime": "2020-09-10T00:20:00" + }, + "format": "6.2.0" + } +} diff --git a/volatility3/framework/symbols/windows/extensions/__init__.py b/volatility3/framework/symbols/windows/extensions/__init__.py index f402c800f6..ae7c45d04c 100755 --- a/volatility3/framework/symbols/windows/extensions/__init__.py +++ b/volatility3/framework/symbols/windows/extensions/__init__.py @@ -94,7 +94,7 @@ def traverse(self, visited = None, depth = 0): # any node other than the root that doesn't have a recognized tag # is just garbage and we skip the node entirely vollog.log(constants.LOGLEVEL_VVV, - "Skipping VAD at {} depth {} with tag {}".format(self.vol.offset, depth, tag)) + f"Skipping VAD at {self.vol.offset} depth {depth} with tag {tag}") return if target: @@ -105,13 +105,13 @@ def traverse(self, visited = None, depth = 0): for vad_node in self.get_left_child().dereference().traverse(visited, depth + 1): yield vad_node except exceptions.InvalidAddressException as excp: - vollog.log(constants.LOGLEVEL_VVV, "Invalid address on LeftChild: {0:#x}".format(excp.invalid_address)) + vollog.log(constants.LOGLEVEL_VVV, f"Invalid address on LeftChild: {excp.invalid_address:#x}") try: for vad_node in self.get_right_child().dereference().traverse(visited, depth + 1): yield vad_node except exceptions.InvalidAddressException as excp: - vollog.log(constants.LOGLEVEL_VVV, "Invalid address on RightChild: {0:#x}".format(excp.invalid_address)) + vollog.log(constants.LOGLEVEL_VVV, f"Invalid address on RightChild: {excp.invalid_address:#x}") def get_right_child(self): """Get the right child member.""" @@ -329,7 +329,7 @@ class EX_FAST_REF(objects.StructType): def dereference(self) -> interfaces.objects.ObjectInterface: if constants.BANG not in self.vol.type_name: - raise ValueError("Invalid symbol table name syntax (no {} found)".format(constants.BANG)) + raise ValueError(f"Invalid symbol table name syntax (no {constants.BANG} found)") # the mask value is different on 32 and 64 bits symbol_table_name = self.vol.type_name.split(constants.BANG)[0] @@ -388,13 +388,13 @@ def is_valid(self) -> bool: self.FileName.Buffer) def file_name_with_device(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: - name = renderers.UnreadableValue() # type: Union[str, interfaces.renderers.BaseAbsentValue] + name: Union[str, interfaces.renderers.BaseAbsentValue] = renderers.UnreadableValue() # this pointer needs to be checked against native_layer_name because the object may # be instantiated from a primary (virtual) layer or a memory (physical) layer. if self._context.layers[self.vol.native_layer_name].is_valid(self.DeviceObject): try: - name = "\\Device\\{}".format(self.DeviceObject.get_device_name()) + name = f"\\Device\\{self.DeviceObject.get_device_name()}" except ValueError: pass @@ -449,7 +449,7 @@ def get_cross_thread_flags(self) -> str: stringCrossThreadFlags = '' for flag in dictCrossThreadFlags: if flags & 2 ** dictCrossThreadFlags[flag]: - stringCrossThreadFlags += '{} '.format(flag) + stringCrossThreadFlags += f'{flag} ' return stringCrossThreadFlags[:-1] if stringCrossThreadFlags else stringCrossThreadFlags @@ -526,7 +526,7 @@ def add_process_layer(self, config_prefix: str = None, preferred_name: str = Non raise TypeError("Parent layer is not a translation layer, unable to construct process layer") # Presumably for 64-bit systems, the DTB is defined as an array, rather than an unsigned long long - dtb = 0 # type: int + dtb: int = 0 if isinstance(self.Pcb.DirectoryTableBase, objects.Array): dtb = self.Pcb.DirectoryTableBase.cast("unsigned long long") else: @@ -534,7 +534,7 @@ def add_process_layer(self, config_prefix: str = None, preferred_name: str = Non dtb = dtb & ((1 << parent_layer.bits_per_register) - 1) if preferred_name is None: - preferred_name = self.vol.layer_name + "_Process{}".format(self.UniqueProcessId) + preferred_name = self.vol.layer_name + f"_Process{self.UniqueProcessId}" # Add the constructed layer and return the name return self._add_process_layer(self._context, dtb, config_prefix, preferred_name) @@ -542,7 +542,7 @@ def add_process_layer(self, config_prefix: str = None, preferred_name: str = Non def get_peb(self) -> interfaces.objects.ObjectInterface: """Constructs a PEB object""" if constants.BANG not in self.vol.type_name: - raise ValueError("Invalid symbol table name syntax (no {} found)".format(constants.BANG)) + raise ValueError(f"Invalid symbol table name syntax (no {constants.BANG} found)") # add_process_layer can raise InvalidAddressException. # if that happens, we let the exception propagate upwards @@ -551,10 +551,10 @@ def get_peb(self) -> interfaces.objects.ObjectInterface: proc_layer = self._context.layers[proc_layer_name] if not proc_layer.is_valid(self.Peb): raise exceptions.InvalidAddressException(proc_layer_name, self.Peb, - "Invalid address at {:0x}".format(self.Peb)) + f"Invalid address at {self.Peb:0x}") sym_table = self.vol.type_name.split(constants.BANG)[0] - peb = self._context.object("{}{}_PEB".format(sym_table, constants.BANG), + peb = self._context.object(f"{sym_table}{constants.BANG}_PEB", layer_name = proc_layer_name, offset = self.Peb) return peb @@ -565,7 +565,7 @@ def load_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: try: peb = self.get_peb() for entry in peb.Ldr.InLoadOrderModuleList.to_list( - "{}{}_LDR_DATA_TABLE_ENTRY".format(self.get_symbol_table_name(), constants.BANG), + f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InLoadOrderLinks"): yield entry except exceptions.InvalidAddressException: @@ -577,7 +577,7 @@ def init_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: try: peb = self.get_peb() for entry in peb.Ldr.InInitializationOrderModuleList.to_list( - "{}{}_LDR_DATA_TABLE_ENTRY".format(self.get_symbol_table_name(), constants.BANG), + f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InInitializationOrderLinks"): yield entry except exceptions.InvalidAddressException: @@ -589,7 +589,7 @@ def mem_order_modules(self) -> Iterable[interfaces.objects.ObjectInterface]: try: peb = self.get_peb() for entry in peb.Ldr.InMemoryOrderModuleList.to_list( - "{}{}_LDR_DATA_TABLE_ENTRY".format(self.get_symbol_table_name(), constants.BANG), + f"{self.get_symbol_table_name()}{constants.BANG}_LDR_DATA_TABLE_ENTRY", "InMemoryOrderLinks"): yield entry except exceptions.InvalidAddressException: @@ -603,7 +603,7 @@ def get_handle_count(self): except exceptions.InvalidAddressException: vollog.log(constants.LOGLEVEL_VVV, - "Cannot access _EPROCESS.ObjectTable.HandleCount at {0:#x}".format(self.vol.offset)) + f"Cannot access _EPROCESS.ObjectTable.HandleCount at {self.vol.offset:#x}") return renderers.UnreadableValue() @@ -626,7 +626,7 @@ def get_session_id(self): except exceptions.InvalidAddressException: vollog.log(constants.LOGLEVEL_VVV, - "Cannot access _EPROCESS.Session.SessionId at {0:#x}".format(self.vol.offset)) + f"Cannot access _EPROCESS.Session.SessionId at {self.vol.offset:#x}") return renderers.UnreadableValue() @@ -972,7 +972,12 @@ def get_available_pages(self) -> Iterable[Tuple[int, int, int]]: # If the entry is not a valid physical address then see if it is in transition. elif mmpte.u.Trans.Transition == 1: - physoffset = mmpte.u.Trans.PageFrameNumber << 12 + # TODO: Fix appropriately in a future release. + # Currently just a temporary workaround to deal with custom bit flag + # in the PFN field for pages in transition state. + # See https://github.com/volatilityfoundation/volatility3/pull/475 + physoffset = (mmpte.u.Trans.PageFrameNumber & (( 1 << 33 ) - 1 ) ) << 12 + yield physoffset, file_offset, self.PAGE_SIZE # Go to the next PTE entry @@ -1097,7 +1102,7 @@ def get_available_pages(self) -> List: if vacb_obj.SharedCacheMap == self.vol.offset: self.save_vacb(vacb_obj, vacb_list) - # If the file is larger than 1 MB, a seperate VACB index array needs to be allocated. + # If the file is larger than 1 MB, a separate VACB index array needs to be allocated. # This is based on how many 256 KB blocks would be required for the size of the file. # This newly allocated VACB index array is found through the Vacbs member of SHARED_CACHE_MAP. vacb_obj = self.Vacbs diff --git a/volatility3/framework/symbols/windows/extensions/crash.py b/volatility3/framework/symbols/windows/extensions/crash.py new file mode 100644 index 0000000000..8d8200aebf --- /dev/null +++ b/volatility3/framework/symbols/windows/extensions/crash.py @@ -0,0 +1,27 @@ +# This file is Copyright 2021 Volatility Foundation and licensed under the Volatility Software License 1.0 +# which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 +# + +from volatility3.framework import interfaces, constants +from volatility3.framework import objects + + +class SUMMARY_DUMP(objects.StructType): + + def get_buffer(self, sub_type: str, count: int) -> interfaces.objects.ObjectInterface: + symbol_table_name = self.get_symbol_table_name() + subtype = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + sub_type) + return self._context.object(object_type = symbol_table_name + constants.BANG + "array", + layer_name = self.vol.layer_name, + offset = self.BufferChar.vol.offset, + count = count, + subtype = subtype) + + def get_buffer_char(self) -> interfaces.objects.ObjectInterface: + return self.get_buffer(sub_type = "unsigned char", count = (self.BitmapSize + 7) // 8) + + def get_buffer_long(self) -> interfaces.objects.ObjectInterface: + return self.get_buffer(sub_type = "unsigned long", count = (self.BitmapSize + 31) // 32) + + +class_types = {'_SUMMARY_DUMP': SUMMARY_DUMP} diff --git a/volatility3/framework/symbols/windows/extensions/network.py b/volatility3/framework/symbols/windows/extensions/network.py index 78b4c77d70..6e13ad45ba 100644 --- a/volatility3/framework/symbols/windows/extensions/network.py +++ b/volatility3/framework/symbols/windows/extensions/network.py @@ -164,7 +164,7 @@ def is_valid(self): return False except exceptions.InvalidAddressException: - vollog.debug("netw obj 0x{:x} invalid due to invalid address access".format(self.vol.offset)) + vollog.debug(f"netw obj 0x{self.vol.offset:x} invalid due to invalid address access") return False return True @@ -200,21 +200,21 @@ def get_remote_address(self): def is_valid(self): if self.State not in self.State.choices.values(): - vollog.debug("invalid due to invalid tcp state {}".format(self.State)) + vollog.debug(f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid tcp state {self.State}") return False try: if self.get_address_family() not in (AF_INET, AF_INET6): - vollog.debug("invalid due to invalid address_family {}".format(self.get_address_family())) + vollog.debug(f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid address_family {self.get_address_family()}") return False if not self.get_local_address() and (not self.get_owner() or self.get_owner().UniqueProcessId == 0 or self.get_owner().UniqueProcessId > 65535): - vollog.debug("invalid due to invalid owner data") + vollog.debug(f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid owner data") return False except exceptions.InvalidAddressException: - vollog.debug("invalid due to invalid address access") + vollog.debug(f"{type(self)} 0x{self.vol.offset:x} invalid due to invalid address access") return False return True diff --git a/volatility3/framework/symbols/windows/extensions/pe.py b/volatility3/framework/symbols/windows/extensions/pe.py index b7b6d71df2..df461318f2 100644 --- a/volatility3/framework/symbols/windows/extensions/pe.py +++ b/volatility3/framework/symbols/windows/extensions/pe.py @@ -22,7 +22,7 @@ def get_nt_header(self) -> interfaces.objects.ObjectInterface: """ if self.e_magic != 0x5a4d: - raise ValueError("e_magic {0:04X} is not a valid DOS signature.".format(self.e_magic)) + raise ValueError(f"e_magic {self.e_magic:04X} is not a valid DOS signature.") layer_name = self.vol.layer_name symbol_table_name = self.get_symbol_table_name() @@ -32,7 +32,7 @@ def get_nt_header(self) -> interfaces.objects.ObjectInterface: offset = self.vol.offset + self.e_lfanew) if nt_header.Signature != 0x4550: - raise ValueError("NT header signature {0:04X} is not a valid".format(nt_header.Signature)) + raise ValueError(f"NT header signature {nt_header.Signature:04X} is not a valid") # this checks if we need a PE32+ header if nt_header.FileHeader.Machine == 34404: @@ -110,7 +110,7 @@ def reconstruct(self) -> Generator[Tuple[int, bytes], None, None]: # no legitimate PE is going to be larger than this if size_of_image > (1024 * 1024 * 100): - raise ValueError("The claimed SizeOfImage is too large: {}".format(size_of_image)) + raise ValueError(f"The claimed SizeOfImage is too large: {size_of_image}") read_layer = self._context.layers[layer_name] @@ -127,13 +127,13 @@ def reconstruct(self) -> Generator[Tuple[int, bytes], None, None]: for sect in nt_header.get_sections(): if sect.VirtualAddress > size_of_image: - raise ValueError("Section VirtualAddress is too large: {}".format(sect.VirtualAddress)) + raise ValueError(f"Section VirtualAddress is too large: {sect.VirtualAddress}") if sect.Misc.VirtualSize > size_of_image: - raise ValueError("Section VirtualSize is too large: {}".format(sect.Misc.VirtualSize)) + raise ValueError(f"Section VirtualSize is too large: {sect.Misc.VirtualSize}") if sect.SizeOfRawData > size_of_image: - raise ValueError("Section SizeOfRawData is too large: {}".format(sect.SizeOfRawData)) + raise ValueError(f"Section SizeOfRawData is too large: {sect.SizeOfRawData}") if sect is not None: # It doesn't matter if this is too big, because it'll get overwritten by the later layers diff --git a/volatility3/framework/symbols/windows/extensions/pool.py b/volatility3/framework/symbols/windows/extensions/pool.py index 8955338400..f75c6c417c 100644 --- a/volatility3/framework/symbols/windows/extensions/pool.py +++ b/volatility3/framework/symbols/windows/extensions/pool.py @@ -5,6 +5,7 @@ from volatility3.framework import objects, interfaces, constants, symbols, exceptions, renderers from volatility3.framework.renderers import conversion +from volatility3.plugins.windows.poolscanner import PoolConstraint vollog = logging.getLogger(__name__) @@ -17,23 +18,25 @@ class POOL_HEADER(objects.StructType): """ def get_object(self, - type_name: str, + constraint: PoolConstraint, use_top_down: bool, - executive: bool = False, kernel_symbol_table: Optional[str] = None, native_layer_name: Optional[str] = None) -> Optional[interfaces.objects.ObjectInterface]: """Carve an object or data structure from a kernel pool allocation Args: - type_name: the data structure type name - native_layer_name: the name of the layer where the data originally lived - object_type: the object type (executive kernel objects only) + constraint: a PoolConstraint object used to get the pool allocation header object + use_top_down: for delineating how a windows version finds the size of the object body kernel_symbol_table: in case objects of a different symbol table are scanned for + native_layer_name: the name of the layer where the data originally lived Returns: An object as found from a POOL_HEADER """ + type_name = constraint.type_name + executive = constraint.object_type is not None + symbol_table_name = self.vol.type_name.split(constants.BANG)[0] if constants.BANG in type_name: symbol_table_name, type_name = type_name.split(constants.BANG)[0:2] @@ -56,7 +59,7 @@ def get_object(self, layer_name = self.vol.layer_name, offset = self.vol.offset + pool_header_size, native_layer_name = native_layer_name) - return mem_object + yield mem_object # otherwise we have an executive object in the pool else: @@ -142,7 +145,7 @@ def get_object(self, native_layer_name = native_layer_name) if mem_object.is_valid(): - return mem_object + yield mem_object except (TypeError, exceptions.InvalidAddressException): pass @@ -150,6 +153,10 @@ def get_object(self, # use the bottom up approach for windows 7 and earlier else: type_size = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + type_name).size + if constraint.additional_structures: + for additional_structure in constraint.additional_structures: + type_size += self._context.symbol_space.get_type(symbol_table_name + constants.BANG + additional_structure).size + rounded_size = conversion.round(type_size, alignment, up = True) mem_object = self._context.object(symbol_table_name + constants.BANG + type_name, @@ -159,10 +166,9 @@ def get_object(self, try: if mem_object.is_valid(): - return mem_object + yield mem_object except (TypeError, exceptions.InvalidAddressException): - return None - return None + pass @classmethod @functools.lru_cache() @@ -175,7 +181,7 @@ def _calculate_optional_header_lengths(cls, context: interfaces.context.ContextI 'HANDLE_REVOCATION_INFO', 'PADDING_INFO' ]: try: - type_name = "{}{}_OBJECT_HEADER_{}".format(symbol_table_name, constants.BANG, header) + type_name = f"{symbol_table_name}{constants.BANG}_OBJECT_HEADER_{header}" header_type = context.symbol_space.get_type(type_name) headers.append(header) sizes.append(header_type.size) @@ -214,7 +220,7 @@ def is_nonpaged_pool(self): class POOL_TRACKER_BIG_PAGES(objects.StructType): """A kernel big page pool tracker.""" - pool_type_lookup = {} # type: Dict[str, str] + pool_type_lookup: Dict[str, str] = {} def _generate_pool_type_lookup(self): # Enumeration._generate_inverse_choices() raises ValueError because multiple enum names map to the same @@ -240,7 +246,7 @@ def get_pool_type(self) -> Union[str, interfaces.renderers.BaseAbsentValue]: if hasattr(self, 'PoolType'): if not self.pool_type_lookup: self._generate_pool_type_lookup() - return self.pool_type_lookup.get(self.PoolType, "Unknown choice {}".format(self.PoolType)) + return self.pool_type_lookup.get(self.PoolType, f"Unknown choice {self.PoolType}") else: return renderers.NotApplicableValue() @@ -259,7 +265,7 @@ class ExecutiveObject(interfaces.objects.ObjectInterface): def get_object_header(self) -> 'OBJECT_HEADER': if constants.BANG not in self.vol.type_name: - raise ValueError("Invalid symbol table name syntax (no {} found)".format(constants.BANG)) + raise ValueError(f"Invalid symbol table name syntax (no {constants.BANG} found)") symbol_table_name = self.vol.type_name.split(constants.BANG)[0] body_offset = self._context.symbol_space.get_type(symbol_table_name + constants.BANG + "_OBJECT_HEADER").relative_child_offset("Body") @@ -315,7 +321,7 @@ def get_object_type(self, type_map: Dict[int, str], cookie: int = None) -> Optio @property def NameInfo(self) -> interfaces.objects.ObjectInterface: if constants.BANG not in self.vol.type_name: - raise ValueError("Invalid symbol table name syntax (no {} found)".format(constants.BANG)) + raise ValueError(f"Invalid symbol table name syntax (no {constants.BANG} found)") symbol_table_name = self.vol.type_name.split(constants.BANG)[0] @@ -329,7 +335,7 @@ def NameInfo(self) -> interfaces.objects.ObjectInterface: kvo = layer.config.get("kernel_virtual_offset", None) if kvo is None: - raise AttributeError("Could not find kernel_virtual_offset for layer: {}".format(self.vol.layer_name)) + raise AttributeError(f"Could not find kernel_virtual_offset for layer: {self.vol.layer_name}") ntkrnlmp = self._context.module(symbol_table_name, layer_name = self.vol.layer_name, offset = kvo) address = ntkrnlmp.get_symbol("ObpInfoMaskToOffset").address diff --git a/volatility3/framework/symbols/windows/extensions/registry.py b/volatility3/framework/symbols/windows/extensions/registry.py index 3cf2469ff5..cbd9052cdd 100644 --- a/volatility3/framework/symbols/windows/extensions/registry.py +++ b/volatility3/framework/symbols/windows/extensions/registry.py @@ -30,23 +30,9 @@ class RegValueTypes(enum.Enum): REG_QWORD = 11 REG_UNKNOWN = 99999 - # TODO: This _missing_() method can replace the get() method below - # if support for Python 3.6 is added in the future - # @classmethod - # def _missing_(cls, value): - # return cls(RegValueTypes.REG_UNKNOWN) - @classmethod - def get(cls, value): - """An alternative method for using this enum when the value may be - unknown. - - This is used to support unknown value requests in Python <3.6. - """ - try: - return cls(value) - except ValueError: - return cls(RegValueTypes.REG_UNKNOWN) + def _missing_(cls, value): + return cls(RegValueTypes.REG_UNKNOWN) class RegKeyFlags(enum.IntEnum): @@ -180,13 +166,13 @@ def _get_subkeys_recursive( for subnode_offset in node.List[::listjump]: if (subnode_offset & 0x7fffffff) > hive.maximum_address: vollog.log(constants.LOGLEVEL_VVV, - "Node found with address outside the valid Hive size: {}".format(hex(subnode_offset))) + f"Node found with address outside the valid Hive size: {hex(subnode_offset)}") else: try: subnode = hive.get_node(subnode_offset) except (exceptions.InvalidAddressException, RegistryFormatException): vollog.log(constants.LOGLEVEL_VVV, - "Failed to get node at {}, skipping".format(hex(subnode_offset))) + f"Failed to get node at {hex(subnode_offset)}, skipping") continue yield from self._get_subkeys_recursive(hive, subnode) @@ -204,12 +190,12 @@ def get_values(self) -> Iterable[interfaces.objects.ObjectInterface]: try: node = hive.get_node(v) except (RegistryInvalidIndex, RegistryFormatException) as excp: - vollog.debug("Invalid address {}".format(excp)) + vollog.debug(f"Invalid address {excp}") continue if node.vol.type_name.endswith(constants.BANG + '_CM_KEY_VALUE'): yield node except (exceptions.InvalidAddressException, RegistryFormatException) as excp: - vollog.debug("Invalid address in get_values iteration: {}".format(excp)) + vollog.debug(f"Invalid address in get_values iteration: {excp}") return def get_name(self) -> interfaces.objects.ObjectInterface: @@ -254,7 +240,7 @@ def decode_data(self) -> Union[int, bytes]: # Remove the high bit datalen = datalen & 0x7fffffff if (0 > datalen or datalen > 4): - raise ValueError("Unable to read inline registry value with excessive length: {}".format(datalen)) + raise ValueError(f"Unable to read inline registry value with excessive length: {datalen}") else: data = layer.read(self.Data.vol.offset, datalen) elif layer.hive.Version == 5 and datalen > 0x4000: @@ -274,18 +260,18 @@ def decode_data(self) -> Union[int, bytes]: # but the length at the start could be negative so just adding 4 to jump past it data = layer.read(self.Data + 4, datalen) - self_type = RegValueTypes.get(self.Type) + self_type = RegValueTypes(self.Type) if self_type == RegValueTypes.REG_DWORD: if len(data) != struct.calcsize("L"): - raise ValueError("Size of data does not match the type of registry value {}".format(self.get_name())) + raise ValueError(f"Size of data does not match the type of registry value {self.get_name()}") return struct.unpack(">L", data)[0] if self_type == RegValueTypes.REG_QWORD: if len(data) != struct.calcsize(" Union[int, bytes]: return b'' # Fall back if it's something weird - vollog.debug("Unknown registry value type encountered: {}".format(self.Type)) + vollog.debug(f"Unknown registry value type encountered: {self.Type}") return data diff --git a/volatility3/framework/symbols/windows/kerb_ecrypt.json b/volatility3/framework/symbols/windows/kerb_ecrypt.json new file mode 100644 index 0000000000..95e1a1d6a4 --- /dev/null +++ b/volatility3/framework/symbols/windows/kerb_ecrypt.json @@ -0,0 +1,97 @@ +{ + "metadata": { + "producer": { + "version": "0.0.1", + "name": "acase-by-hand-from-mimikatz", + "datetime": "2021-03-01T14:30:00.000000" + }, + "format": "6.2.0" + }, + "symbols": { + }, + "enums": { + }, + "user_types": { + "_KERB_ECRYPT": { + "fields": { + "EncryptionType": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "BlockSize": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "KeySize": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "Initialize": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + }, + "Encrypt": { + "offset": 48, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + }, + "Decrypt": { + "offset": 56, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + }, + "Finish": { + "offset": 64, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 128 + } + }, + "base_types": { + "unsigned long": { + "endian": "little", + "kind": "int", + "signed": false, + "size": 4 + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + } + } +} diff --git a/volatility3/framework/symbols/windows/netscan-win10-15063-x64.json b/volatility3/framework/symbols/windows/netscan-win10-15063-x64.json deleted file mode 100644 index c212ea6be1..0000000000 --- a/volatility3/framework/symbols/windows/netscan-win10-15063-x64.json +++ /dev/null @@ -1,388 +0,0 @@ -{ - "base_types": { - "unsigned long": { - "kind": "int", - "size": 4, - "signed": false, - "endian": "little" - }, - "unsigned char": { - "kind": "char", - "size": 1, - "signed": false, - "endian": "little" - }, - "pointer": { - "kind": "int", - "size": 8, - "signed": false, - "endian": "little" - }, - "unsigned int": { - "kind": "int", - "size": 4, - "signed": false, - "endian": "little" - }, - "unsigned short": { - "kind": "int", - "size": 2, - "signed": false, - "endian": "little" - }, - "unsigned be short": { - "kind": "int", - "size": 2, - "signed": false, - "endian": "big" - }, - "long long": { - "endian": "little", - "kind": "int", - "signed": true, - "size": 8 - }, - "long": { - "kind": "int", - "size": 4, - "signed": false, - "endian": "little" - } - }, - "symbols": {}, - "user_types": { - "_UDP_ENDPOINT": { - "fields": { - "Owner": { - "offset": 40, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - - } - }, - "CreateTime": { - "offset": 88, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "LocalAddr": { - "offset": 128, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS_WIN10_UDP" - } - } - }, - "InetAF": { - "offset": 32, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - - } - }, - "Port": { - "offset": 120, - "type": { - "kind": "base", - "name": "unsigned be short" - } - } - }, - "kind": "struct", - "size": 132 - }, - "_TCP_LISTENER": { - "fields": { - "Owner": { - "offset": 48, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - - } - }, - "CreateTime": { - "offset": 64, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "LocalAddr": { - "offset": 96, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS" - } - - } - }, - "InetAF": { - "offset": 40, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - - } - }, - "Port": { - "offset": 114, - "type": { - "kind": "base", - "name": "unsigned be short" - } - } - }, - "kind": "struct", - "size": 116 - }, - "_TCP_ENDPOINT": { - "fields": { - "Owner": { - "offset": 624, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - } - }, - "CreateTime": { - "offset": 616, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "AddrInfo": { - "offset": 24, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_ADDRINFO" - } - } - }, - "InetAF": { - "offset": 16, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - } - }, - "LocalPort": { - "offset": 112, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, - "RemotePort": { - "offset": 114, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, - "State": { - "offset": 108, - "type": { - "kind": "enum", - "name": "TCPStateEnum" - } - } - }, - "kind": "struct", - "size": 632 - }, - "_LOCAL_ADDRESS": { - "fields": { - "pData": { - "offset": 16, - "type": { - "kind": "pointer", - "subtype": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - } - }, - "kind": "struct", - "size": 20 - }, - "_LOCAL_ADDRESS_WIN10_UDP": { - "fields": { - "pData": { - "offset": 0, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - }, - "kind": "struct", - "size": 4 - }, - "_ADDRINFO": { - "fields": { - "Local": { - "offset": 0, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS" - } - } - }, - "Remote": { - "offset": 16, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - }, - "kind": "struct", - "size": 4 - }, - "_IN_ADDR": { - "fields": { - "addr4": { - "offset": 0, - "type": { - "count": 4, - "subtype": { - "kind": "base", - "name": "unsigned char" - }, - "kind": "array" - } - }, - "addr6": { - "offset": 0, - "type": { - "count": 16, - "subtype": { - "kind": "base", - "name": "unsigned char" - }, - "kind": "array" - } - } - }, - "kind": "struct", - "size": 6 - }, - "_INETAF": { - "fields": { - "AddressFamily": { - "offset": 24, - "type": { - "kind": "base", - "name": "unsigned short" - } - } - }, - "kind": "struct", - "size": 26 - }, - "_LARGE_INTEGER": { - "fields": { - "HighPart": { - "offset": 4, - "type": { - "kind": "base", - "name": "long" - } - }, - "LowPart": { - "offset": 0, - "type": { - "kind": "base", - "name": "unsigned long" - } - }, - "QuadPart": { - "offset": 0, - "type": { - "kind": "base", - "name": "long long" - } - }, - "u": { - "offset": 0, - "type": { - "kind": "struct", - "name": "__unnamed_2" - } - } - }, - "kind": "union", - "size": 8 - } - }, - "enums": { - "TCPStateEnum": { - "base": "long", - "constants": { - "CLOSED": 0, - "LISTENING": 1, - "SYN_SENT": 2, - "SYN_RCVD": 3, - "ESTABLISHED": 4, - "FIN_WAIT1": 5, - "FIN_WAIT2": 6, - "CLOSE_WAIT": 7, - "CLOSING": 8, - "LAST_ACK": 9, - "TIME_WAIT": 12, - "DELETE_TCB": 13 - }, - "size": 4 - } - }, - "metadata": { - "producer": { - "version": "0.0.1", - "name": "japhlange-by-hand", - "datetime": "2020-05-29T19:28:34" - }, - "format": "6.0.0" - } -} diff --git a/volatility3/framework/symbols/windows/netscan-win10-17134-x64.json b/volatility3/framework/symbols/windows/netscan-win10-17134-x64.json deleted file mode 100644 index c0890845ab..0000000000 --- a/volatility3/framework/symbols/windows/netscan-win10-17134-x64.json +++ /dev/null @@ -1,388 +0,0 @@ -{ - "base_types": { - "unsigned long": { - "kind": "int", - "size": 4, - "signed": false, - "endian": "little" - }, - "unsigned char": { - "kind": "char", - "size": 1, - "signed": false, - "endian": "little" - }, - "pointer": { - "kind": "int", - "size": 8, - "signed": false, - "endian": "little" - }, - "unsigned int": { - "kind": "int", - "size": 4, - "signed": false, - "endian": "little" - }, - "unsigned short": { - "kind": "int", - "size": 2, - "signed": false, - "endian": "little" - }, - "unsigned be short": { - "kind": "int", - "size": 2, - "signed": false, - "endian": "big" - }, - "long long": { - "endian": "little", - "kind": "int", - "signed": true, - "size": 8 - }, - "long": { - "kind": "int", - "size": 4, - "signed": false, - "endian": "little" - } - }, - "symbols": {}, - "user_types": { - "_UDP_ENDPOINT": { - "fields": { - "Owner": { - "offset": 40, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - - } - }, - "CreateTime": { - "offset": 88, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "LocalAddr": { - "offset": 128, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS_WIN10_UDP" - } - } - }, - "InetAF": { - "offset": 32, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - - } - }, - "Port": { - "offset": 120, - "type": { - "kind": "base", - "name": "unsigned be short" - } - } - }, - "kind": "struct", - "size": 132 - }, - "_TCP_LISTENER": { - "fields": { - "Owner": { - "offset": 48, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - - } - }, - "CreateTime": { - "offset": 64, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "LocalAddr": { - "offset": 96, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS" - } - - } - }, - "InetAF": { - "offset": 40, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - - } - }, - "Port": { - "offset": 114, - "type": { - "kind": "base", - "name": "unsigned be short" - } - } - }, - "kind": "struct", - "size": 116 - }, - "_TCP_ENDPOINT": { - "fields": { - "Owner": { - "offset": 632, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - } - }, - "CreateTime": { - "offset": 648, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "AddrInfo": { - "offset": 24, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_ADDRINFO" - } - } - }, - "InetAF": { - "offset": 16, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - } - }, - "LocalPort": { - "offset": 112, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, - "RemotePort": { - "offset": 114, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, - "State": { - "offset": 108, - "type": { - "kind": "enum", - "name": "TCPStateEnum" - } - } - }, - "kind": "struct", - "size": 632 - }, - "_LOCAL_ADDRESS": { - "fields": { - "pData": { - "offset": 16, - "type": { - "kind": "pointer", - "subtype": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - } - }, - "kind": "struct", - "size": 20 - }, - "_LOCAL_ADDRESS_WIN10_UDP": { - "fields": { - "pData": { - "offset": 0, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - }, - "kind": "struct", - "size": 4 - }, - "_ADDRINFO": { - "fields": { - "Local": { - "offset": 0, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS" - } - } - }, - "Remote": { - "offset": 16, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - }, - "kind": "struct", - "size": 4 - }, - "_IN_ADDR": { - "fields": { - "addr4": { - "offset": 0, - "type": { - "count": 4, - "subtype": { - "kind": "base", - "name": "unsigned char" - }, - "kind": "array" - } - }, - "addr6": { - "offset": 0, - "type": { - "count": 16, - "subtype": { - "kind": "base", - "name": "unsigned char" - }, - "kind": "array" - } - } - }, - "kind": "struct", - "size": 6 - }, - "_INETAF": { - "fields": { - "AddressFamily": { - "offset": 24, - "type": { - "kind": "base", - "name": "unsigned short" - } - } - }, - "kind": "struct", - "size": 26 - }, - "_LARGE_INTEGER": { - "fields": { - "HighPart": { - "offset": 4, - "type": { - "kind": "base", - "name": "long" - } - }, - "LowPart": { - "offset": 0, - "type": { - "kind": "base", - "name": "unsigned long" - } - }, - "QuadPart": { - "offset": 0, - "type": { - "kind": "base", - "name": "long long" - } - }, - "u": { - "offset": 0, - "type": { - "kind": "struct", - "name": "__unnamed_2" - } - } - }, - "kind": "union", - "size": 8 - } - }, - "enums": { - "TCPStateEnum": { - "base": "long", - "constants": { - "CLOSED": 0, - "LISTENING": 1, - "SYN_SENT": 2, - "SYN_RCVD": 3, - "ESTABLISHED": 4, - "FIN_WAIT1": 5, - "FIN_WAIT2": 6, - "CLOSE_WAIT": 7, - "CLOSING": 8, - "LAST_ACK": 9, - "TIME_WAIT": 12, - "DELETE_TCB": 13 - }, - "size": 4 - } - }, - "metadata": { - "producer": { - "version": "0.0.1", - "name": "japhlange-by-hand", - "datetime": "2020-05-29T19:28:34" - }, - "format": "6.0.0" - } -} diff --git a/volatility3/framework/symbols/windows/netscan-win10-17763-x64.json b/volatility3/framework/symbols/windows/netscan-win10-17763-x64.json deleted file mode 100644 index 1eb3c754ce..0000000000 --- a/volatility3/framework/symbols/windows/netscan-win10-17763-x64.json +++ /dev/null @@ -1,388 +0,0 @@ -{ - "base_types": { - "unsigned long": { - "kind": "int", - "size": 4, - "signed": false, - "endian": "little" - }, - "unsigned char": { - "kind": "char", - "size": 1, - "signed": false, - "endian": "little" - }, - "pointer": { - "kind": "int", - "size": 8, - "signed": false, - "endian": "little" - }, - "unsigned int": { - "kind": "int", - "size": 4, - "signed": false, - "endian": "little" - }, - "unsigned short": { - "kind": "int", - "size": 2, - "signed": false, - "endian": "little" - }, - "unsigned be short": { - "kind": "int", - "size": 2, - "signed": false, - "endian": "big" - }, - "long long": { - "endian": "little", - "kind": "int", - "signed": true, - "size": 8 - }, - "long": { - "kind": "int", - "size": 4, - "signed": false, - "endian": "little" - } - }, - "symbols": {}, - "user_types": { - "_UDP_ENDPOINT": { - "fields": { - "Owner": { - "offset": 40, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - - } - }, - "CreateTime": { - "offset": 88, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "LocalAddr": { - "offset": 128, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS_WIN10_UDP" - } - } - }, - "InetAF": { - "offset": 32, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - - } - }, - "Port": { - "offset": 120, - "type": { - "kind": "base", - "name": "unsigned be short" - } - } - }, - "kind": "struct", - "size": 132 - }, - "_TCP_LISTENER": { - "fields": { - "Owner": { - "offset": 48, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - - } - }, - "CreateTime": { - "offset": 64, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "LocalAddr": { - "offset": 96, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS" - } - - } - }, - "InetAF": { - "offset": 40, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - - } - }, - "Port": { - "offset": 114, - "type": { - "kind": "base", - "name": "unsigned be short" - } - } - }, - "kind": "struct", - "size": 116 - }, - "_TCP_ENDPOINT": { - "fields": { - "Owner": { - "offset": 656, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - } - }, - "CreateTime": { - "offset": 672, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "AddrInfo": { - "offset": 24, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_ADDRINFO" - } - } - }, - "InetAF": { - "offset": 16, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - } - }, - "LocalPort": { - "offset": 112, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, - "RemotePort": { - "offset": 114, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, - "State": { - "offset": 108, - "type": { - "kind": "enum", - "name": "TCPStateEnum" - } - } - }, - "kind": "struct", - "size": 632 - }, - "_LOCAL_ADDRESS": { - "fields": { - "pData": { - "offset": 16, - "type": { - "kind": "pointer", - "subtype": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - } - }, - "kind": "struct", - "size": 20 - }, - "_LOCAL_ADDRESS_WIN10_UDP": { - "fields": { - "pData": { - "offset": 0, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - }, - "kind": "struct", - "size": 4 - }, - "_ADDRINFO": { - "fields": { - "Local": { - "offset": 0, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS" - } - } - }, - "Remote": { - "offset": 16, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - }, - "kind": "struct", - "size": 4 - }, - "_IN_ADDR": { - "fields": { - "addr4": { - "offset": 0, - "type": { - "count": 4, - "subtype": { - "kind": "base", - "name": "unsigned char" - }, - "kind": "array" - } - }, - "addr6": { - "offset": 0, - "type": { - "count": 16, - "subtype": { - "kind": "base", - "name": "unsigned char" - }, - "kind": "array" - } - } - }, - "kind": "struct", - "size": 6 - }, - "_INETAF": { - "fields": { - "AddressFamily": { - "offset": 24, - "type": { - "kind": "base", - "name": "unsigned short" - } - } - }, - "kind": "struct", - "size": 26 - }, - "_LARGE_INTEGER": { - "fields": { - "HighPart": { - "offset": 4, - "type": { - "kind": "base", - "name": "long" - } - }, - "LowPart": { - "offset": 0, - "type": { - "kind": "base", - "name": "unsigned long" - } - }, - "QuadPart": { - "offset": 0, - "type": { - "kind": "base", - "name": "long long" - } - }, - "u": { - "offset": 0, - "type": { - "kind": "struct", - "name": "__unnamed_2" - } - } - }, - "kind": "union", - "size": 8 - } - }, - "enums": { - "TCPStateEnum": { - "base": "long", - "constants": { - "CLOSED": 0, - "LISTENING": 1, - "SYN_SENT": 2, - "SYN_RCVD": 3, - "ESTABLISHED": 4, - "FIN_WAIT1": 5, - "FIN_WAIT2": 6, - "CLOSE_WAIT": 7, - "CLOSING": 8, - "LAST_ACK": 9, - "TIME_WAIT": 12, - "DELETE_TCB": 13 - }, - "size": 4 - } - }, - "metadata": { - "producer": { - "version": "0.0.1", - "name": "japhlange-by-hand", - "datetime": "2020-05-29T19:28:34" - }, - "format": "6.0.0" - } -} diff --git a/volatility3/framework/symbols/windows/netscan-win10-18363-x64.json b/volatility3/framework/symbols/windows/netscan-win10-18363-x64.json deleted file mode 100644 index d3537ef48c..0000000000 --- a/volatility3/framework/symbols/windows/netscan-win10-18363-x64.json +++ /dev/null @@ -1,388 +0,0 @@ -{ - "base_types": { - "unsigned long": { - "kind": "int", - "size": 4, - "signed": false, - "endian": "little" - }, - "unsigned char": { - "kind": "char", - "size": 1, - "signed": false, - "endian": "little" - }, - "pointer": { - "kind": "int", - "size": 8, - "signed": false, - "endian": "little" - }, - "unsigned int": { - "kind": "int", - "size": 4, - "signed": false, - "endian": "little" - }, - "unsigned short": { - "kind": "int", - "size": 2, - "signed": false, - "endian": "little" - }, - "unsigned be short": { - "kind": "int", - "size": 2, - "signed": false, - "endian": "big" - }, - "long long": { - "endian": "little", - "kind": "int", - "signed": true, - "size": 8 - }, - "long": { - "kind": "int", - "size": 4, - "signed": false, - "endian": "little" - } - }, - "symbols": {}, - "user_types": { - "_UDP_ENDPOINT": { - "fields": { - "Owner": { - "offset": 40, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - - } - }, - "CreateTime": { - "offset": 88, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "LocalAddr": { - "offset": 136, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS_WIN10_UDP" - } - } - }, - "InetAF": { - "offset": 32, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - - } - }, - "Port": { - "offset": 128, - "type": { - "kind": "base", - "name": "unsigned be short" - } - } - }, - "kind": "struct", - "size": 132 - }, - "_TCP_LISTENER": { - "fields": { - "Owner": { - "offset": 48, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - - } - }, - "CreateTime": { - "offset": 64, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "LocalAddr": { - "offset": 96, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS" - } - - } - }, - "InetAF": { - "offset": 40, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - - } - }, - "Port": { - "offset": 114, - "type": { - "kind": "base", - "name": "unsigned be short" - } - } - }, - "kind": "struct", - "size": 116 - }, - "_TCP_ENDPOINT": { - "fields": { - "Owner": { - "offset": 656, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - } - }, - "CreateTime": { - "offset": 672, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "AddrInfo": { - "offset": 24, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_ADDRINFO" - } - } - }, - "InetAF": { - "offset": 16, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - } - }, - "LocalPort": { - "offset": 112, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, - "RemotePort": { - "offset": 114, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, - "State": { - "offset": 108, - "type": { - "kind": "enum", - "name": "TCPStateEnum" - } - } - }, - "kind": "struct", - "size": 632 - }, - "_LOCAL_ADDRESS": { - "fields": { - "pData": { - "offset": 16, - "type": { - "kind": "pointer", - "subtype": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - } - }, - "kind": "struct", - "size": 20 - }, - "_LOCAL_ADDRESS_WIN10_UDP": { - "fields": { - "pData": { - "offset": 0, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - }, - "kind": "struct", - "size": 4 - }, - "_ADDRINFO": { - "fields": { - "Local": { - "offset": 0, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS" - } - } - }, - "Remote": { - "offset": 16, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - }, - "kind": "struct", - "size": 4 - }, - "_IN_ADDR": { - "fields": { - "addr4": { - "offset": 0, - "type": { - "count": 4, - "subtype": { - "kind": "base", - "name": "unsigned char" - }, - "kind": "array" - } - }, - "addr6": { - "offset": 0, - "type": { - "count": 16, - "subtype": { - "kind": "base", - "name": "unsigned char" - }, - "kind": "array" - } - } - }, - "kind": "struct", - "size": 6 - }, - "_INETAF": { - "fields": { - "AddressFamily": { - "offset": 24, - "type": { - "kind": "base", - "name": "unsigned short" - } - } - }, - "kind": "struct", - "size": 26 - }, - "_LARGE_INTEGER": { - "fields": { - "HighPart": { - "offset": 4, - "type": { - "kind": "base", - "name": "long" - } - }, - "LowPart": { - "offset": 0, - "type": { - "kind": "base", - "name": "unsigned long" - } - }, - "QuadPart": { - "offset": 0, - "type": { - "kind": "base", - "name": "long long" - } - }, - "u": { - "offset": 0, - "type": { - "kind": "struct", - "name": "__unnamed_2" - } - } - }, - "kind": "union", - "size": 8 - } - }, - "enums": { - "TCPStateEnum": { - "base": "long", - "constants": { - "CLOSED": 0, - "LISTENING": 1, - "SYN_SENT": 2, - "SYN_RCVD": 3, - "ESTABLISHED": 4, - "FIN_WAIT1": 5, - "FIN_WAIT2": 6, - "CLOSE_WAIT": 7, - "CLOSING": 8, - "LAST_ACK": 9, - "TIME_WAIT": 12, - "DELETE_TCB": 13 - }, - "size": 4 - } - }, - "metadata": { - "producer": { - "version": "0.0.1", - "name": "japhlange-by-hand", - "datetime": "2020-05-29T19:28:34" - }, - "format": "6.0.0" - } -} diff --git a/volatility3/framework/symbols/windows/netscan-vista-sp12-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-vista-sp12-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/netscan-vista-sp12-x64.json rename to volatility3/framework/symbols/windows/netscan/netscan-vista-sp12-x64.json diff --git a/volatility3/framework/symbols/windows/netscan-vista-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-vista-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/netscan-vista-x64.json rename to volatility3/framework/symbols/windows/netscan/netscan-vista-x64.json diff --git a/volatility3/framework/symbols/windows/netscan-vista-x86.json b/volatility3/framework/symbols/windows/netscan/netscan-vista-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/netscan-vista-x86.json rename to volatility3/framework/symbols/windows/netscan/netscan-vista-x86.json diff --git a/volatility3/framework/symbols/windows/netscan/netscan-win10-10240-x86.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-10240-x86.json new file mode 100644 index 0000000000..b27380f4c4 --- /dev/null +++ b/volatility3/framework/symbols/windows/netscan/netscan-win10-10240-x86.json @@ -0,0 +1,722 @@ +{ + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned be short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "big" + }, + "long long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 8 + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "symbols": {}, + "user_types": { + "_TCP_SYN_ENDPOINT": { + "fields": { + "Owner": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SYN_OWNER" + } + } + }, + "CreateTime": { + "offset": 0, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "ListEntry": { + "offset": 8, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "InetAF": { + "offset": 24, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "LocalPort": { + "offset": 60, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 62, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "LocalAddr": { + "offset": 28, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "RemoteAddress": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 64 + }, + "_TCP_TIMEWAIT_ENDPOINT": { + "fields": { + "CreateTime": { + "offset": 0, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "ListEntry": { + "offset": 20, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "InetAF": { + "offset": 12, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "LocalPort": { + "offset": 28, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 30, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "LocalAddr": { + "offset": 32, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "RemoteAddress": { + "offset": 36, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 40 + }, + "_UDP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 48, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "Next": { + "offset": 76, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_UDP_ENDPOINT" + } + } + }, + "LocalAddr": { + "offset": 56, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "InetAF": { + "offset": 20, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Port": { + "offset": 72, + "type": { + "kind": "base", + "name": "unsigned be short" + } + } + }, + "kind": "struct", + "size": 74 + }, + "_TCP_LISTENER": { + "fields": { + "Owner": { + "offset": 28, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 40, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 60, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + + } + }, + "InetAF": { + "offset": 64, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Port": { + "offset": 70, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "Next": { + "offset": 72, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_LISTENER" + } + } + } + }, + "kind": "struct", + "size": 78 + }, + "_TCP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 432, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + } + }, + "CreateTime": { + "offset": 440, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "ListEntry": { + "offset": 20, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "AddrInfo": { + "offset": 12, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ADDRINFO" + } + } + }, + "InetAF": { + "offset": 8, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + } + }, + "LocalPort": { + "offset": 60, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 62, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "State": { + "offset": 56, + "type": { + "kind": "enum", + "name": "TCPStateEnum" + } + } + }, + "kind": "struct", + "size": 448 + }, + "_LOCAL_ADDRESS": { + "fields": { + "pData": { + "offset": 12, + "type": { + "kind": "pointer", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_ADDRINFO": { + "fields": { + "Local": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "Remote": { + "offset": 12, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_IN_ADDR": { + "fields": { + "addr4": { + "offset": 0, + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + }, + "addr6": { + "offset": 0, + "type": { + "count": 16, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + } + }, + "kind": "struct", + "size": 6 + }, + "_INETAF": { + "fields": { + "AddressFamily": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 14 + }, + "_SYN_OWNER": { + "fields": { + "Process": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + } + } + }, + "kind": "struct", + "size": 14 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 4, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 8 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 4096 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 152, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 144, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 4, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 12, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 64 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 64 + } + }, + "enums": { + "TCPStateEnum": { + "base": "long", + "constants": { + "CLOSED": 0, + "LISTENING": 1, + "SYN_SENT": 2, + "SYN_RCVD": 3, + "ESTABLISHED": 4, + "FIN_WAIT1": 5, + "FIN_WAIT2": 6, + "CLOSE_WAIT": 7, + "CLOSING": 8, + "LAST_ACK": 9, + "TIME_WAIT": 12, + "DELETE_TCB": 13 + }, + "size": 4 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "japhlange-by-hand", + "datetime": "2020-05-29T19:28:34" + }, + "format": "6.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/netscan/netscan-win10-10586-x86.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-10586-x86.json new file mode 100644 index 0000000000..7a6b9827e6 --- /dev/null +++ b/volatility3/framework/symbols/windows/netscan/netscan-win10-10586-x86.json @@ -0,0 +1,722 @@ +{ + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned be short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "big" + }, + "long long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 8 + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "symbols": {}, + "user_types": { + "_TCP_SYN_ENDPOINT": { + "fields": { + "Owner": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SYN_OWNER" + } + } + }, + "CreateTime": { + "offset": 0, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "ListEntry": { + "offset": 8, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "InetAF": { + "offset": 24, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "LocalPort": { + "offset": 60, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 62, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "LocalAddr": { + "offset": 28, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "RemoteAddress": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 64 + }, + "_TCP_TIMEWAIT_ENDPOINT": { + "fields": { + "CreateTime": { + "offset": 0, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "ListEntry": { + "offset": 20, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "InetAF": { + "offset": 12, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "LocalPort": { + "offset": 28, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 30, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "LocalAddr": { + "offset": 32, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "RemoteAddress": { + "offset": 36, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 40 + }, + "_UDP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 40, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "Next": { + "offset": 64, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_UDP_ENDPOINT" + } + } + }, + "LocalAddr": { + "offset": 48, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "InetAF": { + "offset": 52, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Port": { + "offset": 60, + "type": { + "kind": "base", + "name": "unsigned be short" + } + } + }, + "kind": "struct", + "size": 74 + }, + "_TCP_LISTENER": { + "fields": { + "Owner": { + "offset": 20, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 32, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 52, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + + } + }, + "InetAF": { + "offset": 16, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Port": { + "offset": 62, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "Next": { + "offset": 72, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_LISTENER" + } + } + } + }, + "kind": "struct", + "size": 78 + }, + "_TCP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 424, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + } + }, + "CreateTime": { + "offset": 432, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "ListEntry": { + "offset": 32, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "AddrInfo": { + "offset": 4, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ADDRINFO" + } + } + }, + "InetAF": { + "offset": 0, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + } + }, + "LocalPort": { + "offset": 52, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 54, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "State": { + "offset": 48, + "type": { + "kind": "enum", + "name": "TCPStateEnum" + } + } + }, + "kind": "struct", + "size": 448 + }, + "_LOCAL_ADDRESS": { + "fields": { + "pData": { + "offset": 12, + "type": { + "kind": "pointer", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_ADDRINFO": { + "fields": { + "Local": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "Remote": { + "offset": 12, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_IN_ADDR": { + "fields": { + "addr4": { + "offset": 0, + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + }, + "addr6": { + "offset": 0, + "type": { + "count": 16, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + } + }, + "kind": "struct", + "size": 6 + }, + "_INETAF": { + "fields": { + "AddressFamily": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 16 + }, + "_SYN_OWNER": { + "fields": { + "Process": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + } + } + }, + "kind": "struct", + "size": 14 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 4, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 8 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 4096 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 152, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 144, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 12, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 72 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 72 + } + }, + "enums": { + "TCPStateEnum": { + "base": "long", + "constants": { + "CLOSED": 0, + "LISTENING": 1, + "SYN_SENT": 2, + "SYN_RCVD": 3, + "ESTABLISHED": 4, + "FIN_WAIT1": 5, + "FIN_WAIT2": 6, + "CLOSE_WAIT": 7, + "CLOSING": 8, + "LAST_ACK": 9, + "TIME_WAIT": 12, + "DELETE_TCB": 13 + }, + "size": 4 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "japhlange-by-hand", + "datetime": "2021-01-14T18:28:34" + }, + "format": "6.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/netscan-win10-14393-x86.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-14393-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/netscan-win10-14393-x86.json rename to volatility3/framework/symbols/windows/netscan/netscan-win10-14393-x86.json diff --git a/volatility3/framework/symbols/windows/netscan-win7-x86.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-15063-x64.json similarity index 74% rename from volatility3/framework/symbols/windows/netscan-win7-x86.json rename to volatility3/framework/symbols/windows/netscan/netscan-win10-15063-x64.json index 891e7072c7..e15f6eb509 100644 --- a/volatility3/framework/symbols/windows/netscan-win7-x86.json +++ b/volatility3/framework/symbols/windows/netscan/netscan-win10-15063-x64.json @@ -14,7 +14,7 @@ }, "pointer": { "kind": "int", - "size": 4, + "size": 8, "signed": false, "endian": "little" }, @@ -51,98 +51,38 @@ }, "symbols": {}, "user_types": { - "_TCP_SYN_ENDPOINT": { + "_UDP_ENDPOINT": { "fields": { "Owner": { - "offset": 44, + "offset": 40, "type": { "kind": "pointer", "subtype": { "kind": "struct", - "name": "_SYN_OWNER" + "name": "nt_symbols!_EPROCESS" } + } }, "CreateTime": { - "offset": 0, + "offset": 88, "type": { "kind": "union", "name": "_LARGE_INTEGER" } }, - "ListEntry": { - "offset": 8, - "type": { - "kind": "union", - "name": "nt_symbols!_LIST_ENTRY" - } - }, - "InetAF": { - "offset": 36, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - } - }, - "LocalPort": { - "offset": 72, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, - "RemotePort": { - "offset": 74, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, "LocalAddr": { - "offset": 40, + "offset": 128, "type":{ "kind": "pointer", "subtype": { "kind": "struct", - "name": "_LOCAL_ADDRESS" + "name": "_LOCAL_ADDRESS_WIN10_UDP" } } }, - "RemoteAddress": { - "offset": 52, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - }, - "kind": "struct", - "size": 76 - }, - "_TCP_TIMEWAIT_ENDPOINT": { - "fields": { - "CreateTime": { - "offset": 0, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "ListEntry": { - "offset": 0, - "type": { - "kind": "union", - "name": "nt_symbols!_LIST_ENTRY" - } - }, "InetAF": { - "offset": 24, + "offset": 32, "type":{ "kind": "pointer", "subtype": { @@ -152,48 +92,31 @@ } }, - "LocalPort": { - "offset": 40, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, - "RemotePort": { - "offset": 42, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, - "LocalAddr": { - "offset": 44, + "Next": { + "offset": 112, "type":{ "kind": "pointer", "subtype": { "kind": "struct", - "name": "_LOCAL_ADDRESS" + "name": "_UDP_ENDPOINT" } } }, - "RemoteAddress": { - "offset": 48, + "Port": { + "offset": 120, "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } + "kind": "base", + "name": "unsigned be short" } } }, "kind": "struct", - "size": 52 + "size": 132 }, - "_UDP_ENDPOINT": { + "_TCP_LISTENER": { "fields": { "Owner": { - "offset": 24, + "offset": 48, "type": { "kind": "pointer", "subtype": { @@ -203,67 +126,25 @@ } }, - "CreateTime": { - "offset": 48, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "LocalAddr": { - "offset": 56, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS" - } - } - }, - "InetAF": { - "offset": 20, + "Next": { + "offset": 120, "type":{ "kind": "pointer", "subtype": { "kind": "struct", - "name": "_INETAF" - } - - } - }, - "Port": { - "offset": 72, - "type": { - "kind": "base", - "name": "unsigned be short" - } - } - }, - "kind": "struct", - "size": 74 - }, - "_TCP_LISTENER": { - "fields": { - "Owner": { - "offset": 24, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" + "name": "_TCP_LISTENER" } - } }, "CreateTime": { - "offset": 32, + "offset": 64, "type": { "kind": "union", "name": "_LARGE_INTEGER" } }, "LocalAddr": { - "offset": 52, + "offset": 96, "type":{ "kind": "pointer", "subtype": { @@ -274,7 +155,7 @@ } }, "InetAF": { - "offset": 56, + "offset": 40, "type":{ "kind": "pointer", "subtype": { @@ -285,7 +166,7 @@ } }, "Port": { - "offset": 62, + "offset": 114, "type": { "kind": "base", "name": "unsigned be short" @@ -293,37 +174,29 @@ } }, "kind": "struct", - "size": 64 + "size": 116 }, "_TCP_ENDPOINT": { "fields": { "Owner": { - "offset": 372, + "offset": 624, "type": { "kind": "pointer", "subtype": { "kind": "struct", "name": "nt_symbols!_EPROCESS" } - } }, "CreateTime": { - "offset": 0, + "offset": 616, "type": { "kind": "union", "name": "_LARGE_INTEGER" } }, - "ListEntry": { - "offset": 20, - "type": { - "kind": "union", - "name": "nt_symbols!_LIST_ENTRY" - } - }, "AddrInfo": { - "offset": 16, + "offset": 24, "type":{ "kind": "pointer", "subtype": { @@ -333,32 +206,38 @@ } }, "InetAF": { - "offset": 12, + "offset": 16, "type":{ "kind": "pointer", "subtype": { "kind": "struct", "name": "_INETAF" } - + } + }, + "ListEntry": { + "offset": 40, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" } }, "LocalPort": { - "offset": 56, + "offset": 112, "type": { "kind": "base", "name": "unsigned be short" } }, "RemotePort": { - "offset": 58, + "offset": 114, "type": { "kind": "base", "name": "unsigned be short" } }, "State": { - "offset": 52, + "offset": 108, "type": { "kind": "enum", "name": "TCPStateEnum" @@ -366,12 +245,12 @@ } }, "kind": "struct", - "size": 376 + "size": 632 }, "_LOCAL_ADDRESS": { "fields": { "pData": { - "offset": 12, + "offset": 16, "type": { "kind": "pointer", "subtype": { @@ -385,7 +264,23 @@ } }, "kind": "struct", - "size": 16 + "size": 20 + }, + "_LOCAL_ADDRESS_WIN10_UDP": { + "fields": { + "pData": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 4 }, "_ADDRINFO": { "fields": { @@ -400,7 +295,7 @@ } }, "Remote": { - "offset": 8, + "offset": 16, "type": { "kind": "pointer", "subtype": { @@ -411,7 +306,7 @@ } }, "kind": "struct", - "size": 12 + "size": 4 }, "_IN_ADDR": { "fields": { @@ -444,7 +339,7 @@ "_INETAF": { "fields": { "AddressFamily": { - "offset": 12, + "offset": 24, "type": { "kind": "base", "name": "unsigned short" @@ -452,23 +347,7 @@ } }, "kind": "struct", - "size": 14 - }, - "_SYN_OWNER": { - "fields": { - "Process": { - "offset": 24, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - } - } - }, - "kind": "struct", - "size": 14 + "size": 26 }, "_LARGE_INTEGER": { "fields": { @@ -503,6 +382,173 @@ }, "kind": "union", "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 6144 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 32 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 232, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 216, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 128 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 128 } }, "enums": { diff --git a/volatility3/framework/symbols/windows/netscan-win10-15063-x86.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-15063-x86.json similarity index 73% rename from volatility3/framework/symbols/windows/netscan-win10-15063-x86.json rename to volatility3/framework/symbols/windows/netscan/netscan-win10-15063-x86.json index 08ba89dbb4..0f934b7d51 100644 --- a/volatility3/framework/symbols/windows/netscan-win10-15063-x86.json +++ b/volatility3/framework/symbols/windows/netscan/netscan-win10-15063-x86.json @@ -238,6 +238,16 @@ "kind": "base", "name": "unsigned be short" } + }, + "Next": { + "offset": 76, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_UDP_ENDPOINT" + } + } } }, "kind": "struct", @@ -291,6 +301,16 @@ "kind": "base", "name": "unsigned be short" } + }, + "Next": { + "offset": 72, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_LISTENER" + } + } } }, "kind": "struct", @@ -502,6 +522,173 @@ }, "kind": "union", "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 324, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 20, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 4, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 8 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 4096 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 152, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 144, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 4, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 12, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 64 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 64 } }, "enums": { diff --git a/volatility3/framework/symbols/windows/netscan-win10-16299-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-16299-x64.json similarity index 96% rename from volatility3/framework/symbols/windows/netscan-win10-16299-x64.json rename to volatility3/framework/symbols/windows/netscan/netscan-win10-16299-x64.json index 074b854c50..9991438f8f 100644 --- a/volatility3/framework/symbols/windows/netscan-win10-16299-x64.json +++ b/volatility3/framework/symbols/windows/netscan/netscan-win10-16299-x64.json @@ -105,7 +105,7 @@ } }, - "MaskedPrevObj": { + "Next": { "offset": 112, "type":{ "kind": "pointer", @@ -175,7 +175,7 @@ "name": "unsigned be short" } }, - "MaskedPrevObj": { + "Next": { "offset": 120, "type":{ "kind": "pointer", @@ -218,14 +218,11 @@ } } }, - "HashTableEntry": { + "ListEntry": { "offset": 40, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LIST_ENTRY" - } + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" } }, "InetAF": { @@ -259,7 +256,7 @@ "name": "TCPStateEnum" } }, - "MaskedPrevObj": { + "Next": { "offset": 112, "type":{ "kind": "pointer", @@ -459,7 +456,7 @@ }, "_PORT_ASSIGNMENT_ENTRY": { "fields": { - "MaskedObjectPtr": { + "Entry": { "offset": 8, "type": { "kind": "pointer", @@ -558,6 +555,23 @@ }, "kind": "struct", "size": 128 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 128 } }, "enums": { diff --git a/volatility3/framework/symbols/windows/netscan/netscan-win10-17134-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-17134-x64.json new file mode 100644 index 0000000000..a20dd3799a --- /dev/null +++ b/volatility3/framework/symbols/windows/netscan/netscan-win10-17134-x64.json @@ -0,0 +1,605 @@ +{ + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned be short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "big" + }, + "long long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 8 + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "symbols": { + "TcpCompartmentSet": { + "address": 2010312 + }, + "UdpCompartmentSet": { + "address": 2006416 + }, + "PartitionCount": { + "address": 2008196 + }, + "PartitionTable": { + "address": 2008200 + } + }, + "user_types": { + "_UDP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 88, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 128, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS_WIN10_UDP" + } + } + }, + "InetAF": { + "offset": 32, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_UDP_ENDPOINT" + } + } + }, + "Port": { + "offset": 120, + "type": { + "kind": "base", + "name": "unsigned be short" + } + } + }, + "kind": "struct", + "size": 132 + }, + "_TCP_LISTENER": { + "fields": { + "Owner": { + "offset": 48, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 64, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 96, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + + } + }, + "InetAF": { + "offset": 40, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Next": { + "offset": 120, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_LISTENER" + } + } + }, + "Port": { + "offset": 114, + "type": { + "kind": "base", + "name": "unsigned be short" + } + } + }, + "kind": "struct", + "size": 116 + }, + "_TCP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 632, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + } + }, + "CreateTime": { + "offset": 648, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "AddrInfo": { + "offset": 24, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ADDRINFO" + } + } + }, + "InetAF": { + "offset": 16, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + } + }, + "LocalPort": { + "offset": 112, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 114, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "ListEntry": { + "offset": 40, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "State": { + "offset": 108, + "type": { + "kind": "enum", + "name": "TCPStateEnum" + } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_ENDPOINT" + } + } + } + }, + "kind": "struct", + "size": 632 + }, + "_LOCAL_ADDRESS": { + "fields": { + "pData": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + } + }, + "kind": "struct", + "size": 20 + }, + "_LOCAL_ADDRESS_WIN10_UDP": { + "fields": { + "pData": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 4 + }, + "_ADDRINFO": { + "fields": { + "Local": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "Remote": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 4 + }, + "_IN_ADDR": { + "fields": { + "addr4": { + "offset": 0, + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + }, + "addr6": { + "offset": 0, + "type": { + "count": 16, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + } + }, + "kind": "struct", + "size": 6 + }, + "_INETAF": { + "fields": { + "AddressFamily": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 26 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 6144 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 32 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 232, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 216, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 128 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 128 + } + }, + "enums": { + "TCPStateEnum": { + "base": "long", + "constants": { + "CLOSED": 0, + "LISTENING": 1, + "SYN_SENT": 2, + "SYN_RCVD": 3, + "ESTABLISHED": 4, + "FIN_WAIT1": 5, + "FIN_WAIT2": 6, + "CLOSE_WAIT": 7, + "CLOSING": 8, + "LAST_ACK": 9, + "TIME_WAIT": 12, + "DELETE_TCB": 13 + }, + "size": 4 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "japhlange-by-hand", + "datetime": "2020-05-29T19:28:34" + }, + "format": "6.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/netscan-win10-17134-x86.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-17134-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/netscan-win10-17134-x86.json rename to volatility3/framework/symbols/windows/netscan/netscan-win10-17134-x86.json diff --git a/volatility3/framework/symbols/windows/netscan/netscan-win10-17763-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-17763-x64.json new file mode 100644 index 0000000000..7f89607e5f --- /dev/null +++ b/volatility3/framework/symbols/windows/netscan/netscan-win10-17763-x64.json @@ -0,0 +1,605 @@ +{ + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned be short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "big" + }, + "long long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 8 + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "symbols": { + "TcpCompartmentSet": { + "address": 2010312 + }, + "UdpCompartmentSet": { + "address": 2006416 + }, + "PartitionCount": { + "address": 2008196 + }, + "PartitionTable": { + "address": 2008200 + } + }, + "user_types": { + "_UDP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 88, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 128, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS_WIN10_UDP" + } + } + }, + "InetAF": { + "offset": 32, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_UDP_ENDPOINT" + } + } + }, + "Port": { + "offset": 120, + "type": { + "kind": "base", + "name": "unsigned be short" + } + } + }, + "kind": "struct", + "size": 132 + }, + "_TCP_LISTENER": { + "fields": { + "Owner": { + "offset": 48, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 64, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 96, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + + } + }, + "InetAF": { + "offset": 40, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Next": { + "offset": 120, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_LISTENER" + } + } + }, + "Port": { + "offset": 114, + "type": { + "kind": "base", + "name": "unsigned be short" + } + } + }, + "kind": "struct", + "size": 116 + }, + "_TCP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 712, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + } + }, + "CreateTime": { + "offset": 728, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "AddrInfo": { + "offset": 24, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ADDRINFO" + } + } + }, + "InetAF": { + "offset": 16, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + } + }, + "LocalPort": { + "offset": 112, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 114, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "ListEntry": { + "offset": 40, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_ENDPOINT" + } + } + }, + "State": { + "offset": 108, + "type": { + "kind": "enum", + "name": "TCPStateEnum" + } + } + }, + "kind": "struct", + "size": 632 + }, + "_LOCAL_ADDRESS": { + "fields": { + "pData": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + } + }, + "kind": "struct", + "size": 20 + }, + "_LOCAL_ADDRESS_WIN10_UDP": { + "fields": { + "pData": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 4 + }, + "_ADDRINFO": { + "fields": { + "Local": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "Remote": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 4 + }, + "_IN_ADDR": { + "fields": { + "addr4": { + "offset": 0, + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + }, + "addr6": { + "offset": 0, + "type": { + "count": 16, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + } + }, + "kind": "struct", + "size": 6 + }, + "_INETAF": { + "fields": { + "AddressFamily": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 26 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 6144 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 32 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 232, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 216, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 128 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 128 + } + }, + "enums": { + "TCPStateEnum": { + "base": "long", + "constants": { + "CLOSED": 0, + "LISTENING": 1, + "SYN_SENT": 2, + "SYN_RCVD": 3, + "ESTABLISHED": 4, + "FIN_WAIT1": 5, + "FIN_WAIT2": 6, + "CLOSE_WAIT": 7, + "CLOSING": 8, + "LAST_ACK": 9, + "TIME_WAIT": 12, + "DELETE_TCB": 13 + }, + "size": 4 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "japhlange-by-hand", + "datetime": "2020-05-29T19:28:34" + }, + "format": "6.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/netscan/netscan-win10-18362-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-18362-x64.json new file mode 100644 index 0000000000..938b3fa841 --- /dev/null +++ b/volatility3/framework/symbols/windows/netscan/netscan-win10-18362-x64.json @@ -0,0 +1,605 @@ +{ + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 8, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned be short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "big" + }, + "long long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 8 + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "symbols": { + "TcpCompartmentSet": { + "address": 2010312 + }, + "UdpCompartmentSet": { + "address": 2006416 + }, + "PartitionCount": { + "address": 2008196 + }, + "PartitionTable": { + "address": 2008200 + } + }, + "user_types": { + "_UDP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 40, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 88, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 128, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS_WIN10_UDP" + } + } + }, + "InetAF": { + "offset": 32, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_UDP_ENDPOINT" + } + } + }, + "Port": { + "offset": 120, + "type": { + "kind": "base", + "name": "unsigned be short" + } + } + }, + "kind": "struct", + "size": 132 + }, + "_TCP_LISTENER": { + "fields": { + "Owner": { + "offset": 48, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 64, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 96, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + + } + }, + "InetAF": { + "offset": 40, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Next": { + "offset": 120, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_LISTENER" + } + } + }, + "Port": { + "offset": 114, + "type": { + "kind": "base", + "name": "unsigned be short" + } + } + }, + "kind": "struct", + "size": 116 + }, + "_TCP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 656, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + } + }, + "CreateTime": { + "offset": 672, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "AddrInfo": { + "offset": 24, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ADDRINFO" + } + } + }, + "InetAF": { + "offset": 16, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + } + }, + "LocalPort": { + "offset": 112, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 114, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "ListEntry": { + "offset": 40, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_ENDPOINT" + } + } + }, + "State": { + "offset": 108, + "type": { + "kind": "enum", + "name": "TCPStateEnum" + } + } + }, + "kind": "struct", + "size": 632 + }, + "_LOCAL_ADDRESS": { + "fields": { + "pData": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + } + }, + "kind": "struct", + "size": 20 + }, + "_LOCAL_ADDRESS_WIN10_UDP": { + "fields": { + "pData": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 4 + }, + "_ADDRINFO": { + "fields": { + "Local": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "Remote": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 4 + }, + "_IN_ADDR": { + "fields": { + "addr4": { + "offset": 0, + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + }, + "addr6": { + "offset": 0, + "type": { + "count": 16, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + } + }, + "kind": "struct", + "size": 6 + }, + "_INETAF": { + "fields": { + "AddressFamily": { + "offset": 24, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 26 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 6144 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 32 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 224, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 208, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 128 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 128 + } + }, + "enums": { + "TCPStateEnum": { + "base": "long", + "constants": { + "CLOSED": 0, + "LISTENING": 1, + "SYN_SENT": 2, + "SYN_RCVD": 3, + "ESTABLISHED": 4, + "FIN_WAIT1": 5, + "FIN_WAIT2": 6, + "CLOSE_WAIT": 7, + "CLOSING": 8, + "LAST_ACK": 9, + "TIME_WAIT": 12, + "DELETE_TCB": 13 + }, + "size": 4 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "japhlange-by-hand", + "datetime": "2020-05-29T19:28:34" + }, + "format": "6.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/netscan-win10-19041-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-18363-x64.json similarity index 66% rename from volatility3/framework/symbols/windows/netscan-win10-19041-x64.json rename to volatility3/framework/symbols/windows/netscan/netscan-win10-18363-x64.json index ec8e7d8704..9ecfdc6420 100644 --- a/volatility3/framework/symbols/windows/netscan-win10-19041-x64.json +++ b/volatility3/framework/symbols/windows/netscan/netscan-win10-18363-x64.json @@ -72,7 +72,7 @@ } }, "LocalAddr": { - "offset": 168, + "offset": 136, "type":{ "kind": "pointer", "subtype": { @@ -92,7 +92,7 @@ } }, - "MaskedPrevObj": { + "Next": { "offset": 112, "type":{ "kind": "pointer", @@ -103,7 +103,7 @@ } }, "Port": { - "offset": 160, + "offset": 128, "type": { "kind": "base", "name": "unsigned be short" @@ -155,7 +155,7 @@ } }, - "MaskedPrevObj": { + "Next": { "offset": 120, "type":{ "kind": "pointer", @@ -174,12 +174,12 @@ } }, "kind": "struct", - "size": 128 + "size": 116 }, "_TCP_ENDPOINT": { "fields": { "Owner": { - "offset": 728, + "offset": 656, "type": { "kind": "pointer", "subtype": { @@ -189,7 +189,7 @@ } }, "CreateTime": { - "offset": 744, + "offset": 672, "type": { "kind": "union", "name": "_LARGE_INTEGER" @@ -229,6 +229,23 @@ "name": "unsigned be short" } }, + "ListEntry": { + "offset": 40, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_ENDPOINT" + } + } + }, "State": { "offset": 108, "type": { @@ -375,6 +392,173 @@ }, "kind": "union", "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 6144 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 32 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 224, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 208, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 128 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 128 } }, "enums": { diff --git a/volatility3/framework/symbols/windows/netscan-win10-x86.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-19041-x64.json similarity index 73% rename from volatility3/framework/symbols/windows/netscan-win10-x86.json rename to volatility3/framework/symbols/windows/netscan/netscan-win10-19041-x64.json index bbd77d4aa1..0ff62ee999 100644 --- a/volatility3/framework/symbols/windows/netscan-win10-x86.json +++ b/volatility3/framework/symbols/windows/netscan/netscan-win10-19041-x64.json @@ -14,7 +14,7 @@ }, "pointer": { "kind": "int", - "size": 4, + "size": 8, "signed": false, "endian": "little" }, @@ -51,99 +51,48 @@ }, "symbols": {}, "user_types": { - "_TCP_SYN_ENDPOINT": { + "_UDP_ENDPOINT": { "fields": { "Owner": { - "offset": 32, + "offset": 40, "type": { "kind": "pointer", "subtype": { "kind": "struct", - "name": "_SYN_OWNER" + "name": "nt_symbols!_EPROCESS" } + } }, "CreateTime": { - "offset": 0, + "offset": 88, "type": { "kind": "union", "name": "_LARGE_INTEGER" } }, - "ListEntry": { - "offset": 8, - "type": { - "kind": "union", - "name": "nt_symbols!_LIST_ENTRY" - } - }, - "InetAF": { - "offset": 24, + "Next": { + "offset": 112, "type":{ "kind": "pointer", "subtype": { "kind": "struct", - "name": "_INETAF" + "name": "_UDP_ENDPOINT" } - - } - }, - "LocalPort": { - "offset": 60, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, - "RemotePort": { - "offset": 62, - "type": { - "kind": "base", - "name": "unsigned be short" } }, "LocalAddr": { - "offset": 28, + "offset": 168, "type":{ "kind": "pointer", "subtype": { "kind": "struct", - "name": "_LOCAL_ADDRESS" + "name": "_LOCAL_ADDRESS_WIN10_UDP" } } }, - "RemoteAddress": { - "offset": 40, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } - } - }, - "kind": "struct", - "size": 64 - }, - "_TCP_TIMEWAIT_ENDPOINT": { - "fields": { - "CreateTime": { - "offset": 0, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "ListEntry": { - "offset": 20, - "type": { - "kind": "union", - "name": "nt_symbols!_LIST_ENTRY" - } - }, "InetAF": { - "offset": 12, + "offset": 32, "type":{ "kind": "pointer", "subtype": { @@ -153,48 +102,21 @@ } }, - "LocalPort": { - "offset": 28, - "type": { - "kind": "base", - "name": "unsigned be short" - } - }, - "RemotePort": { - "offset": 30, + "Port": { + "offset": 160, "type": { "kind": "base", "name": "unsigned be short" } - }, - "LocalAddr": { - "offset": 32, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_LOCAL_ADDRESS" - } - } - }, - "RemoteAddress": { - "offset": 36, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_IN_ADDR" - } - } } }, "kind": "struct", - "size": 40 + "size": 168 }, - "_UDP_ENDPOINT": { + "_TCP_LISTENER": { "fields": { "Owner": { - "offset": 24, + "offset": 48, "type": { "kind": "pointer", "subtype": { @@ -205,88 +127,46 @@ } }, "CreateTime": { - "offset": 48, + "offset": 64, "type": { "kind": "union", "name": "_LARGE_INTEGER" } }, "LocalAddr": { - "offset": 56, + "offset": 96, "type":{ "kind": "pointer", "subtype": { "kind": "struct", "name": "_LOCAL_ADDRESS" } - } - }, - "InetAF": { - "offset": 20, - "type":{ - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "_INETAF" - } - - } - }, - "Port": { - "offset": 72, - "type": { - "kind": "base", - "name": "unsigned be short" - } - } - }, - "kind": "struct", - "size": 74 - }, - "_TCP_LISTENER": { - "fields": { - "Owner": { - "offset": 28, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } } }, - "CreateTime": { + "InetAF": { "offset": 40, - "type": { - "kind": "union", - "name": "_LARGE_INTEGER" - } - }, - "LocalAddr": { - "offset": 60, "type":{ "kind": "pointer", "subtype": { "kind": "struct", - "name": "_LOCAL_ADDRESS" + "name": "_INETAF" } } }, - "InetAF": { - "offset": 64, + "Next": { + "offset": 120, "type":{ "kind": "pointer", "subtype": { "kind": "struct", - "name": "_INETAF" + "name": "_TCP_LISTENER" } - } }, "Port": { - "offset": 70, + "offset": 114, "type": { "kind": "base", "name": "unsigned be short" @@ -294,12 +174,12 @@ } }, "kind": "struct", - "size": 72 + "size": 128 }, "_TCP_ENDPOINT": { "fields": { "Owner": { - "offset": 432, + "offset": 728, "type": { "kind": "pointer", "subtype": { @@ -309,21 +189,14 @@ } }, "CreateTime": { - "offset": 440, + "offset": 744, "type": { "kind": "union", "name": "_LARGE_INTEGER" } }, - "ListEntry": { - "offset": 20, - "type": { - "kind": "union", - "name": "nt_symbols!_LIST_ENTRY" - } - }, "AddrInfo": { - "offset": 12, + "offset": 24, "type":{ "kind": "pointer", "subtype": { @@ -332,8 +205,15 @@ } } }, + "ListEntry": { + "offset": 40, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, "InetAF": { - "offset": 8, + "offset": 16, "type":{ "kind": "pointer", "subtype": { @@ -343,21 +223,21 @@ } }, "LocalPort": { - "offset": 60, + "offset": 112, "type": { "kind": "base", "name": "unsigned be short" } }, "RemotePort": { - "offset": 62, + "offset": 114, "type": { "kind": "base", "name": "unsigned be short" } }, "State": { - "offset": 56, + "offset": 108, "type": { "kind": "enum", "name": "TCPStateEnum" @@ -365,12 +245,12 @@ } }, "kind": "struct", - "size": 448 + "size": 632 }, "_LOCAL_ADDRESS": { "fields": { "pData": { - "offset": 12, + "offset": 16, "type": { "kind": "pointer", "subtype": { @@ -384,7 +264,23 @@ } }, "kind": "struct", - "size": 16 + "size": 20 + }, + "_LOCAL_ADDRESS_WIN10_UDP": { + "fields": { + "pData": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 4 }, "_ADDRINFO": { "fields": { @@ -399,7 +295,7 @@ } }, "Remote": { - "offset": 12, + "offset": 16, "type": { "kind": "pointer", "subtype": { @@ -410,7 +306,7 @@ } }, "kind": "struct", - "size": 16 + "size": 4 }, "_IN_ADDR": { "fields": { @@ -443,7 +339,7 @@ "_INETAF": { "fields": { "AddressFamily": { - "offset": 12, + "offset": 24, "type": { "kind": "base", "name": "unsigned short" @@ -451,23 +347,7 @@ } }, "kind": "struct", - "size": 14 - }, - "_SYN_OWNER": { - "fields": { - "Process": { - "offset": 24, - "type": { - "kind": "pointer", - "subtype": { - "kind": "struct", - "name": "nt_symbols!_EPROCESS" - } - } - } - }, - "kind": "struct", - "size": 14 + "size": 26 }, "_LARGE_INTEGER": { "fields": { @@ -502,6 +382,173 @@ }, "kind": "union", "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 32 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 6144 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 32 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 224, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 208, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 16, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 192 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 128 } }, "enums": { diff --git a/volatility3/framework/symbols/windows/netscan-win10-19041-x86.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-19041-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/netscan-win10-19041-x86.json rename to volatility3/framework/symbols/windows/netscan/netscan-win10-19041-x86.json diff --git a/volatility3/framework/symbols/windows/netscan-win10-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win10-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/netscan-win10-x64.json rename to volatility3/framework/symbols/windows/netscan/netscan-win10-x64.json diff --git a/volatility3/framework/symbols/windows/netscan-win7-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win7-x64.json similarity index 79% rename from volatility3/framework/symbols/windows/netscan-win7-x64.json rename to volatility3/framework/symbols/windows/netscan/netscan-win7-x64.json index 88aae05356..12dc8df7b3 100644 --- a/volatility3/framework/symbols/windows/netscan-win7-x64.json +++ b/volatility3/framework/symbols/windows/netscan/netscan-win7-x64.json @@ -230,6 +230,16 @@ } }, + "Next": { + "offset": 136, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_UDP_ENDPOINT" + } + } + }, "Port": { "offset": 128, "type": { @@ -239,7 +249,7 @@ } }, "kind": "struct", - "size": 130 + "size": 138 }, "_TCP_LISTENER": { "fields": { @@ -289,6 +299,16 @@ "kind": "base", "name": "unsigned be short" } + }, + "Next": { + "offset": 112, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_LISTENER" + } + } } }, "kind": "struct", @@ -469,65 +489,183 @@ "kind": "struct", "size": 48 }, - "_PARTITION_TABLE": { + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { "fields": { - "HashTable": { + "PortPool": { "offset": 0, "type": { "kind": "pointer", "subtype": { - "kind": "base", - "name": "void" + "kind": "struct", + "name": "_INET_PORT_POOL" } } - }, - "Unknown2": { + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { "offset": 8, "type": { "kind": "pointer", "subtype": { - "kind": "base", - "name": "void" + "kind": "base", + "name": "void" } } - }, - "Unknown3": { - "offset": 16, + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, "type": { - "kind": "pointer", + "count": 256, + "kind": "array", "subtype": { - "kind": "base", - "name": "void" + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" } } - }, - "Unknown4": { - "offset": 24, + } + }, + "kind": "struct", + "size": 4096 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 32, "type": { "kind": "pointer", "subtype": { - "kind": "base", - "name": "void" + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 40 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 160, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } } } }, - "Unknown5": { - "offset": 32, + "PortBitMap": { + "offset": 144, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 8, "type": { "kind": "pointer", "subtype": { - "kind": "base", - "name": "void" + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" } } }, - "Unknown6": { - "offset": 40, + "UnknownHashTable" : { + "offset": 16, "type": { "kind": "pointer", "subtype": { - "kind": "base", - "name": "void" + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" } } } @@ -535,39 +673,22 @@ "kind": "struct", "size": 128 }, - "_LARGE_INTEGER": { - "fields": { - "HighPart": { - "offset": 4, - "type": { - "kind": "base", - "name": "long" - } - }, - "LowPart": { - "offset": 0, - "type": { - "kind": "base", - "name": "unsigned long" - } - }, - "QuadPart": { - "offset": 0, - "type": { - "kind": "base", - "name": "long long" - } + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } }, - "u": { - "offset": 0, - "type": { - "kind": "struct", - "name": "__unnamed_2" - } - } - }, - "kind": "union", - "size": 8 + "kind": "struct", + "size": 128 } }, "enums": { diff --git a/volatility3/framework/symbols/windows/netscan/netscan-win7-x86.json b/volatility3/framework/symbols/windows/netscan/netscan-win7-x86.json new file mode 100644 index 0000000000..03cbe7bf28 --- /dev/null +++ b/volatility3/framework/symbols/windows/netscan/netscan-win7-x86.json @@ -0,0 +1,723 @@ +{ + "base_types": { + "unsigned long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned char": { + "kind": "char", + "size": 1, + "signed": false, + "endian": "little" + }, + "pointer": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned int": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + }, + "unsigned short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "little" + }, + "unsigned be short": { + "kind": "int", + "size": 2, + "signed": false, + "endian": "big" + }, + "long long": { + "endian": "little", + "kind": "int", + "signed": true, + "size": 8 + }, + "long": { + "kind": "int", + "size": 4, + "signed": false, + "endian": "little" + } + }, + "symbols": {}, + "user_types": { + "_TCP_SYN_ENDPOINT": { + "fields": { + "Owner": { + "offset": 44, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_SYN_OWNER" + } + } + }, + "CreateTime": { + "offset": 0, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "ListEntry": { + "offset": 8, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "InetAF": { + "offset": 36, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + } + }, + "LocalPort": { + "offset": 72, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 74, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "LocalAddr": { + "offset": 40, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "RemoteAddress": { + "offset": 52, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 76 + }, + "_TCP_TIMEWAIT_ENDPOINT": { + "fields": { + "CreateTime": { + "offset": 0, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "ListEntry": { + "offset": 0, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "InetAF": { + "offset": 24, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "LocalPort": { + "offset": 40, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 42, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "LocalAddr": { + "offset": 44, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "RemoteAddress": { + "offset": 48, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 52 + }, + "_UDP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 48, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 56, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "InetAF": { + "offset": 20, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Next": { + "offset": 76, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_UDP_ENDPOINT" + } + } + }, + "Port": { + "offset": 72, + "type": { + "kind": "base", + "name": "unsigned be short" + } + } + }, + "kind": "struct", + "size": 74 + }, + "_TCP_LISTENER": { + "fields": { + "Owner": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 32, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "LocalAddr": { + "offset": 52, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + + } + }, + "InetAF": { + "offset": 56, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "Port": { + "offset": 62, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "Next": { + "offset": 64, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_TCP_LISTENER" + } + } + } + }, + "kind": "struct", + "size": 72 + }, + "_TCP_ENDPOINT": { + "fields": { + "Owner": { + "offset": 372, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + + } + }, + "CreateTime": { + "offset": 0, + "type": { + "kind": "union", + "name": "_LARGE_INTEGER" + } + }, + "ListEntry": { + "offset": 20, + "type": { + "kind": "union", + "name": "nt_symbols!_LIST_ENTRY" + } + }, + "AddrInfo": { + "offset": 16, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_ADDRINFO" + } + } + }, + "InetAF": { + "offset": 12, + "type":{ + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INETAF" + } + + } + }, + "LocalPort": { + "offset": 56, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "RemotePort": { + "offset": 58, + "type": { + "kind": "base", + "name": "unsigned be short" + } + }, + "State": { + "offset": 52, + "type": { + "kind": "enum", + "name": "TCPStateEnum" + } + } + }, + "kind": "struct", + "size": 376 + }, + "_LOCAL_ADDRESS": { + "fields": { + "pData": { + "offset": 12, + "type": { + "kind": "pointer", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_ADDRINFO": { + "fields": { + "Local": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_LOCAL_ADDRESS" + } + } + }, + "Remote": { + "offset": 8, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_IN_ADDR" + } + } + } + }, + "kind": "struct", + "size": 12 + }, + "_IN_ADDR": { + "fields": { + "addr4": { + "offset": 0, + "type": { + "count": 4, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + }, + "addr6": { + "offset": 0, + "type": { + "count": 16, + "subtype": { + "kind": "base", + "name": "unsigned char" + }, + "kind": "array" + } + } + }, + "kind": "struct", + "size": 6 + }, + "_INETAF": { + "fields": { + "AddressFamily": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned short" + } + } + }, + "kind": "struct", + "size": 14 + }, + "_SYN_OWNER": { + "fields": { + "Process": { + "offset": 24, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_EPROCESS" + } + } + } + }, + "kind": "struct", + "size": 14 + }, + "_LARGE_INTEGER": { + "fields": { + "HighPart": { + "offset": 4, + "type": { + "kind": "base", + "name": "long" + } + }, + "LowPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "QuadPart": { + "offset": 0, + "type": { + "kind": "base", + "name": "long long" + } + }, + "u": { + "offset": 0, + "type": { + "kind": "struct", + "name": "__unnamed_2" + } + } + }, + "kind": "union", + "size": 8 + }, + "_INET_COMPARTMENT_SET": { + "fields": { + "InetCompartment": { + "offset": 328, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 384 + }, + "_INET_COMPARTMENT": { + "fields": { + "ProtocolCompartment": { + "offset": 32, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PROTOCOL_COMPARTMENT" + } + } + } + }, + "kind": "struct", + "size": 48 + }, + "_PROTOCOL_COMPARTMENT": { + "fields": { + "PortPool": { + "offset": 0, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_INET_PORT_POOL" + } + } + } + }, + "kind": "struct", + "size": 16 + }, + "_PORT_ASSIGNMENT_ENTRY": { + "fields": { + "Entry": { + "offset": 4, + "type": { + "kind": "pointer", + "subtype": { + "kind": "base", + "name": "void" + } + } + } + }, + "kind": "struct", + "size": 8 + }, + "_PORT_ASSIGNMENT_LIST": { + "fields": { + "Assignments": { + "offset": 0, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_ENTRY" + } + } + } + }, + "kind": "struct", + "size": 4096 + }, + "_PORT_ASSIGNMENT": { + "fields": { + "InPaBigPoolBase": { + "offset": 20, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT_LIST" + } + } + } + }, + "kind": "struct", + "size": 24 + }, + "_INET_PORT_POOL": { + "fields": { + "PortAssignments": { + "offset": 88, + "type": { + "count": 256, + "kind": "array", + "subtype": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "_PORT_ASSIGNMENT" + } + } + } + }, + "PortBitMap": { + "offset": 80, + "type": { + "kind": "struct", + "name": "nt_symbols!_RTL_BITMAP" + } + } + }, + "kind": "struct", + "size": 11200 + }, + "_PARTITION": { + "fields": { + "Endpoints" : { + "offset": 4, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + }, + "UnknownHashTable" : { + "offset": 12, + "type": { + "kind": "pointer", + "subtype": { + "kind": "struct", + "name": "nt_symbols!_RTL_DYNAMIC_HASH_TABLE" + } + } + } + }, + "kind": "struct", + "size": 64 + }, + "_PARTITION_TABLE": { + "fields": { + "Partitions": { + "offset": 0, + "type": { + "count": 1, + "kind": "array", + "subtype": { + "kind": "struct", + "name": "_PARTITION" + } + } + } + }, + "kind": "struct", + "size": 64 + } + }, + "enums": { + "TCPStateEnum": { + "base": "long", + "constants": { + "CLOSED": 0, + "LISTENING": 1, + "SYN_SENT": 2, + "SYN_RCVD": 3, + "ESTABLISHED": 4, + "FIN_WAIT1": 5, + "FIN_WAIT2": 6, + "CLOSE_WAIT": 7, + "CLOSING": 8, + "LAST_ACK": 9, + "TIME_WAIT": 12, + "DELETE_TCB": 13 + }, + "size": 4 + } + }, + "metadata": { + "producer": { + "version": "0.0.1", + "name": "japhlange-by-hand", + "datetime": "2020-05-29T19:28:34" + }, + "format": "6.0.0" + } +} diff --git a/volatility3/framework/symbols/windows/netscan-win8-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win8-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/netscan-win8-x64.json rename to volatility3/framework/symbols/windows/netscan/netscan-win8-x64.json diff --git a/volatility3/framework/symbols/windows/netscan-win8-x86.json b/volatility3/framework/symbols/windows/netscan/netscan-win8-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/netscan-win8-x86.json rename to volatility3/framework/symbols/windows/netscan/netscan-win8-x86.json diff --git a/volatility3/framework/symbols/windows/netscan-win81-x64.json b/volatility3/framework/symbols/windows/netscan/netscan-win81-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/netscan-win81-x64.json rename to volatility3/framework/symbols/windows/netscan/netscan-win81-x64.json diff --git a/volatility3/framework/symbols/windows/netscan-win81-x86.json b/volatility3/framework/symbols/windows/netscan/netscan-win81-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/netscan-win81-x86.json rename to volatility3/framework/symbols/windows/netscan/netscan-win81-x86.json diff --git a/volatility3/framework/symbols/windows/pdb.json b/volatility3/framework/symbols/windows/pdb.json index 42c11ec26a..f87b8db9fc 100644 --- a/volatility3/framework/symbols/windows/pdb.json +++ b/volatility3/framework/symbols/windows/pdb.json @@ -1295,6 +1295,54 @@ "kind": "struct", "size": 18 }, + "LF_STRUCTURE_VS19": { + "fields": { + "properties": { + "offset": 0, + "type": { + "kind": "struct", + "name": "Type_Properties" + } + }, + "fields": { + "offset": 4, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "derived_from": { + "offset": 8, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "vtable_shape": { + "offset": 12, + "type": { + "kind": "base", + "name": "unsigned long" + } + }, + "size": { + "offset": 18, + "type": { + "kind": "base", + "name": "unsigned short" + } + }, + "name": { + "offset": 20, + "type": { + "kind": "base", + "name": "string" + } + } + }, + "kind": "struct", + "size": 20 + }, "LF_UDT_SRC_LINE": { "fields": { "udt": { @@ -1305,14 +1353,14 @@ } }, "source_file": { - "offset": 0, + "offset": 4, "type": { "kind": "base", "name": "unsigned long" } }, "line": { - "offset": 0, + "offset": 8, "type": { "kind": "base", "name": "unsigned long" @@ -1349,12 +1397,12 @@ "offset": 12, "type": { "kind": "base", - "name": "string" + "name": "unsigned short" } } }, "kind": "struct", - "size": 12 + "size": 14 }, "LF_UNION": { "fields": { @@ -1626,6 +1674,8 @@ "LF_STRING_ID": 5637, "LF_UDT_SRC_LINE": 5638, "LF_UDT_MOD_SRC_LINE": 5639, + "LF_CLASS_VS19": 5640, + "LF_STRUCTURE_VS19": 5641, "LF_CHAR": 32768, "LF_SHORT": 32769, "LF_USHORT": 32770, diff --git a/volatility3/framework/symbols/windows/pdbconv.py b/volatility3/framework/symbols/windows/pdbconv.py index 5116b8f78b..da8254ffd9 100644 --- a/volatility3/framework/symbols/windows/pdbconv.py +++ b/volatility3/framework/symbols/windows/pdbconv.py @@ -2,20 +2,23 @@ # which is available at https://www.volatilityfoundation.org/license/vsl-v1.0 # import binascii +import bz2 import datetime +import gzip import json import logging +import lzma import os from bisect import bisect from typing import Tuple, Dict, Any, Optional, Union, List -from urllib import request, error +from urllib import request, error, parse from volatility3.framework import contexts, interfaces, constants from volatility3.framework.layers import physical, msf, resources vollog = logging.getLogger(__name__) -primatives = { +primitives = { 0x03: ("void", { "endian": "little", "kind": "void", @@ -262,18 +265,19 @@ def __init__(self, database_name: Optional[str] = None, progress_callback: constants.ProgressCallback = None) -> None: self._layer_name, self._context = self.load_pdb_layer(context, location) - self._dbiheader = None # type: Optional[interfaces.objects.ObjectInterface] + self._dbiheader: Optional[interfaces.objects.ObjectInterface] = None if not progress_callback: progress_callback = lambda x, y: None self._progress_callback = progress_callback - self.types = [ - ] # type: List[Tuple[interfaces.objects.ObjectInterface, Optional[str], interfaces.objects.ObjectInterface]] - self.bases = {} # type: Dict[str, Any] - self.user_types = {} # type: Dict[str, Any] - self.enumerations = {} # type: Dict[str, Any] - self.symbols = {} # type: Dict[str, Any] - self._omap_mapping = [] # type: List[Tuple[int, int]] - self._sections = [] # type: List[interfaces.objects.ObjectInterface] + self.types: List[ + Tuple[interfaces.objects.ObjectInterface, Optional[str], interfaces.objects.ObjectInterface]] = [ + ] + self.bases: Dict[str, Any] = {} + self.user_types: Dict[str, Any] = {} + self.enumerations: Dict[str, Any] = {} + self.symbols: Dict[str, Any] = {} + self._omap_mapping: List[Tuple[int, int]] = [] + self._sections: List[interfaces.objects.ObjectInterface] = [] self.metadata = {"format": "6.1.0", "windows": {}} self._database_name = database_name @@ -350,32 +354,34 @@ def read_ipi_stream(self): ipi_list = [] - type_references = self._read_info_stream(4, "IPI", ipi_list) - - for name in type_references.keys(): - # This doesn't break, because we want to use the last string/pdbname in the list - if name.endswith('.pdb'): - self._database_name = name.split('\\')[-1] + try: + type_references = self._read_info_stream(4, "IPI", ipi_list) + for name in type_references.keys(): + # This doesn't break, because we want to use the last string/pdbname in the list + if name.endswith('.pdb'): + self._database_name = name.split('\\')[-1] + except ValueError: + return None def _read_info_stream(self, stream_number, stream_name, info_list): - vollog.debug("Reading {}".format(stream_name)) + vollog.debug(f"Reading {stream_name}") info_layer = self._context.layers.get(self._layer_name + "_stream" + str(stream_number), None) if not info_layer: - raise ValueError("No TPI stream available") + raise ValueError(f"No {stream_name} stream available") module = self._context.module(module_name = info_layer.pdb_symbol_table, layer_name = info_layer.name, offset = 0) header = module.object(object_type = "TPI_HEADER", offset = 0) # Check the header if not (56 <= header.header_size < 1024): - raise ValueError("{} Stream Header size outside normal bounds".format(stream_name)) + raise ValueError(f"{stream_name} Stream Header size outside normal bounds") if header.index_min < 4096: - raise ValueError("Minimum {} index is 4096, found: {}".format(stream_name, header.index_min)) + raise ValueError(f"Minimum {stream_name} index is 4096, found: {header.index_min}") if header.index_max < header.index_min: raise ValueError("Maximum {} index is smaller than minimum TPI index, found: {} < {} ".format( stream_name, header.index_max, header.index_min)) # Reset the state - info_references = {} # type: Dict[str, int] + info_references: Dict[str, int] = {} offset = header.header_size # Ensure we use the same type everywhere length_type = "unsigned short" @@ -390,8 +396,8 @@ def _read_info_stream(self, stream_number, stream_name, info_list): output, consumed = self.consume_type(module, offset, length) leaf_type, name, value = output for tag_type in ['unnamed', 'anonymous']: - if name == '<{}-tag>'.format(tag_type) or name == '__{}'.format(tag_type): - name = '__{}_'.format(tag_type) + hex(len(info_list) + 0x1000)[2:] + if name == f'<{tag_type}-tag>' or name == f'__{tag_type}': + name = f'__{tag_type}_' + hex(len(info_list) + 0x1000)[2:] if name: info_references[name] = len(info_list) info_list.append((leaf_type, name, value)) @@ -478,16 +484,16 @@ def read_symbol_stream(self): name = None address = None if sym.segment < len(self._sections): - if leaf_type == 0x110e: - # v3 symbol (c-string) - name = self.parse_string(sym.name, False, sym.length - sym.vol.size + 2) - address = self._sections[sym.segment - 1].VirtualAddress + sym.offset - elif leaf_type == 0x1009: + if leaf_type == 0x1009: # v2 symbol (pascal-string) name = self.parse_string(sym.name, True, sym.length - sym.vol.size + 2) address = self._sections[sym.segment - 1].VirtualAddress + sym.offset + elif leaf_type == 0x110e or leaf_type == 0x1127: + # v3 symbol (c-string) + name = self.parse_string(sym.name, False, sym.length - sym.vol.size + 2) + address = self._sections[sym.segment - 1].VirtualAddress + sym.offset else: - vollog.debug("Only v2 and v3 symbols are supported") + vollog.debug(f"Only v2 and v3 symbols are supported: {leaf_type:x}") if name: if self._omap_mapping: address = self.omap_lookup(address) @@ -578,9 +584,9 @@ def get_json(self): def get_type_from_index(self, index: int) -> Union[List[Any], Dict[str, Any]]: """Takes a type index and returns appropriate dictionary.""" if index < 0x1000: - base_name, base = primatives[index & 0xff] + base_name, base = primitives[index & 0xff] self.bases[base_name] = base - result = {"kind": "base", "name": base_name} # type: Union[List[Dict[str, Any]], Dict[str, Any]] + result: Union[List[Dict[str, Any]], Dict[str, Any]] = {"kind": "base", "name": base_name} indirection = (index & 0xf00) if indirection: pointer_name, pointer_base = indirections[indirection] @@ -633,18 +639,18 @@ def get_size_from_index(self, index: int) -> int: """Returns the size of the structure based on the type index provided.""" result = -1 - name = '' # type: Optional[str] + name: Optional[str] = '' if index < 0x1000: if (index & 0xf00): _, base = indirections[index & 0xf00] else: - _, base = primatives[index & 0xff] + _, base = primitives[index & 0xff] result = base['size'] else: leaf_type, name, value = self.types[index - 0x1000] if leaf_type in [ - leaf_type.LF_UNION, leaf_type.LF_CLASS, leaf_type.LF_CLASS_ST, leaf_type.LF_STRUCTURE, - leaf_type.LF_STRUCTURE_ST, leaf_type.LF_INTERFACE + leaf_type.LF_UNION, leaf_type.LF_CLASS, leaf_type.LF_CLASS_ST, leaf_type.LF_STRUCTURE, + leaf_type.LF_STRUCTURE_ST, leaf_type.LF_INTERFACE, leaf_type.LF_CLASS_VS19, leaf_type.LF_STRUCTURE_VS19 ]: if not value.properties.forward_reference: result = value.size @@ -668,9 +674,9 @@ def get_size_from_index(self, index: int) -> int: elif leaf_type in [leaf_type.LF_PROCEDURE]: raise ValueError("LF_PROCEDURE size could not be identified") else: - raise ValueError("Unable to determine size of leaf_type {}".format(leaf_type.lookup())) + raise ValueError(f"Unable to determine size of leaf_type {leaf_type.lookup()}") if result <= 0: - raise ValueError("Invalid size identified: {} ({})".format(index, name)) + raise ValueError(f"Invalid size identified: {index} ({name})") return result ### TYPE HANDLING CODE @@ -688,8 +694,8 @@ def process_types(self, type_references: Dict[str, int]) -> None: self._progress_callback(index * 100 / max_len, "Processing types") leaf_type, name, value = self.types[index] if leaf_type in [ - leaf_type.LF_CLASS, leaf_type.LF_CLASS_ST, leaf_type.LF_STRUCTURE, leaf_type.LF_STRUCTURE_ST, - leaf_type.LF_INTERFACE + leaf_type.LF_CLASS, leaf_type.LF_CLASS_ST, leaf_type.LF_STRUCTURE, leaf_type.LF_STRUCTURE_ST, + leaf_type.LF_INTERFACE, leaf_type.LF_CLASS_VS19, leaf_type.LF_STRUCTURE_VS19 ]: if not value.properties.forward_reference and name: self.user_types[name] = { @@ -722,10 +728,40 @@ def process_types(self, type_references: Dict[str, int]) -> None: # Re-run through for ForwardSizeReferences self.user_types = self.replace_forward_references(self.user_types, type_references) + type_handlers = { + # Leaf_type: ('Structure', has_name, value_attribute) + 'LF_CLASS': ('LF_STRUCTURE', True, 'size'), + 'LF_CLASS_ST': ('LF_STRUCTURE', True, 'size'), + 'LF_STRUCTURE': ('LF_STRUCTURE', True, 'size'), + 'LF_STRUCTURE_ST': ('LF_STRUCTURE', True, 'size'), + 'LF_INTERFACE': ('LF_STRUCTURE', True, 'size'), + 'LF_CLASS_VS19': ('LF_STRUCTURE_VS19', True, 'size'), + 'LF_STRUCTURE_VS19': ('LF_STRUCTURE_VS19', True, 'size'), + 'LF_MEMBER': ('LF_MEMBER', True, 'offset'), + 'LF_MEMBER_ST': ('LF_MEMBER', True, 'offset'), + 'LF_ARRAY': ('LF_ARRAY', True, 'size'), + 'LF_ARRAY_ST': ('LF_ARRAY', True, 'size'), + 'LF_STRIDED_ARRAY': ('LF_ARRAY', True, 'size'), + 'LF_ENUMERATE': ('LF_ENUMERATE', True, 'value'), + 'LF_ARGLIST': ('LF_ENUM', True, None), + 'LF_ENUM': ('LF_ENUM', True, None), + 'LF_UNION': ('LF_UNION', True, None), + 'LF_STRING_ID': ('LF_STRING_ID', True, None), + 'LF_FUNC_ID': ('LF_FUNC_ID', True, None), + 'LF_MODIFIER': ('LF_MODIFIER', False, None), + 'LF_POINTER': ('LF_POINTER', False, None), + 'LF_PROCEDURE': ('LF_PROCEDURE', False, None), + 'LF_FIELDLIST': ('LF_FIELDLIST', False, None), + 'LF_BITFIELD': ('LF_BITFIELD', False, None), + 'LF_UDT_SRC_LINE': ('LF_UDT_SRC_LINE', False, None), + 'LF_UDT_MOD_SRC_LINE': ('LF_UDT_MOD_SRC_LINE', False, None), + 'LF_BUILDINFO': ('LF_BUILDINFO', False, None) + } + def consume_type( - self, module: interfaces.context.ModuleInterface, offset: int, length: int + self, module: interfaces.context.ModuleInterface, offset: int, length: int ) -> Tuple[Tuple[Optional[interfaces.objects.ObjectInterface], Optional[str], Union[ - None, List, interfaces.objects.ObjectInterface]], int]: + None, List, interfaces.objects.ObjectInterface]], int]: """Returns a (leaf_type, name, object) Tuple for a type, and the number of bytes consumed.""" leaf_type = self.context.object(module.get_enumeration("LEAF_TYPE"), @@ -734,61 +770,10 @@ def consume_type( consumed = leaf_type.vol.base_type.size remaining = length - consumed - if leaf_type in [ - leaf_type.LF_CLASS, leaf_type.LF_CLASS_ST, leaf_type.LF_STRUCTURE, leaf_type.LF_STRUCTURE_ST, - leaf_type.LF_INTERFACE - ]: - structure = module.object(object_type = "LF_STRUCTURE", offset = offset + consumed) - name_offset = structure.name.vol.offset - structure.vol.offset - name, value, excess = self.determine_extended_value(leaf_type, structure.size, module, - remaining - name_offset) - structure.size = value - structure.name = name - consumed += remaining - result = leaf_type, name, structure - elif leaf_type in [leaf_type.LF_MEMBER, leaf_type.LF_MEMBER_ST]: - member = module.object(object_type = "LF_MEMBER", offset = offset + consumed) - name_offset = member.name.vol.offset - member.vol.offset - name, value, excess = self.determine_extended_value(leaf_type, member.offset, module, - remaining - name_offset) - member.offset = value - member.name = name - result = leaf_type, name, member - consumed += member.vol.size + len(name) + 1 + excess - elif leaf_type in [leaf_type.LF_ARRAY, leaf_type.LF_ARRAY_ST, leaf_type.LF_STRIDED_ARRAY]: - array = module.object(object_type = "LF_ARRAY", offset = offset + consumed) - name_offset = array.name.vol.offset - array.vol.offset - name, value, excess = self.determine_extended_value(leaf_type, array.size, module, remaining - name_offset) - array.size = value - array.name = name - result = leaf_type, name, array - consumed += remaining - elif leaf_type in [leaf_type.LF_ENUMERATE]: - enum = module.object(object_type = 'LF_ENUMERATE', offset = offset + consumed) - name_offset = enum.name.vol.offset - enum.vol.offset - name, value, excess = self.determine_extended_value(leaf_type, enum.value, module, remaining - name_offset) - enum.value = value - enum.name = name - result = leaf_type, name, enum - consumed += enum.vol.size + len(name) + 1 + excess - elif leaf_type in [leaf_type.LF_ARGLIST, leaf_type.LF_ENUM]: - enum = module.object(object_type = "LF_ENUM", offset = offset + consumed) - name_offset = enum.name.vol.offset - enum.vol.offset - name = self.parse_string(enum.name, leaf_type < leaf_type.LF_ST_MAX, size = remaining - name_offset) - enum.name = name - result = leaf_type, name, enum - consumed += remaining - elif leaf_type in [leaf_type.LF_UNION]: - union = module.object(object_type = "LF_UNION", offset = offset + consumed) - name_offset = union.name.vol.offset - union.vol.offset - name = self.parse_string(union.name, leaf_type < leaf_type.LF_ST_MAX, size = remaining - name_offset) - result = leaf_type, name, union - consumed += remaining - elif leaf_type in [leaf_type.LF_MODIFIER, leaf_type.LF_POINTER, leaf_type.LF_PROCEDURE]: - obj = module.object(object_type = leaf_type.lookup(), offset = offset + consumed) - result = leaf_type, None, obj - consumed += remaining - elif leaf_type in [leaf_type.LF_FIELDLIST]: + type_handler, has_name, value_attribute = self.type_handlers.get(leaf_type.lookup(), + ('LF_UNKNOWN', False, None)) + + if type_handler in ['LF_FIELDLIST']: sub_length = remaining sub_offset = offset + consumed fields = [] @@ -800,25 +785,31 @@ def consume_type( consumed += sub_consumed fields.append(subfield) result = leaf_type, None, fields - elif leaf_type in [leaf_type.LF_BITFIELD]: - bitfield = module.object(object_type = "LF_BITFIELD", offset = offset + consumed) - result = leaf_type, None, bitfield - consumed += remaining - elif leaf_type in [leaf_type.LF_STRING_ID, leaf_type.LF_FUNC_ID]: - string_id = module.object(object_type = leaf_type.lookup(), offset = offset + consumed) - name_offset = string_id.name.vol.offset - string_id.vol.offset - name = self.parse_string(string_id.name, leaf_type < leaf_type.LF_ST_MAX, size = remaining - name_offset) - result = leaf_type, name, string_id - elif leaf_type in [leaf_type.LF_UDT_SRC_LINE, leaf_type.LF_UDT_MOD_SRC_LINE]: - src_line = module.object(object_type = leaf_type.lookup(), offset = offset + consumed) - result = leaf_type, None, src_line - elif leaf_type in [leaf_type.LF_BUILDINFO]: - buildinfo = module.object(object_type = leaf_type.lookup(), offset = offset + consumed) - buildinfo.arguments.count = buildinfo.count - consumed += buildinfo.arguments.vol.size - result = leaf_type, None, buildinfo + elif type_handler in ['LF_BUILDINFO']: + parsed_obj = module.object(object_type = type_handler, offset = offset + consumed) + parsed_obj.arguments.count = parsed_obj.count + consumed += parsed_obj.arguments.vol.size + result = leaf_type, None, parsed_obj + elif type_handler in self.type_handlers: + parsed_obj = module.object(object_type = type_handler, offset = offset + consumed) + current_consumed = remaining + if has_name: + name_offset = parsed_obj.name.vol.offset - parsed_obj.vol.offset + if value_attribute: + name, value, excess = self.determine_extended_value(leaf_type, getattr(parsed_obj, value_attribute), + module, remaining - name_offset) + setattr(parsed_obj, value_attribute, value) + current_consumed = parsed_obj.vol.size + len(name) + 1 + excess + else: + name = self.parse_string(parsed_obj.name, leaf_type < leaf_type.LF_ST_MAX, + size = remaining - name_offset) + parsed_obj.name = name + else: + name = None + result = leaf_type, name, parsed_obj + consumed += current_consumed else: - raise TypeError("Unhandled leaf_type: {}".format(leaf_type)) + raise TypeError(f"Unhandled leaf_type: {leaf_type}") return result, consumed @@ -831,7 +822,7 @@ def consume_padding(self, layer_name: str, offset: int) -> int: def convert_fields(self, fields: int) -> Dict[Optional[str], Dict[str, Any]]: """Converts a field list into a list of fields.""" - result = {} # type: Dict[Optional[str], Dict[str, Any]] + result: Dict[Optional[str], Dict[str, Any]] = {} _, _, fields_struct = self.types[fields] if not isinstance(fields_struct, list): vollog.warning("Fields structure did not contain a list of fields") @@ -928,27 +919,29 @@ def retreive_pdb(self, vollog.info("Download PDB file...") file_name = ".".join(file_name.split(".")[:-1] + ['pdb']) for sym_url in ['http://msdl.microsoft.com/download/symbols']: - url = sym_url + "/{}/{}/".format(file_name, guid) + url = sym_url + f"/{file_name}/{guid}/" result = None for suffix in [file_name, file_name[:-1] + '_']: try: - vollog.debug("Attempting to retrieve {}".format(url + suffix)) + vollog.debug(f"Attempting to retrieve {url + suffix}") + # We have to cache this because the file is opened by a layer and we can't control whether that caches result = resources.ResourceAccessor(progress_callback).open(url + suffix) - except error.HTTPError as excp: - vollog.debug("Failed with {}".format(excp)) - if result: - break + except (error.HTTPError, error.URLError) as excp: + vollog.debug(f"Failed with {excp}") + if result: + break if progress_callback is not None: - progress_callback(100, "Downloading {}".format(url + suffix)) + progress_callback(100, f"Downloading {url + suffix}") if result is None: return None - return result.name + return url + suffix if __name__ == '__main__': import argparse + class PrintedProgress(object): """A progress handler that prints the progress value and the description onto the command line.""" @@ -964,14 +957,15 @@ def __call__(self, progress: Union[int, float], description: str = None): Args: progress: Percentage of progress of the current procedure """ - message = "\rProgress: {0: 7.2f}\t\t{1:}".format(round(progress, 2), description or '') + message = f"\rProgress: {round(progress, 2): 7.2f}\t\t{description or ''}" message_len = len(message) self._max_message_len = max([self._max_message_len, message_len]) print(message, end = (' ' * (self._max_message_len - message_len)) + '\r') + parser = argparse.ArgumentParser( description = "Read PDB files and convert to Volatility 3 Intermediate Symbol Format") - parser.add_argument("-o", "--output", metavar = "OUTPUT", help = "Filename for data output", required = True) + parser.add_argument("-o", "--output", metavar = "OUTPUT", help = "Filename for data output", default = None) file_group = parser.add_argument_group("file", description = "File-based conversion of PDB to ISF") file_group.add_argument("-f", "--file", metavar = "FILE", help = "PDB file to translate to ISF") data_group = parser.add_argument_group("data", description = "Convert based on a GUID and filename pattern") @@ -994,7 +988,10 @@ def __call__(self, progress: Union[int, float], description: str = None): filename = None if args.guid is not None and args.pattern is not None: filename = PdbRetreiver().retreive_pdb(guid = args.guid, file_name = args.pattern, progress_callback = pg_cb) - delfile = True + if filename is None: + parser.error("PDB file could not be retrieved from the internet") + if parse.urlparse(filename, 'file').scheme == 'file': + delfile = True elif args.file: filename = args.file else: @@ -1004,16 +1001,41 @@ def __call__(self, progress: Union[int, float], description: str = None): parser.error("No suitable filename provided or retrieved") ctx = contexts.Context() - if not os.path.exists(filename): - parser.error("File {} does not exists".format(filename)) - location = "file:" + request.pathname2url(filename) + url = parse.urlparse(filename, scheme = 'file') + if url.scheme == 'file': + if not os.path.exists(filename): + parser.error(f"File {filename} does not exists") + location = "file:" + request.pathname2url(os.path.abspath(filename)) + else: + location = filename convertor = PdbReader(ctx, location, database_name = args.pattern, progress_callback = pg_cb) - with open(args.output, "w") as f: - json.dump(convertor.get_json(), f, indent = 2, sort_keys = True) + converted_json = convertor.get_json() + if args.output is None: + if args.guid: + guid = args.guid[:-1] + age = args.guid[-1:] + else: + guid = converted_json['metadata']['windows']['pdb']['GUID'] + age = converted_json['metadata']['windows']['pdb']['age'] + args.output = f"{guid}-{age}.json.xz" + + output_url = os.path.abspath(args.output) + + open_method = open + if args.output.endswith('.gz'): + open_method = gzip.open + elif args.output.endswith('.bz2'): + open_method = bz2.open + elif args.output.endswith('.xz'): + open_method = lzma.open + + with open_method(output_url, "wb") as f: + json_string = json.dumps(converted_json, indent = 2, sort_keys = True) + f.write(bytes(json_string, 'latin-1')) if args.keep: - print("Temporary PDB file: {}".format(filename)) + print(f"Temporary PDB file: {filename}") elif delfile: os.remove(filename) diff --git a/volatility3/framework/symbols/windows/pdbutil.py b/volatility3/framework/symbols/windows/pdbutil.py index 653c953b12..4c3788a567 100644 --- a/volatility3/framework/symbols/windows/pdbutil.py +++ b/volatility3/framework/symbols/windows/pdbutil.py @@ -7,12 +7,13 @@ import logging import lzma import os +import re import struct from typing import Any, Dict, Generator, List, Optional, Tuple, Union -from urllib import request +from urllib import request, parse from volatility3 import symbols -from volatility3.framework import constants, interfaces +from volatility3.framework import constants, interfaces, exceptions from volatility3.framework.configuration.requirements import SymbolTableRequirement from volatility3.framework.symbols import intermed from volatility3.framework.symbols.windows import pdbconv @@ -20,9 +21,12 @@ vollog = logging.getLogger(__name__) -class PDBUtility: +class PDBUtility(interfaces.configuration.VersionableInterface): """Class to handle and manage all getting symbols based on MZ header""" + _version = (1, 0, 0) + _required_framework_version = (2, 0, 0) + @classmethod def symbol_table_from_offset( cls, @@ -64,11 +68,11 @@ def load_windows_symbol_table(cls, symbol_table_class: str, config_path: str = 'pdbutility', progress_callback: constants.ProgressCallback = None): - """Loads (downlading if necessary) a windows symbol table""" + """Loads (downloading if necessary) a windows symbol table""" filter_string = os.path.join(pdb_name.strip('\x00'), guid.upper() + "-" + str(age)) - isf_path = False + isf_path = None # Take the first result of search for the intermediate file for value in intermed.IntermediateSymbolTable.file_symbol_url("windows", filter_string): isf_path = value @@ -82,10 +86,13 @@ def load_windows_symbol_table(cls, break if not isf_path: - vollog.debug("Required symbol library path not found: {}".format(filter_string)) + vollog.debug(f"Required symbol library path not found: {filter_string}") + vollog.info("The symbols can be downloaded later using pdbconv.py -p {} -g {}".format( + pdb_name.strip('\x00'), + guid.upper() + str(age))) return None - vollog.debug("Using symbol library: {}".format(filter_string)) + vollog.debug(f"Using symbol library: {filter_string}") # Set the discovered options join = interfaces.configuration.path_join @@ -163,9 +170,9 @@ def get_guid_from_mz(cls, context: interfaces.context.ContextInterface, layer_na pdb_name = debug_entry.PdbFileName.decode("utf-8").strip('\x00') age = debug_entry.Age - guid = "{:x}{:x}{:x}{}".format(debug_entry.Signature_Data1, debug_entry.Signature_Data2, - debug_entry.Signature_Data3, - binascii.hexlify(debug_entry.Signature_Data4).decode('utf-8')) + guid = "{:08x}{:04x}{:04x}{}".format(debug_entry.Signature_Data1, debug_entry.Signature_Data2, + debug_entry.Signature_Data3, + binascii.hexlify(debug_entry.Signature_Data4).decode('utf-8')) return guid, age, pdb_name @classmethod @@ -193,14 +200,18 @@ def download_pdb_isf(cls, file_name = pdb_name, progress_callback = progress_callback) if filename: - tmp_files.append(filename) - location = "file:" + request.pathname2url(tmp_files[-1]) + url = parse.urlparse(filename, scheme = 'file') + if url.scheme == 'file' or len(url.scheme) == 1: + tmp_files.append(filename) + location = "file:" + request.pathname2url(os.path.abspath(tmp_files[-1])) + else: + location = filename json_output = pdbconv.PdbReader(context, location, pdb_name, progress_callback).get_json() of.write(bytes(json.dumps(json_output, indent = 2, sort_keys = True), 'utf-8')) # After we've successfully written it out, record the fact so we don't clear it out data_written = True else: - vollog.warning("Symbol file could not be found on remote server" + (" " * 100)) + vollog.warning("Symbol file could not be downloaded from remote server" + (" " * 100)) break except PermissionError: vollog.warning("Cannot write necessary symbol file, please check permissions on {}".format( @@ -215,7 +226,7 @@ def download_pdb_isf(cls, try: os.remove(filename) except PermissionError: - vollog.warning("Temporary file could not be removed: {}".format(filename)) + vollog.warning(f"Temporary file could not be removed: {filename}") else: vollog.warning("Cannot write downloaded symbols, please add the appropriate symbols" " or add/modify a symbols directory that is writable") @@ -272,6 +283,47 @@ def pdbname_scan(cls, 'mz_offset': mz_offset } + @classmethod + def symbol_table_from_pdb(cls, context: interfaces.context.ContextInterface, config_path: str, layer_name: str, + pdb_name: str, module_offset: int, module_size: int) -> str: + """Creates symbol table for a module in the specified layer_name. + + Searches the memory section of the loaded module for its PDB GUID + and loads the associated symbol table into the symbol space. + + Args: + context: The context to retrieve required elements (layers, symbol tables) from + config_path: The config path where to find symbol files + layer_name: The name of the layer on which to operate + module_offset: This memory dump's module image offset + module_size: The size of the module for this dump + + Returns: + The name of the constructed and loaded symbol table + """ + + guids = list( + cls.pdbname_scan(context, + layer_name, + context.layers[layer_name].page_size, [bytes(pdb_name, 'latin-1')], + start = module_offset, + end = module_offset + module_size)) + + if not guids: + raise exceptions.VolatilityException( + f"Did not find GUID of {pdb_name} in module @ 0x{module_offset:x}!") + + guid = guids[0] + + vollog.debug(f"Found {guid['pdb_name']}: {guid['GUID']}-{guid['age']}") + + return cls.load_windows_symbol_table(context, + guid["GUID"], + guid["age"], + guid["pdb_name"], + "volatility3.framework.symbols.intermed.IntermediateSymbolTable", + config_path = config_path) + class PdbSignatureScanner(interfaces.layers.ScannerInterface): """A :class:`~volatility3.framework.interfaces.layers.ScannerInterface` @@ -294,20 +346,15 @@ def __init__(self, pdb_names: List[bytes]) -> None: self._pdb_names = pdb_names def __call__(self, data: bytes, data_offset: int) -> Generator[Tuple[str, Any, bytes, int], None, None]: - sig = data.find(b"RSDS") - while sig >= 0: - null = data.find(b'\0', sig + 4 + self._RSDS_format.size) - if null > -1: - if (null - sig - self._RSDS_format.size) <= 100: - name_offset = sig + 4 + self._RSDS_format.size - pdb_name = data[name_offset:null] - if pdb_name in self._pdb_names: - - ## this ordering is intentional due to mixed endianness in the GUID - (g3, g2, g1, g0, g5, g4, g7, g6, g8, g9, ga, gb, gc, gd, ge, gf, a) = \ - self._RSDS_format.unpack(data[sig + 4:name_offset]) - - guid = (16 * '{:02X}').format(g0, g1, g2, g3, g4, g5, g6, g7, g8, g9, ga, gb, gc, gd, ge, gf) - if sig < self.chunk_size: - yield (guid, a, pdb_name, data_offset + sig) - sig = data.find(b"RSDS", sig + 1) + pattern = b'RSDS' + (b'.' * self._RSDS_format.size) + b'(' + b'|'.join( + [re.escape(x) for x in self._pdb_names]) + b')\x00' + for match in re.finditer(pattern, data, flags = re.DOTALL): + pdb_name = data[match.start(0) + 4 + self._RSDS_format.size:match.start(0) + len(match.group()) - 1] + if pdb_name in self._pdb_names: + ## this ordering is intentional due to mixed endianness in the GUID + (g3, g2, g1, g0, g5, g4, g7, g6, g8, g9, ga, gb, gc, gd, ge, gf, a) = \ + self._RSDS_format.unpack(data[match.start(0) + 4:match.start(0) + 4 + self._RSDS_format.size]) + + guid = (16 * '{:02X}').format(g0, g1, g2, g3, g4, g5, g6, g7, g8, g9, ga, gb, gc, gd, ge, gf) + if match.start(0) < self.chunk_size: + yield (guid, a, pdb_name, match.start(0)) diff --git a/volatility3/framework/symbols/windows/poolheader-x86.json b/volatility3/framework/symbols/windows/poolheader-x86.json index d117a8b2c8..72f2c6cc1c 100644 --- a/volatility3/framework/symbols/windows/poolheader-x86.json +++ b/volatility3/framework/symbols/windows/poolheader-x86.json @@ -55,7 +55,7 @@ } }, "kind": "struct", - "size": 16 + "size": 8 } }, "symbols": { diff --git a/volatility3/framework/symbols/windows/services-vista-x64.json b/volatility3/framework/symbols/windows/services/services-vista-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/services-vista-x64.json rename to volatility3/framework/symbols/windows/services/services-vista-x64.json diff --git a/volatility3/framework/symbols/windows/services-vista-x86.json b/volatility3/framework/symbols/windows/services/services-vista-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/services-vista-x86.json rename to volatility3/framework/symbols/windows/services/services-vista-x86.json diff --git a/volatility3/framework/symbols/windows/services-win10-15063-x64.json b/volatility3/framework/symbols/windows/services/services-win10-15063-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/services-win10-15063-x64.json rename to volatility3/framework/symbols/windows/services/services-win10-15063-x64.json diff --git a/volatility3/framework/symbols/windows/services-win10-15063-x86.json b/volatility3/framework/symbols/windows/services/services-win10-15063-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/services-win10-15063-x86.json rename to volatility3/framework/symbols/windows/services/services-win10-15063-x86.json diff --git a/volatility3/framework/symbols/windows/services-win10-16299-x64.json b/volatility3/framework/symbols/windows/services/services-win10-16299-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/services-win10-16299-x64.json rename to volatility3/framework/symbols/windows/services/services-win10-16299-x64.json diff --git a/volatility3/framework/symbols/windows/services-win10-16299-x86.json b/volatility3/framework/symbols/windows/services/services-win10-16299-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/services-win10-16299-x86.json rename to volatility3/framework/symbols/windows/services/services-win10-16299-x86.json diff --git a/volatility3/framework/symbols/windows/services-win8-x64.json b/volatility3/framework/symbols/windows/services/services-win8-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/services-win8-x64.json rename to volatility3/framework/symbols/windows/services/services-win8-x64.json diff --git a/volatility3/framework/symbols/windows/services-win8-x86.json b/volatility3/framework/symbols/windows/services/services-win8-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/services-win8-x86.json rename to volatility3/framework/symbols/windows/services/services-win8-x86.json diff --git a/volatility3/framework/symbols/windows/services-xp-2003-x64.json b/volatility3/framework/symbols/windows/services/services-xp-2003-x64.json similarity index 100% rename from volatility3/framework/symbols/windows/services-xp-2003-x64.json rename to volatility3/framework/symbols/windows/services/services-xp-2003-x64.json diff --git a/volatility3/framework/symbols/windows/services-xp-x86.json b/volatility3/framework/symbols/windows/services/services-xp-x86.json similarity index 100% rename from volatility3/framework/symbols/windows/services-xp-x86.json rename to volatility3/framework/symbols/windows/services/services-xp-x86.json diff --git a/volatility3/plugins/windows/registry/certificates.py b/volatility3/plugins/windows/registry/certificates.py index 909e852fb0..96a7e39775 100644 --- a/volatility3/plugins/windows/registry/certificates.py +++ b/volatility3/plugins/windows/registry/certificates.py @@ -10,7 +10,7 @@ class Certificates(interfaces.plugins.PluginInterface): """Lists the certificates in the registry's Certificate Store.""" - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: @@ -50,7 +50,7 @@ def _generator(self) -> Iterator[Tuple[int, Tuple[str, str, str, str]]]: node_path = hive.get_key(top_key, return_list = True) for (depth, is_key, last_write_time, key_path, volatility, node) in printkey.PrintKey.key_iterator(hive, node_path, recurse = True): - if not is_key and RegValueTypes.get(node.Type).name == "REG_BINARY": + if not is_key and RegValueTypes(node.Type).name == "REG_BINARY": name, certificate_data = self.parse_data(node.decode_data()) unique_key_offset = key_path.index(top_key) + len(top_key) + 1 reg_section = key_path[unique_key_offset:key_path.index("\\", unique_key_offset)] diff --git a/volatility3/plugins/windows/statistics.py b/volatility3/plugins/windows/statistics.py index b3bc8044f1..e6f2016ed2 100644 --- a/volatility3/plugins/windows/statistics.py +++ b/volatility3/plugins/windows/statistics.py @@ -13,7 +13,7 @@ class Statistics(plugins.PluginInterface): - _required_framework_version = (1, 0, 0) + _required_framework_version = (2, 0, 0) @classmethod def get_requirements(cls) -> List[interfaces.configuration.RequirementInterface]: diff --git a/volatility3/schemas/__init__.py b/volatility3/schemas/__init__.py index ea7f0579d6..65329a4f5f 100644 --- a/volatility3/schemas/__init__.py +++ b/volatility3/schemas/__init__.py @@ -18,7 +18,7 @@ def load_cached_validations() -> Set[str]: """Loads up the list of successfully cached json objects, so we don't need to revalidate them.""" - validhashes = set() # type: Set + validhashes: Set = set() if os.path.exists(cached_validation_filepath): with open(cached_validation_filepath, "r") as f: validhashes.update(json.load(f)) @@ -44,7 +44,7 @@ def validate(input: Dict[str, Any], use_cache: bool = True) -> bool: basepath = os.path.abspath(os.path.dirname(__file__)) schema_path = os.path.join(basepath, 'schema-' + format + '.json') if not os.path.exists(schema_path): - vollog.debug("Schema for format not found: {}".format(schema_path)) + vollog.debug(f"Schema for format not found: {schema_path}") return False with open(schema_path, 'r') as s: schema = json.load(s) @@ -53,7 +53,7 @@ def validate(input: Dict[str, Any], use_cache: bool = True) -> bool: def create_json_hash(input: Dict[str, Any], schema: Dict[str, Any]) -> str: """Constructs the hash of the input and schema to create a unique - indentifier for a particular JSON file.""" + identifier for a particular JSON file.""" return hashlib.sha1(bytes(json.dumps((input, schema), sort_keys = True), 'utf-8')).hexdigest()