diff --git a/.github/workflows/sync-with-master.yml b/.github/workflows/sync-with-master.yml new file mode 100644 index 0000000..a6a7a44 --- /dev/null +++ b/.github/workflows/sync-with-master.yml @@ -0,0 +1,21 @@ +on: + push: + branches: + - master + schedule: + - cron: '0 0 * * *' + +jobs: + sync: + strategy: + matrix: + branch: + - 6.0/stage + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + - uses: delphix/actions/sync-with-master@master + with: + branch-to-sync: ${{ matrix.branch }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/sync-with-upstream.yml b/.github/workflows/sync-with-upstream.yml new file mode 100644 index 0000000..6ec3384 --- /dev/null +++ b/.github/workflows/sync-with-upstream.yml @@ -0,0 +1,18 @@ +on: + schedule: + - cron: '0 * * * *' + +jobs: + sync: + runs-on: ubuntu-18.04 + steps: + - uses: actions/checkout@v2 + with: + token: ${{ secrets.DEVOPS_AUTOMATION_TOKEN }} + - uses: delphix/actions/sync-with-upstream@master + with: + upstream-repository: https://github.com/sdimitro/savedump.git + upstream-branch: master + downstream-branch: master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/README.md b/README.md index 6391729..853ffe1 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,12 @@ # savedump -TL;DR; A Python hack I put together that does its best to archive -crash dumps and core dumps together with their required binaries -and debug info in Linux. +![](https://github.com/delphix/savedump/workflows/.github/workflows/main.yml/badge.svg) + +TL;DR; A Python script that creates a best-effort self-contained +archive of a kernel crash dump or userland core dump. The archive +contains the memory dump coupled together with any required +binaries and debug information that it could find at the time it +was invoked. ### Motivation @@ -35,6 +39,26 @@ dumps. What it does lack though is proper tooling to capture a self-contained dump from one system to analyze it in another. This is what this utility is attempting to help with. +### Installation + +Ensure you have the following dependencies: +* Python 3.6 or newer +* [libkdumpfile](https://github.com/ptesarik/libkdumpfile) +* [drgn](https://github.com/osandov/drgn/) +* [gdb](https://www.gnu.org/software/gdb/) + +Note that `libkdumpfile` and `drgn` are only needed for kernel +crash dumps. If you only need `savedump` for userland core dumps +then you only need `python3`. `gdb` is not a hard dependency +either but it is recommeneded for accurate archival of shared +objects in userland core dumps. + +Once all dependencies are installed clone this repo and +run the following command from the root of the repo: +``` +sudo python3 setup.py install +``` + ### How do I use it? To capture a crash dump or a core dump: @@ -99,9 +123,7 @@ As mentioned in the TL;DR; the utility is far from perfect but I do hope to add more functionality to it as cases arise. * [verify BuildID and/or SRCVERSION between dumps and binaries](https://github.com/sdimitro/savedump/issues/6) -* [only package the modules needed in crash dumps](https://github.com/sdimitro/savedump/issues/7) * [make the gdb dependency optional](https://github.com/sdimitro/savedump/issues/9) -* [make the libkdumpfile dependency optional](https://github.com/sdimitro/savedump/issues/8) * [support custom paths for binaries and debug info](https://github.com/sdimitro/savedump/issues/5) * [generate run-sdb.sh](https://github.com/sdimitro/savedump/issues/10) * [support for plain vmcores](https://github.com/sdimitro/savedump/issues/3) diff --git a/debian/control b/debian/control index a41f171..3cd633f 100644 --- a/debian/control +++ b/debian/control @@ -3,13 +3,7 @@ Section: misc Priority: optional Maintainer: Delphix Engineering Standards-Version: 4.1.2 -Build-Depends: debhelper (>= 9), - dh-python, - python3, - python3-distutils, - python3.6-dev, - zlib1g-dev -Suggests: libkdumpfile, drgn +Build-Depends: debhelper (>= 9), dh-python, python3 Package: savedump Architecture: any diff --git a/savedump/savedump.py b/savedump/savedump.py index a433440..8943fd4 100644 --- a/savedump/savedump.py +++ b/savedump/savedump.py @@ -19,8 +19,6 @@ """ import argparse -import distutils -from distutils import dir_util # pylint: disable=unused-import from enum import Enum import os import pathlib @@ -133,16 +131,83 @@ def get_dump_type(path: str) -> Optional[DumpType]: """ +def get_module_paths(osrelease: str, path: str) -> List[str]: + """ + Use drgn on the crash dump specified by `path` and return list + of paths from the `osrelease` kernel modules relevant to the + crash dump. + """ + # + # Similarly to libkdumpfile we import these libraries locally + # here so people who don't have drgn can still use savedump + # for userland core dumps. + # + import drgn # pylint: disable=import-outside-toplevel + from drgn.helpers.linux.list import list_for_each_entry # pylint: disable=import-outside-toplevel + + prog = drgn.program_from_core_dump(path) + + # + # First go through all modules in the dump and create a map + # of [key: module name] -> (value: module srcversion). + # + # Note: + # It would be prefereable to be able to use the binary's + # .build-id to do the matching instead of srcversion. + # Unfortunately there doesn't seem to be a straightforward + # way to get the build-id section of the ELF files recorded + # in the dump. Hopefully that changes in the future. + # + mod_name_srcvers = {} + for mod in list_for_each_entry('struct module', + prog['modules'].address_of_(), 'list'): + mod_name_srcvers[str(mod.name.string_(), + encoding='utf-8')] = str(mod.srcversion.string_(), + encoding='utf-8') + + # + # Go through all modules in /usr/lib/debug/lib/modules/ + # and gather the file paths of the ones that are part of our + # module name-to-srcversion map. + # + system_modules = pathlib.Path( + f"/usr/lib/debug/lib/modules/{osrelease}/").rglob('*.ko') + mod_paths = [] + for modpath in system_modules: + modname = os.path.basename(modpath)[:-3] + if not mod_name_srcvers.get(modname): + continue + + success, output = shell_cmd( + ['modinfo', '--field=srcversion', + str(modpath)]) + if not success: + sys.exit(output) + output = output.strip() + + if output != mod_name_srcvers[modname]: + continue + + mod_paths.append(str(modpath)) + del mod_name_srcvers[modname] + + print(f"found {len(mod_paths)} relevant modules with their debug info...") + print("warning: could not find the debug info of the following modules:") + print(f" {', '.join(mod_name_srcvers.keys())}") + return mod_paths + + def archive_kernel_dump(path: str) -> None: """ Packages the dump together with its vmlinux and modules in a gzipped archive in the working directory. """ + # pylint: disable=too-many-locals # - # We import libkdumpfile specifically here and not - # in the top-level to allow users that don't have + # We import drgn and libkdumpfile specifically here and + # not in the top-level to allow users that don't have # it installed to still be able to use savedump for - # core files. + # userland core files. # import kdumpfile # pylint: disable=import-outside-toplevel @@ -156,18 +221,17 @@ def archive_kernel_dump(path: str) -> None: sys.exit(f"error: cannot find vmlinux at: {vmlinux_path}") print(f"vmlinux found: {vmlinux_path}") - extra_mod_path = f"/usr/lib/debug/lib/modules/{osrelease}/extra" - if not os.path.exists(extra_mod_path): - sys.exit(f"error: cannot find extra mod path: {extra_mod_path}") - print(f"using module path: {extra_mod_path}") + mod_paths = get_module_paths(osrelease, path) archive_dir = f"{nodename}.archive-{dumpname}" pathlib.Path(archive_dir).mkdir(parents=True, exist_ok=True) shutil.copy(path, archive_dir) shutil.copy(vmlinux_path, archive_dir) - archive_extra_mod_path = f"{archive_dir}{extra_mod_path}" - distutils.dir_util.copy_tree(extra_mod_path, archive_extra_mod_path) + for mod_path in mod_paths: + archive_mod_path = f"{archive_dir}{mod_path}" + os.makedirs(os.path.dirname(archive_mod_path), exist_ok=True) + shutil.copy(mod_path, archive_mod_path) # # Generate run-sdb.sh.