Merge pull request #1008 from onekey-sec/drop-python3.8

Drop python 3.8 support

.github/actions/setup-dependencies/action.yml

@@ -2,7 +2,7 @@ inputs:
   python-version:
     description: "Python version to setup"
     required: false
-    default: "3.8"
+    default: "3.9"
 
 name: "Setup dependencies"
 description: "Install all required dependencies for worflows to run."

.github/workflows/documentation.yml

@@ -16,18 +16,18 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: 3.9
 
       - name: Restore pip cache
         uses: actions/cache@v4
         with:
           path: ~/.cache/pip
-          key: pip-3.8
+          key: pip-3.9
 
       - name: Upgrade pip and install poetry
         run: python -m pip install --upgrade pip poetry
@@ -39,13 +39,13 @@ jobs:
           path: |
             ~/.cache/pypoetry/cache
             ~/.cache/pypoetry/artifacts
-          key: poetry-cache-and-artifacts-3.8
+          key: poetry-cache-and-artifacts-3.9
 
       - name: Restore virtualenvs
         uses: actions/cache@v4
         with:
           path: ~/.cache/pypoetry/virtualenvs
-          key: venv-${{ hashFiles('poetry.lock') }}-3.8
+          key: venv-${{ hashFiles('poetry.lock') }}-3.9
 
       - name: Poetry install
         run: poetry install --only docs

.github/workflows/run-tests.yml

@@ -49,7 +49,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
     steps:
       - name: Checkout source code
         uses: actions/checkout@v4

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.8-slim
+FROM python:3.12-slim
 
 RUN mkdir -p /data/input /data/output
 RUN useradd -m unblob

docs/development.md

@@ -26,7 +26,7 @@ where the exciting stuff is.
 
 ### Required tools
 
-- **Python**: unblob requires **Python 3.8** or above. Make sure that
+- **Python**: unblob requires **Python 3.9** or above. Make sure that
   [Python is installed](https://www.python.org/downloads/) on your system.
 
 - **git**: You need it for cloning the repository.
@@ -48,7 +48,7 @@ where the exciting stuff is.
 - **pyenv** (_Recommended_): When you are working with multiple versions of Python,
   pyenv makes it very easy to install and use different versions and make virtualenvs.
   Follow the [instructions on GitHub](https://github.com/pyenv/pyenv) for the installation.
-  If your system already has at least Python 3.8 installed, you don't need it.
+  If your system already has at least Python 3.9 installed, you don't need it.
 
 ### Cloning the Git repository
 
@@ -85,7 +85,7 @@ Or instead of Poetry you can use `pyenv`. You can set the Python interpreter
 version for the local folder only with:
 
 ```
-pyenv local 3.8.12
+pyenv local 3.12.7
 ```
 
 ### Installing Python dependencies

pyproject.toml

@@ -9,9 +9,9 @@ packages = [
 ]
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.9"
 click = "^8.1.7"
-"dissect.cstruct" = ">=2.0,<5.0"
+"dissect.cstruct" = ">=4.0,<5.0"
 attrs = ">=23.1.0"
 structlog = ">=24.1.0"
 arpy = "^2.3.0"
@@ -57,7 +57,7 @@ mkdocstrings-python = "^1.8.0"
 unblob = "unblob.cli:main"
 
 [tool.ruff]
-target-version = "py38"
+target-version = "py39"
 
 [tool.ruff.lint]
 select = [

tests/test_cli.py

@@ -1,5 +1,6 @@
+from collections.abc import Iterable
 from pathlib import Path
-from typing import Iterable, List, Optional, Tuple, Type
+from typing import Optional
 from unittest import mock
 
 import pytest
@@ -157,7 +158,7 @@ def test_help(params):
         ),
     ],
 )
-def test_without_file(params: List[str]):
+def test_without_file(params: list[str]):
     runner = CliRunner()
     result = runner.invoke(unblob.cli.cli, params)
     assert result.exit_code == 2
@@ -238,7 +239,7 @@ def test_archive_success(
     expected_randomness_depth: int,
     expected_process_num: int,
     expected_verbosity: int,
-    expected_progress_reporter: Type[ProgressReporter],
+    expected_progress_reporter: type[ProgressReporter],
     tmp_path: Path,
 ):
     runner = CliRunner()
@@ -254,9 +255,10 @@ def test_archive_success(
     process_file_mock = mock.MagicMock()
     logger_config_mock = mock.MagicMock()
     new_params = [*params, "--extract-dir", str(tmp_path), str(in_path)]
-    with mock.patch.object(
-        unblob.cli, "process_file", process_file_mock
-    ), mock.patch.object(unblob.cli, "configure_logger", logger_config_mock):
+    with (
+        mock.patch.object(unblob.cli, "process_file", process_file_mock),
+        mock.patch.object(unblob.cli, "configure_logger", logger_config_mock),
+    ):
         result = runner.invoke(unblob.cli.cli, new_params)
     assert result.exit_code == 0
     assert "error" not in result.output
@@ -285,7 +287,7 @@ def test_archive_success(
     ],
 )
 def test_keep_extracted_chunks(
-    args: List[str], keep_extracted_chunks: bool, fail_message: str, tmp_path: Path
+    args: list[str], keep_extracted_chunks: bool, fail_message: str, tmp_path: Path
 ):
     runner = CliRunner()
     in_path = (
@@ -321,7 +323,7 @@ def test_keep_extracted_chunks(
     ],
 )
 def test_skip_extension(
-    skip_extension: List[str], expected_skip_extensions: Tuple[str, ...], tmp_path: Path
+    skip_extension: list[str], expected_skip_extensions: tuple[str, ...], tmp_path: Path
 ):
     runner = CliRunner()
     in_path = (
@@ -355,7 +357,7 @@ def test_skip_extension(
     ],
 )
 def test_skip_extraction(
-    args: List[str], skip_extraction: bool, fail_message: str, tmp_path: Path
+    args: list[str], skip_extraction: bool, fail_message: str, tmp_path: Path
 ):
     runner = CliRunner()
     in_path = (
@@ -403,7 +405,7 @@ def test_skip_extraction(
     ],
 )
 def test_clear_skip_magics(
-    args: List[str], skip_magic: Iterable[str], fail_message: str, tmp_path: Path
+    args: list[str], skip_magic: Iterable[str], fail_message: str, tmp_path: Path
 ):
     runner = CliRunner()
     in_path = (

tests/test_file_utils.py

@@ -1,7 +1,6 @@
 import io
 import os
 from pathlib import Path
-from typing import List
 
 import pytest
 
@@ -296,7 +295,7 @@ def test_decode_invalid_values(self, value: bytes):
         (b"abcdef abcdef", b"not-found", []),
     ],
 )
-def test_iterate_patterns(content: bytes, pattern: bytes, expected: List[int]):
+def test_iterate_patterns(content: bytes, pattern: bytes, expected: list[int]):
     file = File.from_bytes(content)
     assert list(iterate_patterns(file, pattern)) == expected
 
@@ -320,7 +319,7 @@ def test_iterate_file(
     start_offset: int,
     size: int,
     buffer_size: int,
-    expected: List[bytes],
+    expected: list[bytes],
 ):
     file = File.from_bytes(content)
     assert list(iterate_file(file, start_offset, size, buffer_size)) == expected
@@ -512,7 +511,7 @@ def test_create_symlink(self, sandbox: FileSystem):
 
         output_path = sandbox.root / "symlink"
         assert not output_path.exists()
-        assert os.readlink(output_path) == "target file"
+        assert output_path.readlink() == Path("target file")
         assert sandbox.problems == []
 
     def test_create_symlink_target_inside_sandbox(self, sandbox: FileSystem):
@@ -525,7 +524,7 @@ def test_create_symlink_target_inside_sandbox(self, sandbox: FileSystem):
         output_path = sandbox.root / "sbin/shell"
         assert output_path.read_bytes() == b"posix shell"
         assert output_path.exists()
-        assert os.readlink(output_path) == "../bin/sh"
+        assert output_path.readlink() == Path("../bin/sh")
         assert sandbox.problems == []
 
     def test_create_symlink_target_outside_sandbox(self, sandbox: FileSystem):
@@ -545,7 +544,7 @@ def test_create_symlink_absolute_paths(self, sandbox: FileSystem):
 
         output_path = sandbox.root / "symlink"
         assert output_path.exists()
-        assert os.readlink(output_path) == "target file"
+        assert output_path.readlink() == Path("target file")
         assert sandbox.problems == []
 
     def test_create_symlink_absolute_paths_self_referenced(self, sandbox: FileSystem):
@@ -554,7 +553,7 @@ def test_create_symlink_absolute_paths_self_referenced(self, sandbox: FileSystem
 
         output_path = sandbox.root / "etc/passwd"
         assert not output_path.exists()
-        assert os.readlink(output_path) == "../etc/passwd"
+        assert output_path.readlink() == Path("../etc/passwd")
         assert sandbox.problems == []
 
     def test_create_symlink_outside_sandbox(self, sandbox: FileSystem):

tests/test_handlers.py

@@ -9,7 +9,6 @@
 
 import inspect
 from pathlib import Path
-from typing import Type
 
 import pytest
 
@@ -43,7 +42,7 @@ def test_all_handlers(
     "handler",
     (pytest.param(handler, id=handler.NAME) for handler in handlers.BUILTIN_HANDLERS),
 )
-def test_missing_handlers_integrations_tests(handler: Type[Handler]):
+def test_missing_handlers_integrations_tests(handler: type[Handler]):
     handler_module_path = Path(inspect.getfile(handler))
     handler_test_path = handler_module_path.relative_to(
         HANDLERS_PACKAGE_PATH

tests/test_processing.py

@@ -1,9 +1,10 @@
 import platform
 import sys
 import zipfile
+from collections.abc import Collection
 from pathlib import Path
 from statistics import mean
-from typing import Collection, List, Optional, Tuple, Type, TypeVar
+from typing import Optional, TypeVar
 
 import attr
 import pytest
@@ -120,7 +121,7 @@ def assert_same_chunks(expected, actual, explanation=None):
     ],
 )
 def test_remove_inner_chunks(
-    chunks: List[ValidChunk], expected: List[ValidChunk], explanation: str
+    chunks: list[ValidChunk], expected: list[ValidChunk], explanation: str
 ):
     assert_same_chunks(expected, remove_inner_chunks(chunks), explanation)
 
@@ -175,7 +176,7 @@ def test_remove_inner_chunks(
     ],
 )
 def test_calculate_unknown_chunks(
-    chunks: List[ValidChunk], file_size: int, expected: List[UnknownChunk]
+    chunks: list[ValidChunk], file_size: int, expected: list[UnknownChunk]
 ):
     assert_same_chunks(expected, calculate_unknown_chunks(chunks, file_size))
 
@@ -222,7 +223,7 @@ def test_format_randomness_plot_error():
         pytest.param([100.0] * 100, "None", id="block_size-can-be-anything3"),
     ],
 )
-def test_format_randomness_plot_no_exception(percentages: List[float], block_size: int):
+def test_format_randomness_plot_no_exception(percentages: list[float], block_size: int):
     assert str(block_size) in format_randomness_plot(
         RandomnessReport(
             shannon=RandomnessMeasurements(
@@ -297,7 +298,7 @@ def fw(tmp_path: Path):
     return base / "fw.zip"
 
 
-def sort_paths(paths: Collection[Path], base: Path) -> Tuple[List[Path], List[Path]]:
+def sort_paths(paths: Collection[Path], base: Path) -> tuple[list[Path], list[Path]]:
     """Sorts paths into two bins, the first one will contain subpaths of base, the second are not.
 
     The first bin will also be converted to relative paths.
@@ -436,7 +437,7 @@ def test_randomness_calculation(tmp_path: Path):
 
     task_result_by_name = {r.task.path.name: r for r in process_result.results}
 
-    def get_all(file_name, report_type: Type[ReportType]) -> List[ReportType]:
+    def get_all(file_name, report_type: type[ReportType]) -> list[ReportType]:
         return task_result_by_name[file_name].filter_reports(report_type)
 
     # ** verification
@@ -505,7 +506,7 @@ def test_skip_extraction(
 
 
 class ConcatenateExtractor(DirectoryExtractor):
-    def extract(self, paths: List[Path], outdir: Path):
+    def extract(self, paths: list[Path], outdir: Path):
         outfile = outdir / "data"
         with outfile.open("wb") as f:
             for path in paths:
@@ -514,7 +515,7 @@ def extract(self, paths: List[Path], outdir: Path):
 
 
 class FailDirExtractor(DirectoryExtractor):
-    def extract(self, paths: List[Path], outdir: Path):
+    def extract(self, paths: list[Path], outdir: Path):
         del paths
         del outdir
         raise ValueError

tests/test_report.py

@@ -1,7 +1,6 @@
 import io
 import json
 from pathlib import Path
-from typing import List
 from unittest.mock import ANY
 from zipfile import ZipFile, ZipInfo
 
@@ -304,7 +303,7 @@ def test_flat_report_structure(hello_kitty: Path, extract_root):
 
 def container_task_results(
     container: Path, extract_root: Path, chunk_id: str
-) -> List[TaskResult]:
+) -> list[TaskResult]:
     """Return expected partial task results for processing the `hello_kitty_container` fixture.
 
     Note, that for some values the `unittest.mock.ANY` is substituted.
@@ -442,11 +441,11 @@ def test_chunk_in_chunk_report_structure(hello_kitty_container: Path, extract_ro
     assert task_results == expected_results
 
 
-def get_normalized_task_results(process_result: ProcessResult) -> List[TaskResult]:
+def get_normalized_task_results(process_result: ProcessResult) -> list[TaskResult]:
     """Normalize away per-run and platform differences."""
     # sort the results - they can potentially have different orders due to multiprocessing
     return sorted(process_result.results, key=lambda tr: (tr.task.depth, tr.task.path))
 
 
-def get_chunk_ids(task_result) -> List[str]:
+def get_chunk_ids(task_result) -> list[str]:
     return [chunk_report.id for chunk_report in task_result.filter_reports(ChunkReport)]