From 00d91d2d6f494f1b8aa10cdaa2cbad0342f2b041 Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Sat, 13 Jul 2024 15:48:25 -0700 Subject: [PATCH 01/16] Rework upstream_utils scripts --- .github/workflows/upstream-utils.yml | 55 +++-- upstream_utils/README.md | 73 ++----- upstream_utils/{update_eigen.py => eigen.py} | 36 ++-- .../{update_expected.py => expected.py} | 24 ++- upstream_utils/{update_fmt.py => fmt.py} | 21 +- upstream_utils/{update_gcem.py => gcem.py} | 27 ++- upstream_utils/{update_json.py => json.py} | 43 ++-- upstream_utils/{update_libuv.py => libuv.py} | 43 ++-- upstream_utils/{update_llvm.py => llvm.py} | 32 +-- .../{update_memory.py => memory.py} | 17 +- upstream_utils/{update_mpack.py => mpack.py} | 34 +-- .../{update_protobuf.py => protobuf.py} | 50 ++--- .../{update_sleipnir.py => sleipnir.py} | 42 ++-- ...update_stack_walker.py => stack_walker.py} | 54 ++--- upstream_utils/upstream_utils.py | 193 ++++++++++++++++++ 15 files changed, 473 insertions(+), 271 deletions(-) rename upstream_utils/{update_eigen.py => eigen.py} (86%) rename upstream_utils/{update_expected.py => expected.py} (69%) rename upstream_utils/{update_fmt.py => fmt.py} (80%) rename upstream_utils/{update_gcem.py => gcem.py} (76%) rename upstream_utils/{update_json.py => json.py} (78%) rename upstream_utils/{update_libuv.py => libuv.py} (86%) rename upstream_utils/{update_llvm.py => llvm.py} (96%) rename upstream_utils/{update_memory.py => memory.py} (91%) rename upstream_utils/{update_mpack.py => mpack.py} (80%) rename upstream_utils/{update_protobuf.py => protobuf.py} (97%) rename upstream_utils/{update_sleipnir.py => sleipnir.py} (76%) rename upstream_utils/{update_stack_walker.py => stack_walker.py} (64%) diff --git a/.github/workflows/upstream-utils.yml b/.github/workflows/upstream-utils.yml index 102aaf621bc..e9c04a45823 100644 --- a/.github/workflows/upstream-utils.yml +++ b/.github/workflows/upstream-utils.yml @@ -30,50 +30,61 @@ jobs: run: | git config --global user.email "you@example.com" git config --global user.name "Your Name" - - name: Run update_eigen.py + - name: Run eigen.py run: | cd upstream_utils - ./update_eigen.py - - name: Run update_fmt.py + ./eigen.py clone + ./eigen.py copy-upstream-to-thirdparty + - name: Run fmt.py run: | cd upstream_utils - ./update_fmt.py - - name: Run update_gcem.py + ./fmt.py clone + ./fmt.py copy-upstream-to-thirdparty + - name: Run gcem.py run: | cd upstream_utils - ./update_gcem.py - - name: Run update_json.py + ./gcem.py clone + ./gcem.py copy-upstream-to-thirdparty + - name: Run json.py run: | cd upstream_utils - ./update_json.py - - name: Run update_libuv.py + ./json.py clone + ./json.py copy-upstream-to-thirdparty + - name: Run libuv.py run: | cd upstream_utils - ./update_libuv.py - - name: Run update_llvm.py + ./libuv.py clone + ./libuv.py copy-upstream-to-thirdparty + - name: Run llvm.py run: | cd upstream_utils - ./update_llvm.py - - name: Run update_mpack.py + ./llvm.py clone + ./llvm.py copy-upstream-to-thirdparty + - name: Run mpack.py run: | cd upstream_utils - ./update_mpack.py - - name: Run update_stack_walker.py + ./mpack.py clone + ./mpack.py copy-upstream-to-thirdparty + - name: Run stack_walker.py run: | cd upstream_utils - ./update_stack_walker.py - - name: Run update_memory.py + ./stack_walker.py clone + ./stack_walker.py copy-upstream-to-thirdparty + - name: Run memory.py run: | cd upstream_utils - ./update_memory.py - - name: Run update_protobuf.py + ./memory.py clone + ./memory.py copy-upstream-to-thirdparty + - name: Run protobuf.py run: | cd upstream_utils - ./update_protobuf.py - - name: Run update_sleipnir.py + ./protobuf.py clone + ./protobuf.py copy-upstream-to-thirdparty + - name: Run sleipnir.py run: | cd upstream_utils - ./update_sleipnir.py + ./sleipnir.py clone + ./sleipnir.py copy-upstream-to-thirdparty - name: Add untracked files to index so they count as changes run: git add -A - name: Check output diff --git a/upstream_utils/README.md b/upstream_utils/README.md index 979dd0d6727..81a33af8cf6 100644 --- a/upstream_utils/README.md +++ b/upstream_utils/README.md @@ -20,59 +20,24 @@ versions. Each library has its own patch directory (e.g., `lib_patches`). The example below will update a hypothetical library called `lib` to the tag `2.0`. -Start in the `upstream_utils` folder. Restore the original repo. +Start in the `upstream_utils` folder. Make sure a clone of the upstream repo exists. ```bash -./update_.py +./.py clone ``` -Navigate to the repo. +Rebase the clone of the upstream repo. ```bash -cd /tmp/lib +./.py rebase 2.0 ``` -Fetch the desired version using one of the following methods. +Update the `upstream_utils` patch files and the tag in the script. ```bash -# Fetch a full branch or tag -git fetch origin 2.0 - -# Fetch just a tag (useful for expensive-to-clone repos) -git fetch --depth 1 origin tag 2.0 -``` - -Rebase any patches onto the new version. If the old version and new version are -on the same branch, run the following. -```bash -git rebase 2.0 +./.py format-patch 2.0 ``` -If the old version and new version are on different branches (e.g., -llvm-project), use interactive rebase instead and remove commits that are common -between the two branches from the list of commits to rebase. In other words, -only commits representing downstream patches should be listed. +Copy the updated upstream files into the thirdparty files within allwpilib. ```bash -git rebase -i 2.0 -``` - -Generate patch files for the new version. -```bash -git format-patch 2.0..HEAD --zero-commit --abbrev=40 --no-signature -``` - -Move the patch files to `upstream_utils`. -``` -mv *.patch allwpilib/upstream_utils/lib_patches -``` - -Navigate back to `upstream_utils`. -```bash -cd allwpilib/upstream_utils -``` - -Modify the version number in the call to `setup_upstream_repo()` in -`update_.py`, then rerun `update_.py` to reimport the thirdparty -files. -```bash -./update_.py +./.py copy-upstream-to-thirdparty ``` ## Adding patch to thirdparty library @@ -80,12 +45,12 @@ files. The example below will add a new patch file to a hypothetical library called `lib` (Replace `` with `llvm`, `fmt`, `eigen`, ... in the following steps). -Start in the `upstream_utils` folder. Restore the original repo. +Start in the `upstream_utils` folder. Make sure a clone of the upstream repo exists. ```bash -./update_.py +./.py clone ``` -Navigate to the repo. +Navigate to the repo. If you can't find it, the directory of the clone is printed at the start of the `clone` command. ```bash cd /tmp/ ``` @@ -96,24 +61,18 @@ git add ... git commit -m "..." ``` -Generate patch files. +Update the `upstream_utils` patch files. ```bash -git format-patch 2.0..HEAD --zero-commit --abbrev=40 --no-signature -``` -where `2.0` is replaced with the version specified in `update_.py`. - -Move the patch files to `upstream_utils`. -``` -mv *.patch allwpilib/upstream_utils/_patches +allwpilib/upstream_utils/.py format-patch 2.0 ``` +where `2.0` is replaced with the version specified in `.py`. Navigate back to `upstream_utils`. ```bash cd allwpilib/upstream_utils ``` -Update the list of patch files in `update_.py`, then rerun -`update_.py` to reimport the thirdparty files. +Update the list of patch files in `.py`, then rerun `.py` to reimport the thirdparty files. ```bash -./update_.py +./.py copy-upstream-to-thirdparty ``` diff --git a/upstream_utils/update_eigen.py b/upstream_utils/eigen.py similarity index 86% rename from upstream_utils/update_eigen.py rename to upstream_utils/eigen.py index fd8cfb4ef60..cc85f65e414 100755 --- a/upstream_utils/update_eigen.py +++ b/upstream_utils/eigen.py @@ -5,11 +5,9 @@ import shutil from upstream_utils import ( - get_repo_root, - clone_repo, comment_out_invalid_includes, walk_cwd_and_copy_if, - git_am, + Lib, ) @@ -94,24 +92,9 @@ def unsupported_inclusions(dp, f): return "MatrixFunctions" in abspath -def main(): - upstream_root = clone_repo( - "https://gitlab.com/libeigen/eigen.git", - # master on 2024-05-22 - "c4d84dfddc9f9edef0fdbe7cf9966d2f4a303198", - shallow=False, - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpimath = os.path.join(wpilib_root, "wpimath") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Disable-warnings.patch", - "0002-Intellisense-fix.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/eigen_patches", f)) - # Delete old install for d in ["src/main/native/thirdparty/eigen/include"]: shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True) @@ -138,10 +121,23 @@ def main(): ) shutil.copyfile( - os.path.join(upstream_root, ".clang-format"), + ".clang-format", os.path.join(wpimath, "src/main/native/thirdparty/eigen/include/.clang-format"), ) +def main(): + name = "eigen" + url = "https://gitlab.com/libeigen/eigen.git" + tag = "c4d84dfddc9f9edef0fdbe7cf9966d2f4a303198" + patch_list = [ + "0001-Disable-warnings.patch", + "0002-Intellisense-fix.patch", + ] + + eigen = Lib(name, url, tag, patch_list, copy_upstream_src) + eigen.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_expected.py b/upstream_utils/expected.py similarity index 69% rename from upstream_utils/update_expected.py rename to upstream_utils/expected.py index 1b459fe5ac3..08569e3561b 100755 --- a/upstream_utils/update_expected.py +++ b/upstream_utils/expected.py @@ -10,26 +10,18 @@ comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo( - "https://github.com/TartanLlama/expected", - # master on 2024-01-25 - "3f0ca7b19253129700a073abfa6d8638d9f7c80c", - shallow=False, - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") # Copy expected header into allwpilib dest_filename = os.path.join( wpiutil, "src/main/native/thirdparty/expected/include/wpi/expected" ) - shutil.copyfile( - os.path.join(upstream_root, "include/tl/expected.hpp"), dest_filename - ) + shutil.copyfile("include/tl/expected.hpp", dest_filename) # Rename namespace from tl to wpi with open(dest_filename) as f: @@ -41,5 +33,15 @@ def main(): f.write(content) +def main(): + name = "expected" + url = "https://github.com/TartanLlama/expected" + # master on 2024-01-25 + tag = "3f0ca7b19253129700a073abfa6d8638d9f7c80c" + + expected = Lib(name, url, tag, [], copy_upstream_src) + expected.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_fmt.py b/upstream_utils/fmt.py similarity index 80% rename from upstream_utils/update_fmt.py rename to upstream_utils/fmt.py index 94aef193f71..8f03859f54c 100755 --- a/upstream_utils/update_fmt.py +++ b/upstream_utils/fmt.py @@ -9,19 +9,13 @@ comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/fmtlib/fmt", "11.0.1") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in ["0001-Suppress-warnings-we-can-t-fix.patch"]: - git_am(os.path.join(wpilib_root, "upstream_utils/fmt_patches", f)) - # Delete old install for d in [ "src/main/native/thirdparty/fmtlib/src", @@ -51,5 +45,16 @@ def main(): ) +def main(): + name = "fmt" + url = "https://github.com/fmtlib/fmt" + tag = "11.0.1" + + patch_list = ["0001-Suppress-warnings-we-can-t-fix.patch"] + + fmt = Lib(name, url, tag, patch_list, copy_upstream_src) + fmt.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_gcem.py b/upstream_utils/gcem.py similarity index 76% rename from upstream_utils/update_gcem.py rename to upstream_utils/gcem.py index 20f62425962..b6fd472a72f 100755 --- a/upstream_utils/update_gcem.py +++ b/upstream_utils/gcem.py @@ -9,22 +9,13 @@ comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/kthohr/gcem.git", "v1.18.0") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpimath = os.path.join(wpilib_root, "wpimath") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Call-std-functions-if-not-constant-evaluated.patch", - "0002-Add-hypot-x-y-z.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/gcem_patches", f)) - # Delete old install for d in [ "src/main/native/thirdparty/gcem/include", @@ -43,5 +34,19 @@ def main(): ) +def main(): + name = "gcem" + url = "https://github.com/kthohr/gcem.git" + tag = "v1.18.0" + + patch_list = [ + "0001-Call-std-functions-if-not-constant-evaluated.patch", + "0002-Add-hypot-x-y-z.patch", + ] + + gcem = Lib(name, url, tag, patch_list, copy_upstream_src) + gcem.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_json.py b/upstream_utils/json.py similarity index 78% rename from upstream_utils/update_json.py rename to upstream_utils/json.py index 9cad98d5fa3..78924ef2a1e 100755 --- a/upstream_utils/update_json.py +++ b/upstream_utils/json.py @@ -8,27 +8,13 @@ clone_repo, walk_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/nlohmann/json", "v3.11.3") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Remove-version-from-namespace.patch", - "0002-Make-serializer-public.patch", - "0003-Make-dump_escaped-take-std-string_view.patch", - "0004-Add-llvm-stream-support.patch", - ]: - git_am( - os.path.join(wpilib_root, "upstream_utils/json_patches", f), - use_threeway=True, - ) - # Delete old install for d in [ "src/main/native/thirdparty/json/include", @@ -36,11 +22,9 @@ def main(): shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) # Create lists of source and destination files - os.chdir(os.path.join(upstream_root, "include/nlohmann")) + os.chdir("include/nlohmann") files = walk_if(".", lambda dp, f: True) - src_include_files = [ - os.path.join(os.path.join(upstream_root, "include/nlohmann"), f) for f in files - ] + src_include_files = [os.path.abspath(f) for f in files] wpiutil_json_root = os.path.join( wpiutil, "src/main/native/thirdparty/json/include/wpi" ) @@ -74,5 +58,24 @@ def main(): f.write(content) +def main(): + name = "json" + url = "https://github.com/nlohmann/json" + tag = "v3.11.3" + + patch_list = [ + "0001-Remove-version-from-namespace.patch", + "0002-Make-serializer-public.patch", + "0003-Make-dump_escaped-take-std-string_view.patch", + "0004-Add-llvm-stream-support.patch", + ] + patch_options = { + "use_threeway": True, + } + + json = Lib(name, url, tag, patch_list, copy_upstream_src, patch_options) + json.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_libuv.py b/upstream_utils/libuv.py similarity index 86% rename from upstream_utils/update_libuv.py rename to upstream_utils/libuv.py index 0290c4d73a8..9a2865838cf 100755 --- a/upstream_utils/update_libuv.py +++ b/upstream_utils/libuv.py @@ -9,30 +9,13 @@ comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/libuv/libuv", "v1.48.0") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpinet = os.path.join(wpilib_root, "wpinet") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Revert-win-process-write-minidumps-when-sending-SIGQ.patch", - "0002-Fix-missing-casts.patch", - "0003-Fix-warnings.patch", - "0004-Preprocessor-cleanup.patch", - "0005-Cleanup-problematic-language.patch", - "0006-Fix-Win32-warning-suppression-pragma.patch", - "0007-Use-C-atomics.patch", - "0008-Remove-static-from-array-indices.patch", - "0009-Add-pragmas-for-missing-libraries-and-set-_WIN32_WIN.patch", - "0010-Remove-swearing.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/libuv_patches", f)) - # Delete old install for d in ["src/main/native/thirdparty/libuv"]: shutil.rmtree(os.path.join(wpinet, d), ignore_errors=True) @@ -71,5 +54,27 @@ def main(): ) +def main(): + name = "libuv" + url = "https://github.com/libuv/libuv" + tag = "v1.48.0" + + patch_list = [ + "0001-Revert-win-process-write-minidumps-when-sending-SIGQ.patch", + "0002-Fix-missing-casts.patch", + "0003-Fix-warnings.patch", + "0004-Preprocessor-cleanup.patch", + "0005-Cleanup-problematic-language.patch", + "0006-Fix-Win32-warning-suppression-pragma.patch", + "0007-Use-C-atomics.patch", + "0008-Remove-static-from-array-indices.patch", + "0009-Add-pragmas-for-missing-libraries-and-set-_WIN32_WIN.patch", + "0010-Remove-swearing.patch", + ] + + libuv = Lib(name, url, tag, patch_list, copy_upstream_src) + libuv.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_llvm.py b/upstream_utils/llvm.py similarity index 96% rename from upstream_utils/update_llvm.py rename to upstream_utils/llvm.py index 9180442a631..394d3cb7a5d 100755 --- a/upstream_utils/update_llvm.py +++ b/upstream_utils/llvm.py @@ -9,6 +9,7 @@ comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) @@ -170,14 +171,20 @@ def overwrite_tests(wpiutil_root, llvm_root): run_global_replacements(wpi_files) -def main(): - upstream_root = clone_repo("https://github.com/llvm/llvm-project", "llvmorg-18.1.8") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): + upstream_root = os.path.abspath(".") wpiutil = os.path.join(wpilib_root, "wpiutil") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ + overwrite_source(wpiutil, upstream_root) + overwrite_tests(wpiutil, upstream_root) + + +def main(): + name = "llvm" + url = "https://github.com/llvm/llvm-project" + tag = "llvmorg-18.1.8" + + patch_list = [ "0001-Remove-StringRef-ArrayRef-and-Optional.patch", "0002-Wrap-std-min-max-calls-in-parens-for-Windows-warning.patch", "0003-Change-unique_function-storage-size.patch", @@ -214,14 +221,13 @@ def main(): "0034-Add-back-removed-raw_string_ostream-write_impl.patch", "0035-Remove-auto-conversion-from-raw_ostream.patch", "0036-Add-SmallVector-erase_if.patch", - ]: - git_am( - os.path.join(wpilib_root, "upstream_utils/llvm_patches", f), - use_threeway=True, - ) + ] + patch_options = { + "use_threeway": True, + } - overwrite_source(wpiutil, upstream_root) - overwrite_tests(wpiutil, upstream_root) + llvm = Lib(name, url, tag, patch_list, copy_upstream_src, patch_options) + llvm.main() if __name__ == "__main__": diff --git a/upstream_utils/update_memory.py b/upstream_utils/memory.py similarity index 91% rename from upstream_utils/update_memory.py rename to upstream_utils/memory.py index 6645aaf60cf..4a618584220 100755 --- a/upstream_utils/update_memory.py +++ b/upstream_utils/memory.py @@ -9,6 +9,7 @@ comment_out_invalid_includes, walk_if, copy_to, + Lib, ) @@ -58,9 +59,7 @@ def run_global_replacements(memory_files): f.write(content) -def main(): - upstream_root = clone_repo("https://github.com/foonathan/memory", "v0.7-3") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") # Delete old install @@ -71,7 +70,6 @@ def main(): shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) # Copy sources - os.chdir(upstream_root) src_files = walk_if("src", lambda dp, f: f.endswith(".cpp") or f.endswith(".hpp")) src_files = copy_to( src_files, os.path.join(wpiutil, "src/main/native/thirdparty/memory") @@ -80,7 +78,7 @@ def main(): run_source_replacements(src_files) # Copy headers - os.chdir(os.path.join(upstream_root, "include", "foonathan")) + os.chdir(os.path.join("include", "foonathan")) include_files = walk_if(".", lambda dp, f: f.endswith(".hpp")) include_files = copy_to( include_files, @@ -100,5 +98,14 @@ def main(): ) +def main(): + name = "memory" + url = "https://github.com/foonathan/memory" + tag = "v0.7-3" + + memory = Lib(name, url, tag, [], copy_upstream_src) + memory.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_mpack.py b/upstream_utils/mpack.py similarity index 80% rename from upstream_utils/update_mpack.py rename to upstream_utils/mpack.py index 68315e933a9..678b189d476 100755 --- a/upstream_utils/update_mpack.py +++ b/upstream_utils/mpack.py @@ -9,12 +9,11 @@ clone_repo, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo("https://github.com/ludocode/mpack", "v1.1.1") - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") # Delete old install @@ -24,19 +23,6 @@ def main(): ]: shutil.rmtree(os.path.join(wpiutil, d), ignore_errors=True) - # Apply patches to upstream Git repo - os.chdir(upstream_root) - - for f in [ - "0001-Don-t-emit-inline-defs.patch", - "0002-Update-amalgamation-script.patch", - "0003-Use-namespace-for-C.patch", - "0004-Group-doxygen-into-MPack-module.patch", - ]: - git_am( - os.path.join(wpilib_root, "upstream_utils/mpack_patches", f), - ) - # Run the amalgmation script subprocess.check_call(["bash", "tools/amalgamate.sh"]) @@ -56,5 +42,21 @@ def main(): ) +def main(): + name = "mpack" + url = "https://github.com/ludocode/mpack" + tag = "v1.1.1" + + patch_list = [ + "0001-Don-t-emit-inline-defs.patch", + "0002-Update-amalgamation-script.patch", + "0003-Use-namespace-for-C.patch", + "0004-Group-doxygen-into-MPack-module.patch", + ] + + mpack = Lib(name, url, tag, [], copy_upstream_src) + mpack.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_protobuf.py b/upstream_utils/protobuf.py similarity index 97% rename from upstream_utils/update_protobuf.py rename to upstream_utils/protobuf.py index 1f004cd0c42..8b2adf5347b 100755 --- a/upstream_utils/update_protobuf.py +++ b/upstream_utils/protobuf.py @@ -11,6 +11,7 @@ walk_cwd_and_copy_if, walk_if, git_am, + Lib, ) protobuf_lite_sources = set( @@ -255,31 +256,10 @@ def matches(dp, f, files): return p in files -def main(): - upstream_root = clone_repo( - "https://github.com/protocolbuffers/protobuf", "v3.21.12" - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): + upstream_root = os.path.abspath(".") wpiutil = os.path.join(wpilib_root, "wpiutil") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Fix-sign-compare-warnings.patch", - "0002-Remove-redundant-move.patch", - "0003-Fix-maybe-uninitialized-warnings.patch", - "0004-Fix-coded_stream-WriteRaw.patch", - "0005-Suppress-enum-enum-conversion-warning.patch", - "0006-Fix-noreturn-function-returning.patch", - "0007-Work-around-GCC-12-restrict-warning-compiler-bug.patch", - "0008-Disable-MSVC-switch-warning.patch", - "0009-Disable-unused-function-warning.patch", - "0010-Disable-pedantic-warning.patch", - "0011-Avoid-use-of-sprintf.patch", - "0012-Suppress-stringop-overflow-warning-false-positives.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/protobuf_patches", f)) - # Delete old install for d in [ "src/main/native/thirdparty/protobuf/src", @@ -304,5 +284,29 @@ def main(): ) +def main(): + name = "protobuf" + url = "https://github.com/protocolbuffers/protobuf" + tag = "v3.21.12" + + patch_list = [ + "0001-Fix-sign-compare-warnings.patch", + "0002-Remove-redundant-move.patch", + "0003-Fix-maybe-uninitialized-warnings.patch", + "0004-Fix-coded_stream-WriteRaw.patch", + "0005-Suppress-enum-enum-conversion-warning.patch", + "0006-Fix-noreturn-function-returning.patch", + "0007-Work-around-GCC-12-restrict-warning-compiler-bug.patch", + "0008-Disable-MSVC-switch-warning.patch", + "0009-Disable-unused-function-warning.patch", + "0010-Disable-pedantic-warning.patch", + "0011-Avoid-use-of-sprintf.patch", + "0012-Suppress-stringop-overflow-warning-false-positives.patch", + ] + + protobuf = Lib(name, url, tag, patch_list, copy_upstream_src) + protobuf.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_sleipnir.py b/upstream_utils/sleipnir.py similarity index 76% rename from upstream_utils/update_sleipnir.py rename to upstream_utils/sleipnir.py index 4b783c82bab..11b20cc29a8 100755 --- a/upstream_utils/update_sleipnir.py +++ b/upstream_utils/sleipnir.py @@ -9,30 +9,13 @@ copy_to, walk_cwd_and_copy_if, git_am, + Lib, ) -def main(): - upstream_root = clone_repo( - "https://github.com/SleipnirGroup/Sleipnir", - # main on 2024-07-09 - "b6ffa2d4fdb99cab1bf79491f715a6a9a86633b5", - shallow=False, - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpimath = os.path.join(wpilib_root, "wpimath") - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Remove-using-enum-declarations.patch", - "0002-Use-fmtlib.patch", - "0003-Remove-unsupported-constexpr.patch", - "0004-Use-wpi-SmallVector.patch", - "0005-Suppress-clang-tidy-false-positives.patch", - ]: - git_am(os.path.join(wpilib_root, "upstream_utils/sleipnir_patches", f)) - # Delete old install for d in [ "src/main/native/thirdparty/sleipnir/src", @@ -41,7 +24,6 @@ def main(): shutil.rmtree(os.path.join(wpimath, d), ignore_errors=True) # Copy Sleipnir source files into allwpilib - os.chdir(upstream_root) src_files = [os.path.join(dp, f) for dp, dn, fn in os.walk("src") for f in fn] src_files = copy_to( src_files, os.path.join(wpimath, "src/main/native/thirdparty/sleipnir") @@ -65,10 +47,28 @@ def main(): ".styleguide-license", ]: shutil.copyfile( - os.path.join(upstream_root, filename), + filename, os.path.join(wpimath, "src/main/native/thirdparty/sleipnir", filename), ) +def main(): + name = "sleipnir" + url = "https://github.com/SleipnirGroup/Sleipnir" + # main on 2024-07-09 + tag = "b6ffa2d4fdb99cab1bf79491f715a6a9a86633b5" + + patch_list = [ + "0001-Remove-using-enum-declarations.patch", + "0002-Use-fmtlib.patch", + "0003-Remove-unsupported-constexpr.patch", + "0004-Use-wpi-SmallVector.patch", + "0005-Suppress-clang-tidy-false-positives.patch", + ] + + sleipnir = Lib(name, url, tag, patch_list, copy_upstream_src) + sleipnir.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/update_stack_walker.py b/upstream_utils/stack_walker.py similarity index 64% rename from upstream_utils/update_stack_walker.py rename to upstream_utils/stack_walker.py index a58886ec799..190f12d04d9 100755 --- a/upstream_utils/update_stack_walker.py +++ b/upstream_utils/stack_walker.py @@ -10,11 +10,12 @@ comment_out_invalid_includes, walk_cwd_and_copy_if, git_am, + Lib, ) -def crlf_to_lf(stackwalker_dir): - for root, _, files in os.walk(stackwalker_dir): +def crlf_to_lf(): + for root, _, files in os.walk("."): if ".git" in root: continue @@ -28,35 +29,13 @@ def crlf_to_lf(stackwalker_dir): with open(filename, "wb") as f: f.write(content) - cwd = os.getcwd() - os.chdir(stackwalker_dir) subprocess.check_call(["git", "add", "-A"]) subprocess.check_call(["git", "commit", "-m", "Fix line endings"]) - os.chdir(cwd) -def main(): - upstream_root = clone_repo( - "https://github.com/JochenKalmbach/StackWalker", - "5b0df7a4db8896f6b6dc45d36e383c52577e3c6b", - shallow=False, - ) - wpilib_root = get_repo_root() +def copy_upstream_src(wpilib_root): wpiutil = os.path.join(wpilib_root, "wpiutil") - # Run CRLF -> LF before trying any patches - crlf_to_lf(upstream_root) - - # Apply patches to upstream Git repo - os.chdir(upstream_root) - for f in [ - "0001-Add-advapi-pragma.patch", - ]: - git_am( - os.path.join(wpilib_root, "upstream_utils/stack_walker_patches", f), - ignore_whitespace=True, - ) - shutil.copy( os.path.join("Main", "StackWalker", "StackWalker.h"), os.path.join(wpiutil, "src/main/native/windows/StackWalker.h"), @@ -68,5 +47,30 @@ def main(): ) +def main(): + name = "stack_walker" + url = "https://github.com/JochenKalmbach/StackWalker" + tag = "5b0df7a4db8896f6b6dc45d36e383c52577e3c6b" + + patch_list = [ + "0001-Add-advapi-pragma.patch", + ] + patch_options = { + "ignore_whitespace": True, + } + + stack_walker = Lib( + name, + url, + tag, + patch_list, + copy_upstream_src, + patch_options, + pre_patch_hook=crlf_to_lf, + pre_patch_commits=1, + ) + stack_walker.main() + + if __name__ == "__main__": main() diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index ccf668b738f..19af3b86ff5 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -1,7 +1,9 @@ +import argparse import os import re import shutil import subprocess +import sys import tempfile @@ -198,3 +200,194 @@ def git_am(patch, use_threeway=False, ignore_whitespace=False): args.append("--ignore-whitespace") subprocess.check_output(args + [patch]) + + +def has_git_rev(rev): + cmd = ["git", "rev-parse", "--verify", "-q", rev] + return subprocess.run(cmd, stdout=subprocess.DEVNULL).returncode == 0 + + +class Lib: + def __init__( + self, + name, + url, + old_tag, + patch_list, + copy_upstream_src, + patch_options={}, + *, + pre_patch_hook=None, + pre_patch_commits=0, + ): + self.name = name + self.url = url + self.old_tag = old_tag + self.patch_list = patch_list + self.copy_upstream_src = copy_upstream_src + self.patch_options = patch_options + self.pre_patch_hook = pre_patch_hook + self.pre_patch_commits = pre_patch_commits + self.wpilib_root = get_repo_root() + + def open_repo(self, *, err_msg_if_absent): + os.chdir(tempfile.gettempdir()) + + repo = os.path.basename(self.url) + dest = os.path.join(os.getcwd(), repo).removesuffix(".git") + + print(f"INFO: Opening repository at {dest}") + + if not os.path.exists(dest): + if err_msg_if_absent is None: + subprocess.run(["git", "clone", "--filter=tree:0", self.url]) + else: + print(err_msg_if_absent, file=sys.stderr) + exit(1) + os.chdir(dest) + + def replace_tag(self, tag): + path = os.path.join(self.wpilib_root, f"upstream_utils/{self.name}.py") + with open(path, "r") as file: + lines = file.readlines() + + previous_text = f"tag = '{self.old_tag}'" + new_text = f"tag = '{tag}'" + for i in range(len(lines)): + lines[i] = lines[i].replace(previous_text, new_text) + + with open(path, "w") as file: + file.writelines(lines) + + def clone(self): + self.open_repo(err_msg_if_absent=None) + + subprocess.run(["git", "switch", "--detach", self.old_tag]) + + def rebase(self, new_tag): + self.open_repo( + err_msg_if_absent='There\'s nothing to rebase. Run the "clone" command first.' + ) + + subprocess.run(["git", "fetch", "origin", new_tag]) + + subprocess.run(["git", "switch", "--detach", self.old_tag]) + + if self.pre_patch_hook is not None: + self.pre_patch_hook() + + for f in self.patch_list: + git_am( + os.path.join( + self.wpilib_root, f"upstream_utils/{self.name}_patches", f + ), + **self.patch_options, + ) + + subprocess.run(["git", "rebase", "--onto", new_tag, self.old_tag]) + + # Detect merge conflict by detecting if we stopped in the middle of a rebase + if has_git_rev("REBASE_HEAD"): + print( + f"Merge conflicts when rebasing onto {new_tag}! You must manually resolve them.", + file=sys.stderr, + ) + + def format_patch(self, tag): + self.open_repo( + err_msg_if_absent='There\'s nothing to run format-patch on. Run the "clone" and "rebase" commands first.' + ) + + if self.pre_patch_commits > 0: + commits_since_tag_output = subprocess.run( + ["git", "log", "--format=format:%h", f"{tag}..HEAD"], + capture_output=True, + ).stdout + commits_since_tag = commits_since_tag_output.count(b"\n") + 1 + tag = f"HEAD~{commits_since_tag - self.pre_patch_commits}" + + subprocess.run( + [ + "git", + "format-patch", + f"{tag}..HEAD", + "--abbrev=40", + "--zero-commit", + "--no-signature", + ] + ) + + patch_dest = os.path.join( + self.wpilib_root, f"upstream_utils/{self.name}_patches" + ) + + if not os.path.exists(patch_dest): + print( + f"WARNING: Patch directory {patch_dest} does not exist", file=sys.stderr + ) + + for f in os.listdir(): + if f.endswith(".patch"): + os.rename(f, os.path.join(patch_dest, f)) + + self.replace_tag(tag) + + def copy_upstream_to_thirdparty(self): + self.open_repo( + err_msg_if_absent='There\'s no repository to copy from. Run the "clone" command first.' + ) + + subprocess.run(["git", "switch", "--detach", self.old_tag]) + + if self.pre_patch_hook is not None: + self.pre_patch_hook() + + for f in self.patch_list: + git_am( + os.path.join( + self.wpilib_root, f"upstream_utils/{self.name}_patches", f + ), + **self.patch_options, + ) + + self.copy_upstream_src(self.wpilib_root) + + def main(self, argv=sys.argv[1:]): + parser = argparse.ArgumentParser( + description=f"CLI manager of the {self.name} upstream library" + ) + subparsers = parser.add_subparsers(dest="subcommand", required=True) + + subparsers.add_parser( + "clone", help="Clones the upstream repository in a local tempdir" + ) + + parser_rebase = subparsers.add_parser( + "rebase", help="Rebases the clone of the upstream repository" + ) + parser_rebase.add_argument("new_tag", help="The tag to rebase onto") + + parser_format_patch = subparsers.add_parser( + "format-patch", + help="Generates patch files for the upstream repository and moves them into the upstream_utils patch directory", + ) + parser_format_patch.add_argument( + "new_tag", help="The tag for the commit before the patches" + ) + + subparsers.add_parser( + "copy-upstream-to-thirdparty", + help="Copies files from the upstream repository into the thirdparty directory in allwpilib", + ) + + args = parser.parse_args(argv) + + self.wpilib_root = get_repo_root() + if args.subcommand == "clone": + self.clone() + elif args.subcommand == "rebase": + self.rebase(args.new_tag) + elif args.subcommand == "format-patch": + self.format_patch(args.new_tag) + elif args.subcommand == "copy-upstream-to-thirdparty": + self.copy_upstream_to_thirdparty() From a1def4473bbb5090a37368e683e2ca3c7c1ee02b Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Sat, 13 Jul 2024 15:53:44 -0700 Subject: [PATCH 02/16] Convert single quote strings into double quote strings --- upstream_utils/upstream_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index 19af3b86ff5..bad31bf7823 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -251,8 +251,8 @@ def replace_tag(self, tag): with open(path, "r") as file: lines = file.readlines() - previous_text = f"tag = '{self.old_tag}'" - new_text = f"tag = '{tag}'" + previous_text = f"tag = \"{self.old_tag}\"" + new_text = f"tag = \"{tag}\"" for i in range(len(lines)): lines[i] = lines[i].replace(previous_text, new_text) @@ -266,7 +266,7 @@ def clone(self): def rebase(self, new_tag): self.open_repo( - err_msg_if_absent='There\'s nothing to rebase. Run the "clone" command first.' + err_msg_if_absent="There's nothing to rebase. Run the \"clone\" command first." ) subprocess.run(["git", "fetch", "origin", new_tag]) @@ -295,7 +295,7 @@ def rebase(self, new_tag): def format_patch(self, tag): self.open_repo( - err_msg_if_absent='There\'s nothing to run format-patch on. Run the "clone" and "rebase" commands first.' + err_msg_if_absent="There's nothing to run format-patch on. Run the \"clone\" and \"rebase\" commands first." ) if self.pre_patch_commits > 0: @@ -334,7 +334,7 @@ def format_patch(self, tag): def copy_upstream_to_thirdparty(self): self.open_repo( - err_msg_if_absent='There\'s no repository to copy from. Run the "clone" command first.' + err_msg_if_absent="There's no repository to copy from. Run the \"clone\" command first." ) subprocess.run(["git", "switch", "--detach", self.old_tag]) From a5bf968010e22cdcad63431fffe6de64cd27f5f7 Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Sat, 13 Jul 2024 16:04:54 -0700 Subject: [PATCH 03/16] Fix upstream_utils README --- upstream_utils/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/upstream_utils/README.md b/upstream_utils/README.md index 81a33af8cf6..859306f7a34 100644 --- a/upstream_utils/README.md +++ b/upstream_utils/README.md @@ -50,6 +50,12 @@ Start in the `upstream_utils` folder. Make sure a clone of the upstream repo exi ./.py clone ``` +Update the clone of the upstream repo. +```bash +./.py rebase 2.0 +``` +where `2.0` is replaced with the version specified in `.py`. + Navigate to the repo. If you can't find it, the directory of the clone is printed at the start of the `clone` command. ```bash cd /tmp/ From 47f7a75b8e98c0803866fecd2c17e1df7a2ad2b1 Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Sat, 13 Jul 2024 16:12:21 -0700 Subject: [PATCH 04/16] Revert changes to single quote strings Turns out that is how wpiformat wants it --- upstream_utils/upstream_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index bad31bf7823..9433d1f1f04 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -251,8 +251,8 @@ def replace_tag(self, tag): with open(path, "r") as file: lines = file.readlines() - previous_text = f"tag = \"{self.old_tag}\"" - new_text = f"tag = \"{tag}\"" + previous_text = f'tag = "{self.old_tag}"' + new_text = f'tag = "{tag}"' for i in range(len(lines)): lines[i] = lines[i].replace(previous_text, new_text) @@ -266,7 +266,7 @@ def clone(self): def rebase(self, new_tag): self.open_repo( - err_msg_if_absent="There's nothing to rebase. Run the \"clone\" command first." + err_msg_if_absent='There\'s nothing to rebase. Run the "clone" command first.' ) subprocess.run(["git", "fetch", "origin", new_tag]) @@ -295,7 +295,7 @@ def rebase(self, new_tag): def format_patch(self, tag): self.open_repo( - err_msg_if_absent="There's nothing to run format-patch on. Run the \"clone\" and \"rebase\" commands first." + err_msg_if_absent='There\'s nothing to run format-patch on. Run the "clone" and "rebase" commands first.' ) if self.pre_patch_commits > 0: @@ -334,7 +334,7 @@ def format_patch(self, tag): def copy_upstream_to_thirdparty(self): self.open_repo( - err_msg_if_absent="There's no repository to copy from. Run the \"clone\" command first." + err_msg_if_absent='There\'s no repository to copy from. Run the "clone" command first.' ) subprocess.run(["git", "switch", "--detach", self.old_tag]) From d25c6536401337a91a17685ee5965ac5ea66122e Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Sun, 14 Jul 2024 17:38:51 -0700 Subject: [PATCH 05/16] Add some conveniences --- upstream_utils/upstream_utils.py | 130 ++++++++++++++++++++++++------- 1 file changed, 103 insertions(+), 27 deletions(-) diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index 9433d1f1f04..cd3c7f47eea 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -230,11 +230,47 @@ def __init__( self.pre_patch_commits = pre_patch_commits self.wpilib_root = get_repo_root() - def open_repo(self, *, err_msg_if_absent): - os.chdir(tempfile.gettempdir()) + def check_patches(self): + patch_directory_patches = set() + patch_directory = os.path.join( + self.wpilib_root, f"upstream_utils/{self.name}_patches" + ) + if os.path.exists(patch_directory): + for f in os.listdir(patch_directory): + if f.endswith(".patch"): + patch_directory_patches.add(f) + patches = set(self.patch_list) + patch_directory_only = sorted(patch_directory_patches - patches) + patch_list_only = sorted(patches - patch_directory_patches) + common_patches = sorted(patch_directory_patches & patches) + warning = False + if patch_directory_only: + print( + f"WARNING: The patch directory has patches {patch_directory_only} not in the patch list" + ) + warning = True + if patch_list_only: + print( + f"WARNING: The patch list has patches {patch_list_only} not in the patch directory" + ) + warning = True + if warning and common_patches: + print( + f" Note: The patch directory and the patch list both have patches {common_patches}" + ) + def get_repo_path(self, tempdir=None): + if tempdir is None: + tempdir = tempfile.gettempdir() repo = os.path.basename(self.url) - dest = os.path.join(os.getcwd(), repo).removesuffix(".git") + dest = os.path.join(tempdir, repo) + dest = dest.removesuffix(".git") + return dest + + def open_repo(self, *, err_msg_if_absent): + os.chdir(tempfile.gettempdir() + + dest = self.get_repo_path(os.getcwd()) print(f"INFO: Opening repository at {dest}") @@ -246,6 +282,18 @@ def open_repo(self, *, err_msg_if_absent): exit(1) os.chdir(dest) + def apply_patches(self): + if self.pre_patch_hook is not None: + self.pre_patch_hook() + + for f in self.patch_list: + git_am( + os.path.join( + self.wpilib_root, f"upstream_utils/{self.name}_patches", f + ), + **self.patch_options, + ) + def replace_tag(self, tag): path = os.path.join(self.wpilib_root, f"upstream_utils/{self.name}.py") with open(path, "r") as file: @@ -254,16 +302,41 @@ def replace_tag(self, tag): previous_text = f'tag = "{self.old_tag}"' new_text = f'tag = "{tag}"' for i in range(len(lines)): - lines[i] = lines[i].replace(previous_text, new_text) + if previous_text in lines[i]: + if i - 1 >= 0 and "#" in lines[i - 1]: + print( + f"WARNING: Automatically updating tag in line {i + 1} with a comment above it that may need updating.", + file=sys.stderr, + ) + lines[i] = lines[i].replace(previous_text, new_text) with open(path, "w") as file: file.writelines(lines) + def info(self): + print(f"Repository name: {self.name}") + print(f"Upstream URL: {self.url}") + print(f"Upstream tag: {self.old_tag}") + print(f"Path to upstream clone: {self.get_repo_path()}") + print(f"Patches to apply: {self.patch_list}") + print(f"Patch options: {self.patch_options}") + print(f"Pre patch commits: {self.pre_patch_commits}") + print(f"WPILib root: {self.wpilib_root}") + def clone(self): self.open_repo(err_msg_if_absent=None) subprocess.run(["git", "switch", "--detach", self.old_tag]) + def reset(self): + self.open_repo( + err_msg_if_absent='There\'s nothing to reset. Run the "clone" command first.' + ) + + subprocess.run(["git", "switch", "--detach", self.old_tag]) + + self.apply_patches() + def rebase(self, new_tag): self.open_repo( err_msg_if_absent='There\'s nothing to rebase. Run the "clone" command first.' @@ -273,16 +346,7 @@ def rebase(self, new_tag): subprocess.run(["git", "switch", "--detach", self.old_tag]) - if self.pre_patch_hook is not None: - self.pre_patch_hook() - - for f in self.patch_list: - git_am( - os.path.join( - self.wpilib_root, f"upstream_utils/{self.name}_patches", f - ), - **self.patch_options, - ) + self.apply_patches() subprocess.run(["git", "rebase", "--onto", new_tag, self.old_tag]) @@ -293,7 +357,10 @@ def rebase(self, new_tag): file=sys.stderr, ) - def format_patch(self, tag): + def format_patch(self, tag=None): + if tag is None: + tag = self.old_tag + self.open_repo( err_msg_if_absent='There\'s nothing to run format-patch on. Run the "clone" and "rebase" commands first.' ) @@ -339,16 +406,7 @@ def copy_upstream_to_thirdparty(self): subprocess.run(["git", "switch", "--detach", self.old_tag]) - if self.pre_patch_hook is not None: - self.pre_patch_hook() - - for f in self.patch_list: - git_am( - os.path.join( - self.wpilib_root, f"upstream_utils/{self.name}_patches", f - ), - **self.patch_options, - ) + self.apply_patches() self.copy_upstream_src(self.wpilib_root) @@ -358,10 +416,19 @@ def main(self, argv=sys.argv[1:]): ) subparsers = parser.add_subparsers(dest="subcommand", required=True) + subparsers.add_parser( + "info", help="Displays information about the upstream library" + ) + subparsers.add_parser( "clone", help="Clones the upstream repository in a local tempdir" ) + subparsers.add_parser( + "reset", + help="Resets the clone of the upstream repository to the tag and applies patches", + ) + parser_rebase = subparsers.add_parser( "rebase", help="Rebases the clone of the upstream repository" ) @@ -372,7 +439,10 @@ def main(self, argv=sys.argv[1:]): help="Generates patch files for the upstream repository and moves them into the upstream_utils patch directory", ) parser_format_patch.add_argument( - "new_tag", help="The tag for the commit before the patches" + "new_tag", + nargs="?", + default=self.old_tag, + help="The tag for the commit before the patches", ) subparsers.add_parser( @@ -383,11 +453,17 @@ def main(self, argv=sys.argv[1:]): args = parser.parse_args(argv) self.wpilib_root = get_repo_root() - if args.subcommand == "clone": + if args.subcommand == "info": + self.info() + elif args.subcommand == "clone": self.clone() + elif args.subcommand == "reset": + self.reset() elif args.subcommand == "rebase": self.rebase(args.new_tag) elif args.subcommand == "format-patch": self.format_patch(args.new_tag) elif args.subcommand == "copy-upstream-to-thirdparty": self.copy_upstream_to_thirdparty() + + self.check_patches() From 164366ee646ec910ea41a873e14f614dbabb6746 Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Sun, 14 Jul 2024 18:00:29 -0700 Subject: [PATCH 06/16] Update upstream_utils README --- upstream_utils/README.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/upstream_utils/README.md b/upstream_utils/README.md index 859306f7a34..3fbc3ef9712 100644 --- a/upstream_utils/README.md +++ b/upstream_utils/README.md @@ -52,9 +52,8 @@ Start in the `upstream_utils` folder. Make sure a clone of the upstream repo exi Update the clone of the upstream repo. ```bash -./.py rebase 2.0 +./.py reset ``` -where `2.0` is replaced with the version specified in `.py`. Navigate to the repo. If you can't find it, the directory of the clone is printed at the start of the `clone` command. ```bash @@ -67,15 +66,14 @@ git add ... git commit -m "..." ``` -Update the `upstream_utils` patch files. +Navigate back to `upstream_utils`. ```bash -allwpilib/upstream_utils/.py format-patch 2.0 +cd allwpilib/upstream_utils ``` -where `2.0` is replaced with the version specified in `.py`. -Navigate back to `upstream_utils`. +Update the `upstream_utils` patch files. ```bash -cd allwpilib/upstream_utils +./.py format-patch ``` Update the list of patch files in `.py`, then rerun `.py` to reimport the thirdparty files. From 016ecafa32cd8d962531afa1afda2fdcd405bf86 Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Sun, 14 Jul 2024 17:58:48 -0700 Subject: [PATCH 07/16] Fix bug with autoupdating the stack_walker tag --- upstream_utils/upstream_utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index cd3c7f47eea..54e47dd245b 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -268,7 +268,7 @@ def get_repo_path(self, tempdir=None): return dest def open_repo(self, *, err_msg_if_absent): - os.chdir(tempfile.gettempdir() + os.chdir(tempfile.gettempdir()) dest = self.get_repo_path(os.getcwd()) @@ -365,19 +365,20 @@ def format_patch(self, tag=None): err_msg_if_absent='There\'s nothing to run format-patch on. Run the "clone" and "rebase" commands first.' ) + start_commit = tag if self.pre_patch_commits > 0: commits_since_tag_output = subprocess.run( ["git", "log", "--format=format:%h", f"{tag}..HEAD"], capture_output=True, ).stdout commits_since_tag = commits_since_tag_output.count(b"\n") + 1 - tag = f"HEAD~{commits_since_tag - self.pre_patch_commits}" + start_commit = f"HEAD~{commits_since_tag - self.pre_patch_commits}" subprocess.run( [ "git", "format-patch", - f"{tag}..HEAD", + f"{start_commit}..HEAD", "--abbrev=40", "--zero-commit", "--no-signature", From 71b9f07096b94fce3409672715762af59e1f0d11 Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Sun, 14 Jul 2024 20:12:28 -0700 Subject: [PATCH 08/16] Pass patch_list to mpack constructor --- upstream_utils/mpack.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upstream_utils/mpack.py b/upstream_utils/mpack.py index 678b189d476..ad395377a64 100755 --- a/upstream_utils/mpack.py +++ b/upstream_utils/mpack.py @@ -54,7 +54,7 @@ def main(): "0004-Group-doxygen-into-MPack-module.patch", ] - mpack = Lib(name, url, tag, [], copy_upstream_src) + mpack = Lib(name, url, tag, patch_list, copy_upstream_src) mpack.main() From 53e90d0f4a178139f4f10ff41bbda4bbc944a6ac Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Mon, 15 Jul 2024 19:22:48 -0700 Subject: [PATCH 09/16] Fix format-patch Clear the patch directory first Use shutil.move instead to handle different filesystems for temp dir and allwpilib --- upstream_utils/upstream_utils.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index 54e47dd245b..233daa23446 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -393,10 +393,16 @@ def format_patch(self, tag=None): print( f"WARNING: Patch directory {patch_dest} does not exist", file=sys.stderr ) + else: + shutil.rmtree(patch_dest) + is_first = True for f in os.listdir(): if f.endswith(".patch"): - os.rename(f, os.path.join(patch_dest, f)) + if is_first: + os.mkdir(patch_dest) + is_first = False + shutil.move(f, patch_dest) self.replace_tag(tag) From 2c239248259061f511bbf3dc5285aa73b2157ec0 Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Tue, 16 Jul 2024 12:03:03 -0700 Subject: [PATCH 10/16] Add root tag --- upstream_utils/upstream_utils.py | 52 +++++++++++++++++++++++++++++--- 1 file changed, 47 insertions(+), 5 deletions(-) diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index 233daa23446..28b73aca2de 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -282,6 +282,41 @@ def open_repo(self, *, err_msg_if_absent): exit(1) os.chdir(dest) + def get_root_tags(self): + root_tag_output = subprocess.run( + ["git", "tag", "--list", "upstream_utils_root-*"], + capture_output=True, + text=True, + ).stdout + return root_tag_output.splitlines() + + def get_root_tag(self): + root_tags = self.get_root_tags() + if len(root_tags) == 0: + print( + "ERROR: Could not determine root tag: No tags match 'upstream_utils_root-*'", + file=sys.stderr, + ) + exit(1) + if len(root_tags) > 1: + print( + f"ERROR: Could not determine root tag: Multiple candidates: {root_tags}", + file=sys.stderr, + ) + exit(1) + return root_tags[0] + + def set_root_tag(self, tag): + root_tags = self.get_root_tags() + + if len(root_tags) > 1: + print(f"WARNING: Deleting multiple root tags {root_tags}", file=sys.stderr) + + for root_tag in root_tags: + subprocess.run(["git", "tag", "-d", root_tag]) + + subprocess.run(["git", "tag", f"upstream_utils_root-{tag}", tag]) + def apply_patches(self): if self.pre_patch_hook is not None: self.pre_patch_hook() @@ -328,6 +363,8 @@ def clone(self): subprocess.run(["git", "switch", "--detach", self.old_tag]) + self.set_root_tag(self.old_tag) + def reset(self): self.open_repo( err_msg_if_absent='There\'s nothing to reset. Run the "clone" command first.' @@ -348,6 +385,8 @@ def rebase(self, new_tag): self.apply_patches() + self.set_root_tag(new_tag) + subprocess.run(["git", "rebase", "--onto", new_tag, self.old_tag]) # Detect merge conflict by detecting if we stopped in the middle of a rebase @@ -358,13 +397,16 @@ def rebase(self, new_tag): ) def format_patch(self, tag=None): - if tag is None: - tag = self.old_tag - self.open_repo( err_msg_if_absent='There\'s nothing to run format-patch on. Run the "clone" and "rebase" commands first.' ) + if tag is None: + tag = self.get_root_tag() + script_tag = tag.removeprefix("upstream_utils_root-") + else: + script_tag = tag + start_commit = tag if self.pre_patch_commits > 0: commits_since_tag_output = subprocess.run( @@ -404,7 +446,7 @@ def format_patch(self, tag=None): is_first = False shutil.move(f, patch_dest) - self.replace_tag(tag) + self.replace_tag(script_tag) def copy_upstream_to_thirdparty(self): self.open_repo( @@ -448,7 +490,7 @@ def main(self, argv=sys.argv[1:]): parser_format_patch.add_argument( "new_tag", nargs="?", - default=self.old_tag, + default=None, help="The tag for the commit before the patches", ) From 1e17563ec17f36170124d7d0e0eafb5a3c520169 Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Tue, 16 Jul 2024 13:27:39 -0700 Subject: [PATCH 11/16] Set the root tag in reset() as well --- upstream_utils/upstream_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index 28b73aca2de..560fbe048a6 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -372,6 +372,8 @@ def reset(self): subprocess.run(["git", "switch", "--detach", self.old_tag]) + self.set_root_tag(self.old_tag) + self.apply_patches() def rebase(self, new_tag): From 7bf00ddcce169fd09b4a8e3f2add7f13924dc00d Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Tue, 16 Jul 2024 13:43:53 -0700 Subject: [PATCH 12/16] Add docstrings --- upstream_utils/upstream_utils.py | 80 +++++++++++++++++++++++++++++++- 1 file changed, 78 insertions(+), 2 deletions(-) diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index 560fbe048a6..0dd3544d3ea 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -203,6 +203,14 @@ def git_am(patch, use_threeway=False, ignore_whitespace=False): def has_git_rev(rev): + """Checks whether the Git repository in the current directory has the given revision. + + Keyword arguments: + rev -- The revision to check + + Returns: + True if the revision exists, otherwise False. + """ cmd = ["git", "rev-parse", "--verify", "-q", rev] return subprocess.run(cmd, stdout=subprocess.DEVNULL).returncode == 0 @@ -212,7 +220,7 @@ def __init__( self, name, url, - old_tag, + tag, patch_list, copy_upstream_src, patch_options={}, @@ -220,9 +228,23 @@ def __init__( pre_patch_hook=None, pre_patch_commits=0, ): + """Initializes a Lib instance. + + Keyword arguments: + name -- The name of the library. + url -- The URL of the upstream repository. + tag -- The tag in the upstream repository to use. Can be any (e.g., commit hash or tag). + patch_list -- The list of patches in the patch directory to apply. + copy_upstream_src -- A callable that takes the path to the wpilib root and copies the files from the clone of the upstream into the appropriate thirdparty directory. Will only be called when the current directory is the upstream clone. + patch_options -- The dictionary of options to use when applying patches. Corresponds to the parameters of git_am. + + Keyword-only arguments: + pre_patch_hook -- Optional callable taking no parameters that will be called before applying patches. + pre_patch_commits -- Number of commits added by pre_patch_hook. + """ self.name = name self.url = url - self.old_tag = old_tag + self.old_tag = tag self.patch_list = patch_list self.copy_upstream_src = copy_upstream_src self.patch_options = patch_options @@ -231,6 +253,7 @@ def __init__( self.wpilib_root = get_repo_root() def check_patches(self): + """Checks the patch list supplied to the constructor against the patches in the patch directory.""" patch_directory_patches = set() patch_directory = os.path.join( self.wpilib_root, f"upstream_utils/{self.name}_patches" @@ -260,6 +283,14 @@ def check_patches(self): ) def get_repo_path(self, tempdir=None): + """Returns the path to the clone of the upstream repository. + + Keyword argument: + tempdir -- The path to the temporary directory to use. If None (the default), uses tempfile.gettempdir(). + + Returns: + The path to the clone of the upstream repository. Will be absolute if tempdir is absolute. + """ if tempdir is None: tempdir = tempfile.gettempdir() repo = os.path.basename(self.url) @@ -268,6 +299,11 @@ def get_repo_path(self, tempdir=None): return dest def open_repo(self, *, err_msg_if_absent): + """Changes the current working directory to the upstream repository. If err_msg_if_absent is not None and the upstream repository does not exist, the program exits with return code 1. + + Keyword-only argument: + err_msg_if_absent -- The error message to print to stderr if the upstream repository does not exist. If None, the upstream repository will be cloned without emitting any warnings. + """ os.chdir(tempfile.gettempdir()) dest = self.get_repo_path(os.getcwd()) @@ -283,6 +319,11 @@ def open_repo(self, *, err_msg_if_absent): os.chdir(dest) def get_root_tags(self): + """Returns a list of potential root tags. + + Returns: + A list of the potential root tags. + """ root_tag_output = subprocess.run( ["git", "tag", "--list", "upstream_utils_root-*"], capture_output=True, @@ -291,6 +332,11 @@ def get_root_tags(self): return root_tag_output.splitlines() def get_root_tag(self): + """Returns the root tag (the default tag to apply the patches relative to). If there are multiple candidates, prints an error to stderr and the program exits with return code 1. + + Returns: + The root tag. + """ root_tags = self.get_root_tags() if len(root_tags) == 0: print( @@ -307,6 +353,11 @@ def get_root_tag(self): return root_tags[0] def set_root_tag(self, tag): + """Sets the root tag, deleting any potential candidates first. + + Keyword argument: + tag -- The tag to set as the root tag. + """ root_tags = self.get_root_tags() if len(root_tags) > 1: @@ -318,6 +369,7 @@ def set_root_tag(self, tag): subprocess.run(["git", "tag", f"upstream_utils_root-{tag}", tag]) def apply_patches(self): + """Applies the patches listed in the patch list to the current directory.""" if self.pre_patch_hook is not None: self.pre_patch_hook() @@ -330,6 +382,11 @@ def apply_patches(self): ) def replace_tag(self, tag): + """Replaces the tag in the script. + + Keyword argument: + tag -- The tag to replace the script tag with. + """ path = os.path.join(self.wpilib_root, f"upstream_utils/{self.name}.py") with open(path, "r") as file: lines = file.readlines() @@ -349,6 +406,7 @@ def replace_tag(self, tag): file.writelines(lines) def info(self): + """Prints info about the library to stdout.""" print(f"Repository name: {self.name}") print(f"Upstream URL: {self.url}") print(f"Upstream tag: {self.old_tag}") @@ -359,6 +417,7 @@ def info(self): print(f"WPILib root: {self.wpilib_root}") def clone(self): + """Clones the upstream repository and sets it up.""" self.open_repo(err_msg_if_absent=None) subprocess.run(["git", "switch", "--detach", self.old_tag]) @@ -366,6 +425,7 @@ def clone(self): self.set_root_tag(self.old_tag) def reset(self): + """Resets the clone of the upstream repository to the state specified by the script and patches.""" self.open_repo( err_msg_if_absent='There\'s nothing to reset. Run the "clone" command first.' ) @@ -377,6 +437,11 @@ def reset(self): self.apply_patches() def rebase(self, new_tag): + """Rebases the patches. + + Keyword argument: + new_tag -- The tag to rebase onto. + """ self.open_repo( err_msg_if_absent='There\'s nothing to rebase. Run the "clone" command first.' ) @@ -399,6 +464,11 @@ def rebase(self, new_tag): ) def format_patch(self, tag=None): + """Generates patch files for the upstream repository and moves them into the patch directory. + + Keyword argument: + tag -- The tag of the commit of the upstream repository the patch commit were applied onto. If None (the default), the root tag will be used. + """ self.open_repo( err_msg_if_absent='There\'s nothing to run format-patch on. Run the "clone" and "rebase" commands first.' ) @@ -451,6 +521,7 @@ def format_patch(self, tag=None): self.replace_tag(script_tag) def copy_upstream_to_thirdparty(self): + """Copies files from the upstream repository into the thirdparty directory.""" self.open_repo( err_msg_if_absent='There\'s no repository to copy from. Run the "clone" command first.' ) @@ -462,6 +533,11 @@ def copy_upstream_to_thirdparty(self): self.copy_upstream_src(self.wpilib_root) def main(self, argv=sys.argv[1:]): + """Processes the given arguments. + + Keyword argument: + argv -- The arguments to process. Defaults to the arguments passed to the program. + """ parser = argparse.ArgumentParser( description=f"CLI manager of the {self.name} upstream library" ) From 7792a0c902f4937624aba0ea15cbb4782663a5de Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Tue, 16 Jul 2024 14:03:22 -0700 Subject: [PATCH 13/16] Break up long lines --- upstream_utils/upstream_utils.py | 62 +++++++++++++++++++++++--------- 1 file changed, 45 insertions(+), 17 deletions(-) diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index 0dd3544d3ea..6512374adb5 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -203,7 +203,8 @@ def git_am(patch, use_threeway=False, ignore_whitespace=False): def has_git_rev(rev): - """Checks whether the Git repository in the current directory has the given revision. + """Checks whether the Git repository in the current directory has the given + revision. Keyword arguments: rev -- The revision to check @@ -233,13 +234,20 @@ def __init__( Keyword arguments: name -- The name of the library. url -- The URL of the upstream repository. - tag -- The tag in the upstream repository to use. Can be any (e.g., commit hash or tag). + tag -- The tag in the upstream repository to use. Can be any + (e.g., commit hash or tag). patch_list -- The list of patches in the patch directory to apply. - copy_upstream_src -- A callable that takes the path to the wpilib root and copies the files from the clone of the upstream into the appropriate thirdparty directory. Will only be called when the current directory is the upstream clone. - patch_options -- The dictionary of options to use when applying patches. Corresponds to the parameters of git_am. + copy_upstream_src -- A callable that takes the path to the wpilib root + and copies the files from the clone of the upstream + into the appropriate thirdparty directory. Will + only be called when the current directory is the + upstream clone. + patch_options -- The dictionary of options to use when applying patches. + Corresponds to the parameters of git_am. Keyword-only arguments: - pre_patch_hook -- Optional callable taking no parameters that will be called before applying patches. + pre_patch_hook -- Optional callable taking no parameters that will be + called before applying patches. pre_patch_commits -- Number of commits added by pre_patch_hook. """ self.name = name @@ -253,7 +261,9 @@ def __init__( self.wpilib_root = get_repo_root() def check_patches(self): - """Checks the patch list supplied to the constructor against the patches in the patch directory.""" + """Checks that the patch list supplied to the constructor matches the + patches in the patch directory. + """ patch_directory_patches = set() patch_directory = os.path.join( self.wpilib_root, f"upstream_utils/{self.name}_patches" @@ -286,10 +296,12 @@ def get_repo_path(self, tempdir=None): """Returns the path to the clone of the upstream repository. Keyword argument: - tempdir -- The path to the temporary directory to use. If None (the default), uses tempfile.gettempdir(). + tempdir -- The path to the temporary directory to use. If None (the + default), uses tempfile.gettempdir(). Returns: - The path to the clone of the upstream repository. Will be absolute if tempdir is absolute. + The path to the clone of the upstream repository. Will be absolute if + tempdir is absolute. """ if tempdir is None: tempdir = tempfile.gettempdir() @@ -299,10 +311,14 @@ def get_repo_path(self, tempdir=None): return dest def open_repo(self, *, err_msg_if_absent): - """Changes the current working directory to the upstream repository. If err_msg_if_absent is not None and the upstream repository does not exist, the program exits with return code 1. + """Changes the current working directory to the upstream repository. If + err_msg_if_absent is not None and the upstream repository does not + exist, the program exits with return code 1. Keyword-only argument: - err_msg_if_absent -- The error message to print to stderr if the upstream repository does not exist. If None, the upstream repository will be cloned without emitting any warnings. + err_msg_if_absent -- The error message to print to stderr if the + upstream repository does not exist. If None, the upstream repository + will be cloned without emitting any warnings. """ os.chdir(tempfile.gettempdir()) @@ -332,7 +348,9 @@ def get_root_tags(self): return root_tag_output.splitlines() def get_root_tag(self): - """Returns the root tag (the default tag to apply the patches relative to). If there are multiple candidates, prints an error to stderr and the program exits with return code 1. + """Returns the root tag (the default tag to apply the patches relative + to). If there are multiple candidates, prints an error to stderr and the + program exits with return code 1. Returns: The root tag. @@ -369,7 +387,9 @@ def set_root_tag(self, tag): subprocess.run(["git", "tag", f"upstream_utils_root-{tag}", tag]) def apply_patches(self): - """Applies the patches listed in the patch list to the current directory.""" + """Applies the patches listed in the patch list to the current + directory. + """ if self.pre_patch_hook is not None: self.pre_patch_hook() @@ -425,7 +445,9 @@ def clone(self): self.set_root_tag(self.old_tag) def reset(self): - """Resets the clone of the upstream repository to the state specified by the script and patches.""" + """Resets the clone of the upstream repository to the state specified by + the script and patches. + """ self.open_repo( err_msg_if_absent='There\'s nothing to reset. Run the "clone" command first.' ) @@ -464,10 +486,13 @@ def rebase(self, new_tag): ) def format_patch(self, tag=None): - """Generates patch files for the upstream repository and moves them into the patch directory. + """Generates patch files for the upstream repository and moves them into + the patch directory. Keyword argument: - tag -- The tag of the commit of the upstream repository the patch commit were applied onto. If None (the default), the root tag will be used. + tag -- The tag of the commit of the upstream repository the patch + commits were applied onto. If None (the default), the root tag will be + used. """ self.open_repo( err_msg_if_absent='There\'s nothing to run format-patch on. Run the "clone" and "rebase" commands first.' @@ -521,7 +546,9 @@ def format_patch(self, tag=None): self.replace_tag(script_tag) def copy_upstream_to_thirdparty(self): - """Copies files from the upstream repository into the thirdparty directory.""" + """Copies files from the upstream repository into the thirdparty + directory. + """ self.open_repo( err_msg_if_absent='There\'s no repository to copy from. Run the "clone" command first.' ) @@ -536,7 +563,8 @@ def main(self, argv=sys.argv[1:]): """Processes the given arguments. Keyword argument: - argv -- The arguments to process. Defaults to the arguments passed to the program. + argv -- The arguments to process. Defaults to the arguments passed to + the program. """ parser = argparse.ArgumentParser( description=f"CLI manager of the {self.name} upstream library" From 5eb7a57af4a76621b4f4b4c7e3e466ac668ee2f4 Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Tue, 16 Jul 2024 15:15:23 -0700 Subject: [PATCH 14/16] Set root tag after applying patches in reset() --- upstream_utils/upstream_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index 6512374adb5..7c91530949d 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -454,10 +454,10 @@ def reset(self): subprocess.run(["git", "switch", "--detach", self.old_tag]) - self.set_root_tag(self.old_tag) - self.apply_patches() + self.set_root_tag(self.old_tag) + def rebase(self, new_tag): """Rebases the patches. From ba722876dbc4e87c6beeb98a8726d879c523bedb Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Tue, 16 Jul 2024 16:45:51 -0700 Subject: [PATCH 15/16] Correct upstream_utils README Don't redundantly specify the tag --- upstream_utils/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/upstream_utils/README.md b/upstream_utils/README.md index 3fbc3ef9712..2e200ae5b52 100644 --- a/upstream_utils/README.md +++ b/upstream_utils/README.md @@ -32,7 +32,7 @@ Rebase the clone of the upstream repo. Update the `upstream_utils` patch files and the tag in the script. ```bash -./.py format-patch 2.0 +./.py format-patch ``` Copy the updated upstream files into the thirdparty files within allwpilib. From 516222ee22b07acd57577e0976865d5fff924131 Mon Sep 17 00:00:00 2001 From: Joseph Eng Date: Tue, 16 Jul 2024 16:52:57 -0700 Subject: [PATCH 16/16] Remove format-patch tag argument --- upstream_utils/upstream_utils.py | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/upstream_utils/upstream_utils.py b/upstream_utils/upstream_utils.py index 7c91530949d..5851df11f95 100644 --- a/upstream_utils/upstream_utils.py +++ b/upstream_utils/upstream_utils.py @@ -485,29 +485,21 @@ def rebase(self, new_tag): file=sys.stderr, ) - def format_patch(self, tag=None): + def format_patch(self): """Generates patch files for the upstream repository and moves them into the patch directory. - - Keyword argument: - tag -- The tag of the commit of the upstream repository the patch - commits were applied onto. If None (the default), the root tag will be - used. """ self.open_repo( err_msg_if_absent='There\'s nothing to run format-patch on. Run the "clone" and "rebase" commands first.' ) - if tag is None: - tag = self.get_root_tag() - script_tag = tag.removeprefix("upstream_utils_root-") - else: - script_tag = tag + root_tag = self.get_root_tag() + script_tag = root_tag.removeprefix("upstream_utils_root-") - start_commit = tag + start_commit = root_tag if self.pre_patch_commits > 0: commits_since_tag_output = subprocess.run( - ["git", "log", "--format=format:%h", f"{tag}..HEAD"], + ["git", "log", "--format=format:%h", f"{start_commit}..HEAD"], capture_output=True, ).stdout commits_since_tag = commits_since_tag_output.count(b"\n") + 1 @@ -593,12 +585,6 @@ def main(self, argv=sys.argv[1:]): "format-patch", help="Generates patch files for the upstream repository and moves them into the upstream_utils patch directory", ) - parser_format_patch.add_argument( - "new_tag", - nargs="?", - default=None, - help="The tag for the commit before the patches", - ) subparsers.add_parser( "copy-upstream-to-thirdparty", @@ -617,7 +603,7 @@ def main(self, argv=sys.argv[1:]): elif args.subcommand == "rebase": self.rebase(args.new_tag) elif args.subcommand == "format-patch": - self.format_patch(args.new_tag) + self.format_patch() elif args.subcommand == "copy-upstream-to-thirdparty": self.copy_upstream_to_thirdparty()