From ceba27b8526dda9a7d0c1234e1b894b099fe5cb3 Mon Sep 17 00:00:00 2001 From: Julien Cristau Date: Fri, 29 Mar 2024 12:11:54 +0100 Subject: [PATCH] Bug 1882852 - remove vendored createprecomplete in iscript and signingscript This needed to be kept in sync with the code in mozilla-central, or bad things happen, which bit us in https://bugzilla.mozilla.org/show_bug.cgi?id=1882322 Instead, directly add remove instructions for the extra signature files we're adding, leaving the rest of the file unchanged. --- iscript/src/iscript/autograph.py | 33 +++++--- iscript/src/iscript/createprecomplete.py | 76 ------------------- iscript/tests/test_autograph.py | 14 ++-- iscript/tox.ini | 2 - .../src/signingscript/createprecomplete.py | 76 ------------------- signingscript/src/signingscript/sign.py | 37 +++++---- signingscript/tests/test_sign.py | 14 ++-- signingscript/tox.ini | 2 +- 8 files changed, 57 insertions(+), 197 deletions(-) delete mode 100644 iscript/src/iscript/createprecomplete.py delete mode 100644 signingscript/src/signingscript/createprecomplete.py diff --git a/iscript/src/iscript/autograph.py b/iscript/src/iscript/autograph.py index 045650fd4..2a133a225 100644 --- a/iscript/src/iscript/autograph.py +++ b/iscript/src/iscript/autograph.py @@ -16,7 +16,6 @@ from mozpack import mozjar from requests_hawk import HawkAuth -from iscript.createprecomplete import generate_precomplete from iscript.exceptions import IScriptError from scriptworker_client.aio import raise_future_exceptions, retry_async from scriptworker_client.utils import makedirs, rm @@ -101,9 +100,9 @@ async def sign_widevine_dir(config, sign_config, app_dir): all_files.append(to) await raise_future_exceptions(tasks) remove_extra_files(app_dir, all_files) - # Regenerate the `precomplete` file, which is used for cleanup before + # Update the `precomplete` file, which is used for cleanup before # applying a complete mar. - _run_generate_precomplete(config, app_dir) + _update_precomplete(config, app_dir) return app_dir @@ -143,16 +142,26 @@ def _get_widevine_signing_files(file_list): return files -# _run_generate_precomplete {{{1 -def _run_generate_precomplete(config, app_dir): +# _update_precomplete {{{1 +def _update_precomplete(config, app_dir): """Regenerate `precomplete` file with widevine sig paths for complete mar.""" log.info("Generating `precomplete` file...") - path = _ensure_one_precomplete(app_dir, "before") - with open(path, "r") as fh: + precomplete = _ensure_one_precomplete(app_dir) + with open(precomplete, "r") as fh: before = fh.readlines() - generate_precomplete(os.path.dirname(path)) - path = _ensure_one_precomplete(app_dir, "after") - with open(path, "r") as fh: + with open(precomplete, "w") as fh: + for line in before: + fh.write(line) + instr, path = line.strip().split(None, 1) + if instr != "remove": + continue + if not path or path[0] != '"' or path[-1] != '"': + continue + file = path[1:-1] + if _get_widevine_signing_files([file]): + sigfile = _get_mac_sigpath(file) + fh.write('remove "{}"\n'.format(sigfile)) + with open(precomplete, "r") as fh: after = fh.readlines() # Create diff file makedirs(os.path.join(config["artifact_dir"], "public", "logs")) @@ -163,13 +172,13 @@ def _run_generate_precomplete(config, app_dir): # _ensure_one_precomplete {{{1 -def _ensure_one_precomplete(tmp_dir, adj): +def _ensure_one_precomplete(tmp_dir): """Ensure we only have one `precomplete` file in `tmp_dir`.""" precompletes = glob.glob(os.path.join(tmp_dir, "**", "precomplete"), recursive=True) if len(precompletes) < 1: raise IScriptError('No `precomplete` file found in "%s"', tmp_dir) if len(precompletes) > 1: - raise IScriptError('More than one `precomplete` file %s in "%s"', adj, tmp_dir) + raise IScriptError('More than one `precomplete` file in "%s"', tmp_dir) return precompletes[0] diff --git a/iscript/src/iscript/createprecomplete.py b/iscript/src/iscript/createprecomplete.py deleted file mode 100644 index 70763aa00..000000000 --- a/iscript/src/iscript/createprecomplete.py +++ /dev/null @@ -1,76 +0,0 @@ -# Any copyright is dedicated to the Public Domain. -# http://creativecommons.org/publicdomain/zero/1.0/ - -# Creates the precomplete file containing the remove and rmdir application -# update instructions which is used to remove files and directories that are no -# longer present in a complete update. The current working directory is used for -# the location to enumerate and to create the precomplete file. - -import os -import sys - - -def get_build_entries(root_path): - """Iterates through the root_path, creating a list for each file and - directory. Excludes any file paths ending with channel-prefs.js. - """ - rel_file_path_set = set() - rel_dir_path_set = set() - for root, dirs, files in os.walk(root_path): - for file_name in files: - parent_dir_rel_path = root[len(root_path) + 1 :] - rel_path_file = os.path.join(parent_dir_rel_path, file_name) - rel_path_file = rel_path_file.replace("\\", "/") - if not ( - rel_path_file.endswith("channel-prefs.js") - or rel_path_file.endswith("update-settings.ini") - or "/ChannelPrefs.framework/" in rel_path_file - or rel_path_file.startswith("ChannelPrefs.framework/") - or "/UpdateSettings.framework/" in rel_path_file - or rel_path_file.startswith("UpdateSettings.framework/") - or "distribution/" in rel_path_file - ): - rel_file_path_set.add(rel_path_file) - - for dir_name in dirs: - parent_dir_rel_path = root[len(root_path) + 1 :] - rel_path_dir = os.path.join(parent_dir_rel_path, dir_name) - rel_path_dir = rel_path_dir.replace("\\", "/") + "/" - if rel_path_dir.find("distribution/") == -1: - rel_dir_path_set.add(rel_path_dir) - - rel_file_path_list = list(rel_file_path_set) - rel_file_path_list.sort(reverse=True) - rel_dir_path_list = list(rel_dir_path_set) - rel_dir_path_list.sort(reverse=True) - - return rel_file_path_list, rel_dir_path_list - - -def generate_precomplete(root_path): - """Creates the precomplete file containing the remove and rmdir - application update instructions. The given directory is used - for the location to enumerate and to create the precomplete file. - """ - rel_path_precomplete = "precomplete" - # If inside a Mac bundle use the root of the bundle for the path. - if os.path.basename(root_path) == "Resources": - root_path = os.path.abspath(os.path.join(root_path, "../../")) - rel_path_precomplete = "Contents/Resources/precomplete" - - precomplete_file_path = os.path.join(root_path, rel_path_precomplete) - # Open the file so it exists before building the list of files and open it - # in binary mode to prevent OS specific line endings. - precomplete_file = open(precomplete_file_path, "wb") - rel_file_path_list, rel_dir_path_list = get_build_entries(root_path) - for rel_file_path in rel_file_path_list: - precomplete_file.write('remove "{}"\n'.format(rel_file_path).encode("utf-8")) - - for rel_dir_path in rel_dir_path_list: - precomplete_file.write('rmdir "{}"\n'.format(rel_dir_path).encode("utf-8")) - - precomplete_file.close() - - -if __name__ == "__main__": - generate_precomplete(os.getcwd()) diff --git a/iscript/tests/test_autograph.py b/iscript/tests/test_autograph.py index 822dd1086..2a50d2597 100644 --- a/iscript/tests/test_autograph.py +++ b/iscript/tests/test_autograph.py @@ -135,8 +135,7 @@ def fake_isfile(path): mocker.patch.object(autograph, "sign_widevine_with_autograph", new=noop_async) mocker.patch.object(autograph, "makedirs", new=noop_sync) - mocker.patch.object(autograph, "generate_precomplete", new=noop_sync) - mocker.patch.object(autograph, "_run_generate_precomplete", new=noop_sync) + mocker.patch.object(autograph, "_update_precomplete", new=noop_sync) mocker.patch.object(os.path, "isfile", new=fake_isfile) mocker.patch.object(os, "walk", new=fake_walk) @@ -172,22 +171,21 @@ def test_get_widevine_signing_files(filenames, expected): assert autograph._get_widevine_signing_files(filenames) == expected -# _run_generate_precomplete {{{1 +# _update_precomplete {{{1 @pytest.mark.parametrize("num_precomplete,raises", ((1, False), (0, True), (2, True))) -def test_run_generate_precomplete(tmp_path, num_precomplete, raises, mocker): - mocker.patch.object(autograph, "generate_precomplete", new=noop_sync) +def test_update_precomplete(tmp_path, num_precomplete, raises, mocker): work_dir = tmp_path / "work" config = {"artifact_dir": tmp_path / "artifacts"} for i in range(0, num_precomplete): path = os.path.join(work_dir, "foo", str(i)) makedirs(path) with open(os.path.join(path, "precomplete"), "w") as fh: - fh.write("blah") + fh.write('remove "blah"\n') if raises: with pytest.raises(IScriptError): - autograph._run_generate_precomplete(config, work_dir) + autograph._update_precomplete(config, work_dir) else: - autograph._run_generate_precomplete(config, work_dir) + autograph._update_precomplete(config, work_dir) # remove_extra_files {{{1 diff --git a/iscript/tox.ini b/iscript/tox.ini index d397e8cb2..91e536105 100644 --- a/iscript/tox.ini +++ b/iscript/tox.ini @@ -58,5 +58,3 @@ addopts = -vv -s --color=yes [coverage:run] branch = true -omit = - src/iscript/createprecomplete.py diff --git a/signingscript/src/signingscript/createprecomplete.py b/signingscript/src/signingscript/createprecomplete.py deleted file mode 100644 index 70763aa00..000000000 --- a/signingscript/src/signingscript/createprecomplete.py +++ /dev/null @@ -1,76 +0,0 @@ -# Any copyright is dedicated to the Public Domain. -# http://creativecommons.org/publicdomain/zero/1.0/ - -# Creates the precomplete file containing the remove and rmdir application -# update instructions which is used to remove files and directories that are no -# longer present in a complete update. The current working directory is used for -# the location to enumerate and to create the precomplete file. - -import os -import sys - - -def get_build_entries(root_path): - """Iterates through the root_path, creating a list for each file and - directory. Excludes any file paths ending with channel-prefs.js. - """ - rel_file_path_set = set() - rel_dir_path_set = set() - for root, dirs, files in os.walk(root_path): - for file_name in files: - parent_dir_rel_path = root[len(root_path) + 1 :] - rel_path_file = os.path.join(parent_dir_rel_path, file_name) - rel_path_file = rel_path_file.replace("\\", "/") - if not ( - rel_path_file.endswith("channel-prefs.js") - or rel_path_file.endswith("update-settings.ini") - or "/ChannelPrefs.framework/" in rel_path_file - or rel_path_file.startswith("ChannelPrefs.framework/") - or "/UpdateSettings.framework/" in rel_path_file - or rel_path_file.startswith("UpdateSettings.framework/") - or "distribution/" in rel_path_file - ): - rel_file_path_set.add(rel_path_file) - - for dir_name in dirs: - parent_dir_rel_path = root[len(root_path) + 1 :] - rel_path_dir = os.path.join(parent_dir_rel_path, dir_name) - rel_path_dir = rel_path_dir.replace("\\", "/") + "/" - if rel_path_dir.find("distribution/") == -1: - rel_dir_path_set.add(rel_path_dir) - - rel_file_path_list = list(rel_file_path_set) - rel_file_path_list.sort(reverse=True) - rel_dir_path_list = list(rel_dir_path_set) - rel_dir_path_list.sort(reverse=True) - - return rel_file_path_list, rel_dir_path_list - - -def generate_precomplete(root_path): - """Creates the precomplete file containing the remove and rmdir - application update instructions. The given directory is used - for the location to enumerate and to create the precomplete file. - """ - rel_path_precomplete = "precomplete" - # If inside a Mac bundle use the root of the bundle for the path. - if os.path.basename(root_path) == "Resources": - root_path = os.path.abspath(os.path.join(root_path, "../../")) - rel_path_precomplete = "Contents/Resources/precomplete" - - precomplete_file_path = os.path.join(root_path, rel_path_precomplete) - # Open the file so it exists before building the list of files and open it - # in binary mode to prevent OS specific line endings. - precomplete_file = open(precomplete_file_path, "wb") - rel_file_path_list, rel_dir_path_list = get_build_entries(root_path) - for rel_file_path in rel_file_path_list: - precomplete_file.write('remove "{}"\n'.format(rel_file_path).encode("utf-8")) - - for rel_dir_path in rel_dir_path_list: - precomplete_file.write('rmdir "{}"\n'.format(rel_dir_path).encode("utf-8")) - - precomplete_file.close() - - -if __name__ == "__main__": - generate_precomplete(os.getcwd()) diff --git a/signingscript/src/signingscript/sign.py b/signingscript/src/signingscript/sign.py index af5b13915..ce6d23ce9 100644 --- a/signingscript/src/signingscript/sign.py +++ b/signingscript/src/signingscript/sign.py @@ -31,7 +31,6 @@ from winsign.crypto import load_pem_certs from signingscript import task, utils -from signingscript.createprecomplete import generate_precomplete from signingscript.exceptions import SigningScriptError from signingscript.rcodesign import RCodesignError, rcodesign_notarize, rcodesign_notary_wait, rcodesign_staple @@ -297,9 +296,9 @@ async def sign_widevine_zip(context, orig_path, fmt): all_files.append(to) await raise_future_exceptions(tasks) remove_extra_files(tmp_dir, all_files) - # Regenerate the `precomplete` file, which is used for cleanup before + # Update the `precomplete` file, which is used for cleanup before # applying a complete mar. - _run_generate_precomplete(context, tmp_dir) + _update_precomplete(context, tmp_dir) await _create_zipfile(context, orig_path, all_files, mode="w", tmp_dir=tmp_dir) return orig_path @@ -354,9 +353,9 @@ async def sign_widevine_tar(context, orig_path, fmt): all_files.append(to) await raise_future_exceptions(tasks) remove_extra_files(tmp_dir, all_files) - # Regenerate the `precomplete` file, which is used for cleanup before + # Update the `precomplete` file, which is used for cleanup before # applying a complete mar. - _run_generate_precomplete(context, tmp_dir) + _update_precomplete(context, tmp_dir) await _create_tarfile(context, orig_path, all_files, compression, tmp_dir=tmp_dir) return orig_path @@ -558,16 +557,26 @@ def _get_omnija_signing_files(file_list): return files -# _run_generate_precomplete {{{1 -def _run_generate_precomplete(context, tmp_dir): +# _update_precomplete {{{1 +def _update_precomplete(context, tmp_dir): """Regenerate `precomplete` file with widevine sig paths for complete mar.""" log.info("Generating `precomplete` file...") - path = _ensure_one_precomplete(tmp_dir, "before") - with open(path, "r") as fh: + precomplete = _ensure_one_precomplete(tmp_dir) + with open(precomplete, "r") as fh: before = fh.readlines() - generate_precomplete(os.path.dirname(path)) - path = _ensure_one_precomplete(tmp_dir, "after") - with open(path, "r") as fh: + with open(precomplete, "w") as fh: + for line in before: + fh.write(line) + instr, path = line.strip().split(None, 1) + if instr != "remove": + continue + if not path or path[0] != '"' or path[-1] != '"': + continue + file = path[1:-1] + if _get_widevine_signing_files([file]): + sigfile = _get_mac_sigpath(file) + fh.write('remove "{}"\n'.format(sigfile)) + with open(precomplete, "r") as fh: after = fh.readlines() # Create diff file diff_path = os.path.join(context.config["work_dir"], "precomplete.diff") @@ -578,14 +587,14 @@ def _run_generate_precomplete(context, tmp_dir): # _ensure_one_precomplete {{{1 -def _ensure_one_precomplete(tmp_dir, adj): +def _ensure_one_precomplete(tmp_dir): """Ensure we only have one `precomplete` file in `tmp_dir`.""" return get_single_item_from_sequence( glob.glob(os.path.join(tmp_dir, "**", "precomplete"), recursive=True), condition=lambda _: True, ErrorClass=SigningScriptError, no_item_error_message='No `precomplete` file found in "{}"'.format(tmp_dir), - too_many_item_error_message='More than one `precomplete` file {} in "{}"'.format(adj, tmp_dir), + too_many_item_error_message='More than one `precomplete` file in "{}"'.format(tmp_dir), ) diff --git a/signingscript/tests/test_sign.py b/signingscript/tests/test_sign.py index 9384eae0f..4515790f8 100644 --- a/signingscript/tests/test_sign.py +++ b/signingscript/tests/test_sign.py @@ -525,10 +525,9 @@ def fake_isfile(path): mocker.patch.object(sign, "sign_file", new=noop_async) mocker.patch.object(sign, "sign_widevine_with_autograph", new=noop_async) mocker.patch.object(sign, "makedirs", new=noop_sync) - mocker.patch.object(sign, "generate_precomplete", new=noop_sync) + mocker.patch.object(sign, "_update_precomplete", new=noop_sync) mocker.patch.object(sign, "_create_tarfile", new=noop_async) mocker.patch.object(sign, "_create_zipfile", new=noop_async) - mocker.patch.object(sign, "_run_generate_precomplete", new=noop_sync) mocker.patch.object(os.path, "isfile", new=fake_isfile) if raises: @@ -576,21 +575,20 @@ def test_get_widevine_signing_files(filenames, expected): assert sign._get_widevine_signing_files(filenames) == expected -# _run_generate_precomplete {{{1 +# _update_precomplete {{{1 @pytest.mark.parametrize("num_precomplete,raises", ((1, False), (0, True), (2, True))) -def test_run_generate_precomplete(context, num_precomplete, raises, mocker): - mocker.patch.object(sign, "generate_precomplete", new=noop_sync) +def test_update_precomplete(context, num_precomplete, raises, mocker): work_dir = context.config["work_dir"] for i in range(0, num_precomplete): path = os.path.join(work_dir, "foo", str(i)) makedirs(path) with open(os.path.join(path, "precomplete"), "w") as fh: - fh.write("blah") + fh.write('remove "blah"\n') if raises: with pytest.raises(SigningScriptError): - sign._run_generate_precomplete(context, work_dir) + sign._update_precomplete(context, work_dir) else: - sign._run_generate_precomplete(context, work_dir) + sign._update_precomplete(context, work_dir) # remove_extra_files {{{1 diff --git a/signingscript/tox.ini b/signingscript/tox.ini index fdb06647d..b15061d18 100644 --- a/signingscript/tox.ini +++ b/signingscript/tox.ini @@ -49,4 +49,4 @@ addopts = -vv --color=yes [coverage:run] branch = True -omit = tests/*,src/signingscript/createprecomplete.py,src/signingscript/vendored/* +omit = tests/*,src/signingscript/vendored/*