From 60cf77be7eca573cd6d2c7fbd9ef4041eee1290f Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 6 May 2024 11:54:56 +0200 Subject: [PATCH 01/40] Bazel: add codeql specific packaging library This encapsulate arch specific logic, local installation and separation of zip files into generic and arch-specific parts as required by the internal build. --- misc/bazel/internal/BUILD.bazel | 1 + misc/bazel/internal/install.py | 25 +++ misc/bazel/pkg.bzl | 264 ++++++++++++++++++++++++++++++++ misc/bazel/pkg_runfiles.bzl | 33 ---- swift/BUILD.bazel | 131 +++++++--------- swift/downgrades/BUILD.bazel | 4 +- swift/extractor/BUILD.bazel | 7 +- swift/tools/BUILD.bazel | 24 +-- 8 files changed, 358 insertions(+), 131 deletions(-) create mode 100644 misc/bazel/internal/install.py create mode 100644 misc/bazel/pkg.bzl delete mode 100644 misc/bazel/pkg_runfiles.bzl diff --git a/misc/bazel/internal/BUILD.bazel b/misc/bazel/internal/BUILD.bazel index e69de29bb2d1..d9663e7f0c01 100644 --- a/misc/bazel/internal/BUILD.bazel +++ b/misc/bazel/internal/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["install.py"]) diff --git a/misc/bazel/internal/install.py b/misc/bazel/internal/install.py new file mode 100644 index 000000000000..f57cae04596c --- /dev/null +++ b/misc/bazel/internal/install.py @@ -0,0 +1,25 @@ +import argparse +import pathlib +import shutil +import subprocess +from python.runfiles import runfiles + +runfiles = runfiles.Create() +if not runfiles: + raise Exception("Installer should be run with `bazel run`") + +parser = argparse.ArgumentParser() +parser.add_argument("--destdir", type=pathlib.Path, required=True) +parser.add_argument("--script", required=True) +parser.add_argument("--build-file", required=True) +opts = parser.parse_args() + +script = runfiles.Rlocation(opts.script) +build_file = runfiles.Rlocation(opts.build_file) +destdir = pathlib.Path(build_file).parent / opts.destdir + +if destdir.exists(): + shutil.rmtree(destdir) + +destdir.mkdir(parents=True) +subprocess.run([script, "--destdir", destdir], check=True) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl new file mode 100644 index 000000000000..a5e8ef3eac2a --- /dev/null +++ b/misc/bazel/pkg.bzl @@ -0,0 +1,264 @@ +""" +Wrappers and helpers around `rules_pkg` to build codeql packs. +""" + +load("@rules_pkg//pkg:install.bzl", "pkg_install") +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files", _strip_prefix = "strip_prefix") +load("@rules_pkg//pkg:pkg.bzl", "pkg_zip") +load("@rules_python//python:defs.bzl", "py_binary") +load("//:defs.bzl", "codeql_platform") + +def _make_internal(name): + def internal(suffix): + return "%s-%s" % (name, suffix) + + return internal + +def _get_subrule(label, suffix): + if ":" in label or "/" not in label: + return "%s-%s" % (label, suffix) + path, _, pkg = label.rpartition("/") + return "%s/%s:%s-%s" % (path, pkg, pkg, suffix) + +def codeql_pkg_files( + *, + name, + arch_specific = False, + srcs = None, + exes = None, + renames = None, + prefix = None, + visibility = None, + **kwargs): + """ + Wrapper around `pkg_files`. Added functionality: + * `exes` get their file attributes set to be executable. This is important only for POSIX files, there's no need + to mark windows executables as such + * `arch_specific` auto-adds the codeql platform specific directory (linux64, osx64 or windows64), and will be + consumed by a downstream `codeql_pack` to create the arch specific zip. + This should be consumed by `codeql_pkg_filegroup` and `codeql_pack` only. + """ + internal = _make_internal(name) + main_rule = internal("generic") + empty_rule = internal("arch") + if arch_specific: + main_rule, empty_rule = empty_rule, main_rule + prefix = (prefix + "/" if prefix else "") + codeql_platform + pkg_files( + name = empty_rule, + srcs = [], + visibility = visibility, + ) + if not srcs and not exes: + fail("either srcs or exes should be specified for %s" % name) + if srcs and exes: + if renames: + src_renames = {k: v for k, v in renames.items() if k in srcs} + exe_renames = {k: v for k, v in renames.items() if k in exes} + else: + src_renames = None + exe_renames = None + pkg_files( + name = internal("srcs"), + srcs = srcs, + renames = src_renames, + prefix = prefix, + visibility = ["//visibility:private"], + **kwargs + ) + pkg_files( + name = internal("exes"), + srcs = exes, + renames = exe_renames, + prefix = prefix, + visibility = ["//visibility:private"], + attributes = pkg_attributes(mode = "0755"), + **kwargs + ) + pkg_filegroup( + name = main_rule, + srcs = [internal("srcs"), internal("exes")], + visibility = visibility, + ) + else: + pkg_files( + name = main_rule, + srcs = srcs or exes, + attributes = pkg_attributes(mode = "0755") if exes else None, + prefix = prefix, + visibility = visibility, + **kwargs + ) + native.filegroup( + name = name, + srcs = [main_rule], + visibility = visibility, + ) + +def codeql_pkg_wrap(*, name, srcs, arch_specific = False, prefix = None, visibility = None, **kwargs): + """ + Wrap a native `rules_pkg` rule, providing the `arch_specific` functionality and making it consumable by + `codeql_pkg_filegroup` and `codeql_pack`. + """ + internal = _make_internal(name) + main_rule = internal("generic") + empty_rule = internal("arch") + if arch_specific: + main_rule, empty_rule = empty_rule, main_rule + prefix = (prefix + "/" if prefix else "") + codeql_platform + pkg_filegroup( + name = main_rule, + srcs = srcs, + prefix = prefix, + visibility = visibility, + **kwargs + ) + pkg_files( + name = empty_rule, + srcs = [], + visibility = visibility, + ) + native.filegroup( + name = name, + srcs = [main_rule], + visibility = visibility, + ) + +def codeql_pkg_filegroup( + *, + name, + srcs, + srcs_select = None, + visibility = None, + **kwargs): + """ + Combine `codeql_pkg_files` and other `codeql_pkg_filegroup` rules, similar to what `pkg_filegroup` does. + `srcs` is not selectable, but `srcs_select` accepts the same dictionary that a select would accept. + """ + internal = _make_internal(name) + + def transform(srcs, suffix): + return [_get_subrule(src, suffix) for src in srcs] + + pkg_filegroup( + name = internal("generic"), + srcs = transform(srcs, "generic") + (select( + {k: transform(v, "generic") for k, v in srcs_select.items()}, + ) if srcs_select else []), + visibility = visibility, + **kwargs + ) + pkg_filegroup( + name = internal("arch"), + srcs = transform(srcs, "arch") + (select( + {k: transform(v, "arch") for k, v in srcs_select.items()}, + ) if srcs_select else []), + visibility = visibility, + **kwargs + ) + native.filegroup( + name = name, + srcs = [internal("generic"), internal("arch")], + visibility = visibility, + ) + +def codeql_pack(*, name, srcs, zip_prefix = None, zip_filename = "extractor", visibility = visibility, install_dest = "extractor-pack", **kwargs): + """ + Define a codeql pack. This accepts the same arguments as `codeql_pkg_filegroup`, and additionally: + * defines a `-generic-zip` target creating a `-generic.zip` archive with the generic bits, + prefixed with `zip_prefix` (`name` by default) + * defines a `-arch-zip` target creating a `-.zip` archive with the + arch-specific bits, prefixed with `zip_prefix` (`name` by default) + * defines a runnable `-installer` target that will install the pack in `install_dest`, relative to where the + rule is used. The install destination can be overridden appending `-- --destdir=...` to the `bazel run` + invocation. This installation does not use the `zip_prefix`. + """ + internal = _make_internal(name) + zip_prefix = zip_prefix or name + zip_filename = zip_filename or name + codeql_pkg_filegroup( + name = name, + srcs = srcs, + visibility = visibility, + **kwargs + ) + codeql_pkg_filegroup( + name = internal("zip-contents"), + srcs = [name], + prefix = zip_prefix, + visibility = ["//visibility:private"], + ) + pkg_zip( + name = internal("generic-zip"), + srcs = [internal("zip-contents-generic")], + package_file_name = zip_filename + "-generic.zip", + visibility = visibility, + ) + pkg_zip( + name = internal("arch-zip"), + srcs = [internal("zip-contents-arch")], + package_file_name = zip_filename + "-" + codeql_platform + ".zip", + visibility = visibility, + ) + pkg_install( + name = internal("script"), + srcs = [internal("generic"), internal("arch")], + visibility = ["//visibility:private"], + ) + native.filegroup( + # used to locate current source directory + name = internal("build-file"), + srcs = ["BUILD.bazel"], + visibility = ["//visibility:private"], + ) + py_binary( + name = internal("installer"), + srcs = ["//misc/bazel/internal:install.py"], + main = "//misc/bazel/internal:install.py", + data = [internal("build-file"), internal("script")], + deps = ["@rules_python//python/runfiles"], + args = [ + "--build-file=$(rlocationpath %s)" % internal("build-file"), + "--script=$(rlocationpath %s)" % internal("script"), + "--destdir", + install_dest, + ], + visibility = visibility, + ) + +strip_prefix = _strip_prefix + +def _runfiles_group_impl(ctx): + files = [] + for src in ctx.attr.srcs: + rf = src[DefaultInfo].default_runfiles + if rf != None: + files.append(rf.files) + return [ + DefaultInfo( + files = depset(transitive = files), + ), + ] + +_runfiles_group = rule( + implementation = _runfiles_group_impl, + attrs = { + "srcs": attr.label_list(), + }, +) + +def codeql_pkg_runfiles(*, name, exes, **kwargs): + """ + Create a `codeql_pkg_files` with all runfiles from files in `exes`, flattened together. + """ + internal = _make_internal(name) + _runfiles_group( + name = internal("runfiles"), + srcs = exes, + visibility = ["//visibility:private"], + ) + codeql_pkg_files( + name = name, + exes = [internal("runfiles")], + **kwargs + ) diff --git a/misc/bazel/pkg_runfiles.bzl b/misc/bazel/pkg_runfiles.bzl deleted file mode 100644 index 3d3bd8c028d5..000000000000 --- a/misc/bazel/pkg_runfiles.bzl +++ /dev/null @@ -1,33 +0,0 @@ -load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files") - -def _runfiles_group_impl(ctx): - files = [] - for src in ctx.attr.srcs: - rf = src[DefaultInfo].default_runfiles - if rf != None: - files.append(rf.files) - return [ - DefaultInfo( - files = depset(transitive = files), - ), - ] - -_runfiles_group = rule( - implementation = _runfiles_group_impl, - attrs = { - "srcs": attr.label_list(), - }, -) - -def pkg_runfiles(*, name, srcs, **kwargs): - internal_name = "_%s_runfiles" % name - _runfiles_group( - name = internal_name, - srcs = srcs, - ) - kwargs.setdefault("attributes", pkg_attributes(mode = "0755")) - pkg_files( - name = name, - srcs = [internal_name], - **kwargs - ) diff --git a/swift/BUILD.bazel b/swift/BUILD.bazel index 1ced5c9f1ca0..7735da4fa699 100644 --- a/swift/BUILD.bazel +++ b/swift/BUILD.bazel @@ -1,8 +1,11 @@ -load("@rules_pkg//pkg:install.bzl", "pkg_install") -load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files") -load("//:defs.bzl", "codeql_platform") -load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles") -load("//misc/bazel/cmake:cmake.bzl", "generate_cmake") +load( + "//misc/bazel:pkg.bzl", + "codeql_pack", + "codeql_pkg_filegroup", + "codeql_pkg_files", + "codeql_pkg_runfiles", + "codeql_pkg_wrap", +) filegroup( name = "schema", @@ -22,7 +25,7 @@ filegroup( visibility = ["//swift:__subpackages__"], ) -pkg_files( +codeql_pkg_files( name = "dbscheme_files", srcs = [ "ql/lib/swift.dbscheme.stats", @@ -30,100 +33,78 @@ pkg_files( ], ) -pkg_files( +codeql_pkg_files( name = "manifest", srcs = ["codeql-extractor.yml"], ) -pkg_filegroup( - name = "extractor-pack-generic", - srcs = [ - ":manifest", - "//swift/tools", - ] + select({ - "@platforms//os:windows": [], - "//conditions:default": [ - ":dbscheme_files", - "//swift/downgrades", - ], - }), - visibility = ["//visibility:public"], -) - -pkg_filegroup( +codeql_pkg_filegroup( name = "extractor", srcs = ["//swift/extractor:pkg"], - prefix = "tools/" + codeql_platform, + prefix = "tools", ) -pkg_runfiles( - name = "swift-autobuilder", - srcs = ["//swift/swift-autobuilder"], - prefix = "tools/" + codeql_platform, +codeql_pkg_files( + name = "autobuilder-incompatible-os", + arch_specific = True, + exes = ["//swift/tools/diagnostics:autobuilder-incompatible-os"], + prefix = "tools", ) -pkg_runfiles( - name = "diagnostics", - srcs = ["//swift/tools/diagnostics:autobuilder-incompatible-os"], - prefix = "tools/" + codeql_platform, +codeql_pkg_runfiles( + name = "autobuilder", + arch_specific = True, + exes = ["//swift/swift-autobuilder"], + prefix = "tools", ) -pkg_filegroup( - name = "resource-dir-arch", +codeql_pkg_wrap( + name = "resource-dir", srcs = ["//swift/third_party/swift-llvm-support:swift-resource-dir"], - prefix = "resource-dir/" + codeql_platform, - visibility = ["//visibility:public"], + arch_specific = True, + prefix = "resource-dir", ) -pkg_filegroup( - name = "extractor-pack-arch", - srcs = select({ - "@platforms//os:windows": [], - "//conditions:default": [ +codeql_pack( + name = "swift", + srcs = [ + ":dbscheme_files", + ":manifest", + "//swift/downgrades", + "//swift/tools", + ], + srcs_select = { + "@platforms//os:macos": [ ":extractor", - ":resource-dir-arch", + ":resource-dir", + ":autobuilder", ], - }) + select({ - "@platforms//os:macos": [ - ":swift-autobuilder", + "@platforms//os:linux": [ + ":extractor", + ":resource-dir", + ":autobuilder-incompatible-os", ], - "//conditions:default": [ - ":diagnostics", + "@platforms//os:windows": [ + ":autobuilder-incompatible-os", ], - }), - visibility = ["//visibility:public"], -) - -pkg_filegroup( - name = "extractor-pack", - srcs = [ - ":extractor-pack-arch", - ":extractor-pack-generic", - ], + }, visibility = ["//visibility:public"], ) -pkg_install( - name = "_create_extractor_pack", - srcs = ["//swift:extractor-pack"], +alias( + name = "create-extractor-pack", + actual = ":swift-installer", ) -py_binary( - name = "create-extractor-pack", - srcs = ["create_extractor_pack.py"], - main = "create_extractor_pack.py", - deps = [":_create_extractor_pack"], +# TODO: aliases for internal repo backward compatibility +alias( + name = "extractor-pack-generic", + actual = "swift-generic", + visibility = ["//visibility:public"], ) -# TODO this is unneeded here but still used in the internal repo. Remove once it's not -generate_cmake( - name = "cmake", - targets = [ - "//swift/extractor:extractor.real", - "//swift/logging/tests/assertion-diagnostics:assert-false", - ] + select({ - "@platforms//os:linux": ["//swift/tools/diagnostics:autobuilder-incompatible-os"], - "@platforms//os:macos": ["//swift/swift-autobuilder"], - }), +alias( + name = "extractor-pack-arch", + actual = "swift-arch", visibility = ["//visibility:public"], ) diff --git a/swift/downgrades/BUILD.bazel b/swift/downgrades/BUILD.bazel index 5f643aea185d..7f5f6fe4dd6e 100644 --- a/swift/downgrades/BUILD.bazel +++ b/swift/downgrades/BUILD.bazel @@ -1,6 +1,6 @@ -load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") +load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix") -pkg_files( +codeql_pkg_files( name = "downgrades", srcs = glob( ["**"], diff --git a/swift/extractor/BUILD.bazel b/swift/extractor/BUILD.bazel index 3acdbf014e34..02b1a91e3f9a 100644 --- a/swift/extractor/BUILD.bazel +++ b/swift/extractor/BUILD.bazel @@ -1,4 +1,4 @@ -load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles") +load("//misc/bazel:pkg.bzl", "codeql_pkg_runfiles") load("//swift:rules.bzl", "swift_cc_binary") swift_cc_binary( @@ -29,9 +29,10 @@ sh_binary( data = [":extractor.real"], ) -pkg_runfiles( +codeql_pkg_runfiles( name = "pkg", - srcs = [":extractor"], + arch_specific = True, excludes = ["extractor.sh"], # script gets copied as "extractor", no need for the original .sh file + exes = [":extractor"], visibility = ["//swift:__pkg__"], ) diff --git a/swift/tools/BUILD.bazel b/swift/tools/BUILD.bazel index e59561bf528d..5c08835fe352 100644 --- a/swift/tools/BUILD.bazel +++ b/swift/tools/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") +load("//misc/bazel:pkg.bzl", "codeql_pkg_files") sh_binary( name = "qltest", @@ -16,29 +16,17 @@ sh_binary( srcs = ["identify-environment.sh"], ) -pkg_files( - name = "scripts", +codeql_pkg_files( + name = "tools", srcs = [ "autobuild.cmd", + "tracing-config.lua", + ], + exes = [ ":autobuild", ":identify-environment", ":qltest", ], - attributes = pkg_attributes(mode = "0755"), - prefix = "tools", -) - -pkg_files( - name = "tracing-config", - srcs = ["tracing-config.lua"], prefix = "tools", -) - -pkg_filegroup( - name = "tools", - srcs = [ - ":scripts", - ":tracing-config", - ], visibility = ["//swift:__pkg__"], ) From 4d93e8a732786102f2298967825131dacec29071 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 23 May 2024 16:34:34 +0200 Subject: [PATCH 02/40] Bazel: move codeql packaging rules away from some macros --- misc/bazel/pkg.bzl | 272 ++++++++++++++++++++++------------------ swift/BUILD.bazel | 80 ++++++------ swift/tools/BUILD.bazel | 1 - 3 files changed, 187 insertions(+), 166 deletions(-) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index a5e8ef3eac2a..375ec779f06c 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -5,11 +5,12 @@ Wrappers and helpers around `rules_pkg` to build codeql packs. load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files", _strip_prefix = "strip_prefix") load("@rules_pkg//pkg:pkg.bzl", "pkg_zip") +load("@rules_pkg//pkg:providers.bzl", "PackageFilegroupInfo", "PackageFilesInfo") load("@rules_python//python:defs.bzl", "py_binary") load("//:defs.bzl", "codeql_platform") def _make_internal(name): - def internal(suffix): + def internal(suffix = "internal"): return "%s-%s" % (name, suffix) return internal @@ -20,151 +21,173 @@ def _get_subrule(label, suffix): path, _, pkg = label.rpartition("/") return "%s/%s:%s-%s" % (path, pkg, pkg, suffix) +_PackageFileWrapperInfo = provider(fields = {"pfi": "", "src": "", "arch_specific": ""}) + +CodeqlFilesInfo = provider( + doc = """Wrapper around `rules_pkg` `PackageFilesInfo` carrying information about generic and arch-specific files.""", + fields = { + "files": "list of `_PackageFileWrapperInfo`.", + }, +) + +def _codeql_pkg_filegroup_impl(ctx): + prefix = ctx.attr.prefix + if prefix: + prefix += "/" + generic_prefix = prefix + if ctx.attr.arch_specific: + if ctx.target_platform_has_constraint(ctx.attr._windows[platform_common.ConstraintValueInfo]): + plat = "windows64" + elif ctx.target_platform_has_constraint(ctx.attr._macos[platform_common.ConstraintValueInfo]): + plat = "osx64" + else: + plat = "linux64" + prefix = prefix + plat + "/" + + def transform_pfi(pfi, src, prefix = prefix, arch_specific = ctx.attr.arch_specific): + return _PackageFileWrapperInfo( + pfi = PackageFilesInfo( + attributes = pfi.attributes, + dest_src_map = {prefix + d: s for d, s in pfi.dest_src_map.items()}, + ), + src = src, + arch_specific = arch_specific, + ) + + files = [] + + for src in ctx.attr.srcs: + if PackageFilesInfo in src: + pfi = src[PackageFilesInfo] + files.append(transform_pfi(pfi, src.label)) + elif PackageFilegroupInfo in src: + pfgi = src[PackageFilegroupInfo] + if pfgi.pkg_dirs or pfgi.pkg_symlinks: + fail("while assembling %s found %s which contains `pkg_dirs` or `pkg_symlinks` targets" % + (ctx.label, src.label) + ", which is not currently supported") + files += [transform_pfi(pfi, src) for pfi, src in pfgi.pkg_files] + else: + cfi = src[CodeqlFilesInfo] + files += [ + transform_pfi( + pfwi.pfi, + pfwi.src, + # if it was already arch specific the plat prefix was already added + generic_prefix if pfwi.arch_specific else prefix, + pfwi.arch_specific or ctx.attr.arch_specific, + ) + for pfwi in cfi.files + ] + + return [ + CodeqlFilesInfo( + files = files, + ), + DefaultInfo( + files = depset(transitive = [src[DefaultInfo].files for src in ctx.attr.srcs]), + ), + ] + +codeql_pkg_filegroup = rule( + implementation = _codeql_pkg_filegroup_impl, + doc = """CodeQL specific packaging mapping. No `pkg_mkdirs` or `pkg_symlink` rules are supported, either directly + or transitively. Only `pkg_files` and `pkg_filegroup` thereof are allowed.""", + attrs = { + "srcs": attr.label_list( + doc = "List of arch-agnostic `pkg_files`, `pkg_filegroup` or `codeql_pkg_filegroup` targets", + providers = [ + [PackageFilesInfo, DefaultInfo], + [PackageFilegroupInfo, DefaultInfo], + [CodeqlFilesInfo, DefaultInfo], + ], + default = [], + ), + "prefix": attr.string(doc = "Prefix to add to the files", default = ""), + "arch_specific": attr.bool(doc = "Whether the included files should be treated as arch-specific"), + "_windows": attr.label(default = "@platforms//os:windows"), + "_macos": attr.label(default = "@platforms//os:macos"), + }, +) + def codeql_pkg_files( *, name, - arch_specific = False, srcs = None, exes = None, - renames = None, + arch_specific = False, prefix = None, visibility = None, **kwargs): """ Wrapper around `pkg_files`. Added functionality: - * `exes` get their file attributes set to be executable. This is important only for POSIX files, there's no need - to mark windows executables as such - * `arch_specific` auto-adds the codeql platform specific directory (linux64, osx64 or windows64), and will be - consumed by a downstream `codeql_pack` to create the arch specific zip. - This should be consumed by `codeql_pkg_filegroup` and `codeql_pack` only. + * `exes` will get their file attributes set to be executable. This is important only for POSIX files, there's no + need to mark windows executables as such + If `exes` and `srcs` are both used, the resulting rule is a `pkg_filegroup` one. """ internal = _make_internal(name) - main_rule = internal("generic") - empty_rule = internal("arch") - if arch_specific: - main_rule, empty_rule = empty_rule, main_rule - prefix = (prefix + "/" if prefix else "") + codeql_platform - pkg_files( - name = empty_rule, - srcs = [], - visibility = visibility, - ) - if not srcs and not exes: - fail("either srcs or exes should be specified for %s" % name) + if "attributes" in kwargs: + fail("codeql_pkg_files does not support `attributes`. Use `exes` to mark executable files.") + internal_srcs = [] if srcs and exes: - if renames: - src_renames = {k: v for k, v in renames.items() if k in srcs} - exe_renames = {k: v for k, v in renames.items() if k in exes} - else: - src_renames = None - exe_renames = None pkg_files( name = internal("srcs"), srcs = srcs, - renames = src_renames, - prefix = prefix, visibility = ["//visibility:private"], **kwargs ) pkg_files( name = internal("exes"), srcs = exes, - renames = exe_renames, - prefix = prefix, visibility = ["//visibility:private"], - attributes = pkg_attributes(mode = "0755"), + attributes = pkg_attributes(mode = "755"), **kwargs ) - pkg_filegroup( - name = main_rule, - srcs = [internal("srcs"), internal("exes")], - visibility = visibility, - ) + internal_srcs = [internal("srcs"), internal("exes")] else: pkg_files( - name = main_rule, + name = internal(), srcs = srcs or exes, - attributes = pkg_attributes(mode = "0755") if exes else None, - prefix = prefix, visibility = visibility, + attributes = pkg_attributes(mode = "755") if exes else None, **kwargs ) - native.filegroup( + internal_srcs = [internal()] + codeql_pkg_filegroup( name = name, - srcs = [main_rule], - visibility = visibility, - ) - -def codeql_pkg_wrap(*, name, srcs, arch_specific = False, prefix = None, visibility = None, **kwargs): - """ - Wrap a native `rules_pkg` rule, providing the `arch_specific` functionality and making it consumable by - `codeql_pkg_filegroup` and `codeql_pack`. - """ - internal = _make_internal(name) - main_rule = internal("generic") - empty_rule = internal("arch") - if arch_specific: - main_rule, empty_rule = empty_rule, main_rule - prefix = (prefix + "/" if prefix else "") + codeql_platform - pkg_filegroup( - name = main_rule, - srcs = srcs, + srcs = internal_srcs, + arch_specific = arch_specific, prefix = prefix, visibility = visibility, - **kwargs - ) - pkg_files( - name = empty_rule, - srcs = [], - visibility = visibility, - ) - native.filegroup( - name = name, - srcs = [main_rule], - visibility = visibility, ) -def codeql_pkg_filegroup( +def _extract_pkg_filegroup_impl(ctx): + src = ctx.attr.src[CodeqlFilesInfo] + pfi_lbls = [(pfwi.pfi, pfwi.src) for pfwi in src.files if pfwi.arch_specific == ctx.attr.arch_specific] + files = [depset(pfi.dest_src_map.values()) for pfi, _ in pfi_lbls] + return [ + PackageFilegroupInfo(pkg_files = pfi_lbls, pkg_dirs = [], pkg_symlinks = []), + DefaultInfo(files = depset(transitive = files)), + ] + +_extrac_pkg_filegroup = rule( + implementation = _extract_pkg_filegroup_impl, + attrs = { + "src": attr.label(providers = [CodeqlFilesInfo, DefaultInfo]), + "arch_specific": attr.bool(), + }, +) + +def codeql_pack( *, name, - srcs, - srcs_select = None, + srcs = None, + zip_prefix = None, + zip_filename = "extractor", visibility = None, + install_dest = "extractor-pack", **kwargs): """ - Combine `codeql_pkg_files` and other `codeql_pkg_filegroup` rules, similar to what `pkg_filegroup` does. - `srcs` is not selectable, but `srcs_select` accepts the same dictionary that a select would accept. - """ - internal = _make_internal(name) - - def transform(srcs, suffix): - return [_get_subrule(src, suffix) for src in srcs] - - pkg_filegroup( - name = internal("generic"), - srcs = transform(srcs, "generic") + (select( - {k: transform(v, "generic") for k, v in srcs_select.items()}, - ) if srcs_select else []), - visibility = visibility, - **kwargs - ) - pkg_filegroup( - name = internal("arch"), - srcs = transform(srcs, "arch") + (select( - {k: transform(v, "arch") for k, v in srcs_select.items()}, - ) if srcs_select else []), - visibility = visibility, - **kwargs - ) - native.filegroup( - name = name, - srcs = [internal("generic"), internal("arch")], - visibility = visibility, - ) - -def codeql_pack(*, name, srcs, zip_prefix = None, zip_filename = "extractor", visibility = visibility, install_dest = "extractor-pack", **kwargs): - """ - Define a codeql pack. This accepts the same arguments as `codeql_pkg_filegroup`, and additionally: + Define a codeql pack. This accepts `pkg_files`, `pkg_filegroup` or their `codeql_*` counterparts as `srcs`. * defines a `-generic-zip` target creating a `-generic.zip` archive with the generic bits, prefixed with `zip_prefix` (`name` by default) * defines a `-arch-zip` target creating a `-.zip` archive with the @@ -182,31 +205,32 @@ def codeql_pack(*, name, srcs, zip_prefix = None, zip_filename = "extractor", vi visibility = visibility, **kwargs ) - codeql_pkg_filegroup( - name = internal("zip-contents"), - srcs = [name], - prefix = zip_prefix, - visibility = ["//visibility:private"], - ) - pkg_zip( - name = internal("generic-zip"), - srcs = [internal("zip-contents-generic")], - package_file_name = zip_filename + "-generic.zip", - visibility = visibility, - ) - pkg_zip( - name = internal("arch-zip"), - srcs = [internal("zip-contents-arch")], - package_file_name = zip_filename + "-" + codeql_platform + ".zip", - visibility = visibility, - ) + for kind in ("generic", "arch"): + _extrac_pkg_filegroup( + name = internal(kind), + src = name, + arch_specific = kind == "arch", + visibility = ["//visibility:private"], + ) + pkg_filegroup( + name = internal(kind + "-zip-contents"), + srcs = [internal(kind)], + prefix = zip_prefix, + visibility = ["//visibility:private"], + ) + pkg_zip( + name = internal(kind + "-zip"), + srcs = [internal(kind + "-zip-contents")], + package_file_name = zip_filename + "-" + (codeql_platform if kind == "arch" else kind) + ".zip", + visibility = visibility, + ) pkg_install( name = internal("script"), srcs = [internal("generic"), internal("arch")], visibility = ["//visibility:private"], ) native.filegroup( - # used to locate current source directory + # used to locate current src directory name = internal("build-file"), srcs = ["BUILD.bazel"], visibility = ["//visibility:private"], diff --git a/swift/BUILD.bazel b/swift/BUILD.bazel index 7735da4fa699..90bdb8515ac1 100644 --- a/swift/BUILD.bazel +++ b/swift/BUILD.bazel @@ -4,7 +4,6 @@ load( "codeql_pkg_filegroup", "codeql_pkg_files", "codeql_pkg_runfiles", - "codeql_pkg_wrap", ) filegroup( @@ -25,69 +24,68 @@ filegroup( visibility = ["//swift:__subpackages__"], ) -codeql_pkg_files( - name = "dbscheme_files", - srcs = [ - "ql/lib/swift.dbscheme.stats", - "//swift/extractor/trap:generated_dbscheme", - ], -) - -codeql_pkg_files( - name = "manifest", - srcs = ["codeql-extractor.yml"], -) - -codeql_pkg_filegroup( - name = "extractor", - srcs = ["//swift/extractor:pkg"], - prefix = "tools", -) - codeql_pkg_files( name = "autobuilder-incompatible-os", - arch_specific = True, exes = ["//swift/tools/diagnostics:autobuilder-incompatible-os"], - prefix = "tools", ) codeql_pkg_runfiles( name = "autobuilder", arch_specific = True, exes = ["//swift/swift-autobuilder"], +) + +codeql_pkg_filegroup( + name = "tools-arch", + srcs = select({ + "@platforms//os:macos": [ + ":autobuilder", + "//swift/extractor:pkg", + ], + "@platforms//os:linux": [ + ":autobuilder-incompatible-os", + "//swift/extractor:pkg", + ], + "@platforms//os:windows": [ + ":autobuilder-incompatible-os", + ], + }), + arch_specific = True, +) + +codeql_pkg_filegroup( + name = "tools", + srcs = [ + ":tools-arch", + "//swift/tools", + ], prefix = "tools", ) -codeql_pkg_wrap( +codeql_pkg_filegroup( name = "resource-dir", srcs = ["//swift/third_party/swift-llvm-support:swift-resource-dir"], arch_specific = True, prefix = "resource-dir", ) +codeql_pkg_files( + name = "root-files", + srcs = [ + "codeql-extractor.yml", + "ql/lib/swift.dbscheme.stats", + "//swift/extractor/trap:generated_dbscheme", + ], +) + codeql_pack( name = "swift", srcs = [ - ":dbscheme_files", - ":manifest", + ":resource-dir", + ":root-files", + ":tools", "//swift/downgrades", - "//swift/tools", ], - srcs_select = { - "@platforms//os:macos": [ - ":extractor", - ":resource-dir", - ":autobuilder", - ], - "@platforms//os:linux": [ - ":extractor", - ":resource-dir", - ":autobuilder-incompatible-os", - ], - "@platforms//os:windows": [ - ":autobuilder-incompatible-os", - ], - }, visibility = ["//visibility:public"], ) diff --git a/swift/tools/BUILD.bazel b/swift/tools/BUILD.bazel index 5c08835fe352..ae04a587a944 100644 --- a/swift/tools/BUILD.bazel +++ b/swift/tools/BUILD.bazel @@ -27,6 +27,5 @@ codeql_pkg_files( ":identify-environment", ":qltest", ], - prefix = "tools", visibility = ["//swift:__pkg__"], ) From e8b857b79e50a16374105fbeb011e0ed3ed9d495 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 24 May 2024 11:07:29 +0200 Subject: [PATCH 03/40] Bazel/Swift: add zip imports to packs --- .bazelrc | 6 +- .gitattributes | 8 +- MODULE.bazel | 27 + misc/bazel/internal/bin/BUILD.bazel | 8 + misc/bazel/internal/bin/linux/ripunzip | 3 + misc/bazel/internal/bin/macos/ripunzip | 3 + misc/bazel/internal/bin/windows/ripunzip.exe | 3 + misc/bazel/internal/bin/zipmerge/BUILD.bazel | 20 + misc/bazel/internal/bin/zipmerge/zipmerge.cpp | 529 ++++++++++++++++++ misc/bazel/internal/bin/zipmerge/zipmerge.h | 37 ++ .../internal/bin/zipmerge/zipmerge_main.cpp | 5 + misc/bazel/internal/install.py | 18 +- misc/bazel/pkg.bzl | 156 +++++- swift/BUILD.bazel | 3 +- swift/third_party/resource-dir/BUILD.bazel | 9 + .../resource-dir/resource-dir-linux.zip | 3 + .../resource-dir/resource-dir-macos.zip | 3 + swift/third_party/resource-dir/update.sh | 0 18 files changed, 805 insertions(+), 36 deletions(-) create mode 100644 misc/bazel/internal/bin/BUILD.bazel create mode 100755 misc/bazel/internal/bin/linux/ripunzip create mode 100755 misc/bazel/internal/bin/macos/ripunzip create mode 100644 misc/bazel/internal/bin/windows/ripunzip.exe create mode 100644 misc/bazel/internal/bin/zipmerge/BUILD.bazel create mode 100644 misc/bazel/internal/bin/zipmerge/zipmerge.cpp create mode 100644 misc/bazel/internal/bin/zipmerge/zipmerge.h create mode 100644 misc/bazel/internal/bin/zipmerge/zipmerge_main.cpp create mode 100644 swift/third_party/resource-dir/BUILD.bazel create mode 100644 swift/third_party/resource-dir/resource-dir-linux.zip create mode 100644 swift/third_party/resource-dir/resource-dir-macos.zip create mode 100644 swift/third_party/resource-dir/update.sh diff --git a/.bazelrc b/.bazelrc index ca2bb8caee47..a444a97c2df7 100644 --- a/.bazelrc +++ b/.bazelrc @@ -10,10 +10,10 @@ common --override_module=semmle_code=%workspace%/misc/bazel/semmle_code_stub build --repo_env=CC=clang --repo_env=CXX=clang++ -build:linux --cxxopt=-std=c++20 +build:linux --cxxopt=-std=c++20 --host_cxxopt=-std=c++20 # we currently cannot built the swift extractor for ARM -build:macos --cxxopt=-std=c++20 --copt=-arch --copt=x86_64 --linkopt=-arch --linkopt=x86_64 -build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor +build:macos --cxxopt=-std=c++20 --host_cxxopt=-std=c++20 --copt=-arch --copt=x86_64 --linkopt=-arch --linkopt=x86_64 +build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor --host_cxxopt=/std:c++20 --host_cxxopt=/Zc:preprocessor # this requires developer mode, but is required to have pack installer functioning startup --windows_enable_symlinks diff --git a/.gitattributes b/.gitattributes index aa397aab0d20..d2cf32fa7100 100644 --- a/.gitattributes +++ b/.gitattributes @@ -77,4 +77,10 @@ ruby/extractor/cargo-bazel-lock.json -merge # auto-generated files for the C# build csharp/paket.lock linguist-generated=true # needs eol=crlf, as `paket` touches this file and saves it als crlf -csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf +csharp/.paket/Paket.Restore.targets linguist-generated=true eol=cr + +# ripunzip tool +/misc/bazel/internal/bin/*/ripunzip* filter=lfs diff=lfs merge=lfs -text + +# swift prebuilt resources +/swift/third_party/resource-dir/*.zip filter=lfs diff=lfs merge=lfs -text diff --git a/MODULE.bazel b/MODULE.bazel index 27479e1978f7..e7f24827c23e 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -57,6 +57,33 @@ use_repo(node, "nodejs", "nodejs_toolchains") go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") go_sdk.download(version = "1.22.2") +lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files") + +lfs_files( + name = "ripunzip-linux", + srcs = ["//misc/bazel/internal/bin:linux/ripunzip"], +) + +lfs_files( + name = "ripunzip-windows", + srcs = ["//misc/bazel/internal/bin:windows/ripunzip.exe"], +) + +lfs_files( + name = "ripunzip-macos", + srcs = ["//misc/bazel/internal/bin:macos/ripunzip"], +) + +lfs_files( + name = "swift-resource-dir-linux", + srcs = ["//swift/third_party/resource-dir:resource-dir-linux.zip"], +) + +#lfs_files( +# name = "swift-resource-dir-macos", +# srcs = ["//swift/third_party/resource-dir:resource-dir-macos.zip"], +#) + register_toolchains( "@nodejs_toolchains//:all", ) diff --git a/misc/bazel/internal/bin/BUILD.bazel b/misc/bazel/internal/bin/BUILD.bazel new file mode 100644 index 000000000000..74ec7a86e8d5 --- /dev/null +++ b/misc/bazel/internal/bin/BUILD.bazel @@ -0,0 +1,8 @@ +load("@bazel_skylib//rules:native_binary.bzl", "native_binary") + +native_binary( + name = "ripunzip", + src = select({"@platforms//os:" + os: "@ripunzip-" + os for os in ("linux", "windows", "macos")}), + out = "ripunzip.exe", + visibility = ["//visibility:public"], +) diff --git a/misc/bazel/internal/bin/linux/ripunzip b/misc/bazel/internal/bin/linux/ripunzip new file mode 100755 index 000000000000..151e21ddff3f --- /dev/null +++ b/misc/bazel/internal/bin/linux/ripunzip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:609d90e14e1cdd0351eee3d45a6e09e36cc44b3289e912a19945a49f4fdc416d +size 5775960 diff --git a/misc/bazel/internal/bin/macos/ripunzip b/misc/bazel/internal/bin/macos/ripunzip new file mode 100755 index 000000000000..6ac67e6f0161 --- /dev/null +++ b/misc/bazel/internal/bin/macos/ripunzip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91e21ae4c75fd79dee4afc16aa79cef5a404080a4784360923edd23b2b2e988d +size 5316808 diff --git a/misc/bazel/internal/bin/windows/ripunzip.exe b/misc/bazel/internal/bin/windows/ripunzip.exe new file mode 100644 index 000000000000..bae1e9566149 --- /dev/null +++ b/misc/bazel/internal/bin/windows/ripunzip.exe @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7623346ec339ab7bf4179297dc129dcedeed932c49e594e6cfddfd1c9c2ff11c +size 4527104 diff --git a/misc/bazel/internal/bin/zipmerge/BUILD.bazel b/misc/bazel/internal/bin/zipmerge/BUILD.bazel new file mode 100644 index 000000000000..6aa11d4fe505 --- /dev/null +++ b/misc/bazel/internal/bin/zipmerge/BUILD.bazel @@ -0,0 +1,20 @@ +cc_library( + name = "lib", + srcs = [ + "zipmerge.cpp", + ], + hdrs = ["zipmerge.h"], +) + +cc_binary( + name = "zipmerge", + srcs = [ + "zipmerge_main.cpp", + ], + visibility = ["//visibility:public"], + deps = [ + ":lib", + ], +) + +#TODO port tests from internal repo diff --git a/misc/bazel/internal/bin/zipmerge/zipmerge.cpp b/misc/bazel/internal/bin/zipmerge/zipmerge.cpp new file mode 100644 index 000000000000..216c415e8581 --- /dev/null +++ b/misc/bazel/internal/bin/zipmerge/zipmerge.cpp @@ -0,0 +1,529 @@ +/* + Utility for munging zip files. + + The high-level pseudo-code is: + for each input zip Z: + for each file F in Z: + F.name = adjust(F.name) + if F.name should be included: + write F to the output zip + + File inclusion testing consists of two parts: + 1. Don't include anything matching an explicit removal list. + 2. If the same filename occurs in multiple input zips, only include the file from the last input + zip. + + Filename adjustment consists of optionally prepending a prefix to the filename. +*/ + +#include "misc/bazel/internal/bin/zipmerge/zipmerge.h" + +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#define unlink(s) DeleteFileA(s) +#else +#include +#include +#include +#include +#include +#endif + +#include + +namespace { +struct { + FILE* file; + uint32_t num_bytes_written; + uint16_t num_files_written; +} output_zip{}; // The zip file being written. + +struct { + uint8_t* bytes; + uint16_t length; +} filename_prefix{}; // A string to prepend to all filenames added to the output file. + +constexpr size_t maximum_input_files = 1000; +struct { + int count; + struct { + const char* prefix; + const char* name; + } entries[maximum_input_files]; +} input_files; // A list of input zip files. + +static bool verbose; // If true, more things are written to stdout. +static const char* output_file_name; // The name of the output zip file. +static const char* + current_input_file_name; // The name of the current input zip file (used for diagnostics). + +constexpr size_t filename_hash_table_size = 0x20000; +typedef struct { + uint32_t hash; + uint32_t len; + const uint8_t* data; +} hash_entry_t; + +// A hash set containing the name of everything so far written to the output file. +static hash_entry_t filename_hash_table[filename_hash_table_size]; + +constexpr size_t maximum_removals = 1000; +struct removal_entry { + // A removal entry can either be a literal string, or a wildcard containing a single "*". + // In the former case, the literal string is called the head. In the latter case, the + // segment before the "*" is called the head, and the segment after the "*" is called the tail. + uint32_t head_len; + uint32_t tail_len; // zero for literal removals, possibly zero for wildcard removals + const uint8_t* head; + const uint8_t* tail; // NULL for literal removals, non-NULL for wildcard removals +}; + +struct { + int count; + removal_entry entries[maximum_removals]; +} removals; // A list of files and directories to ignore in input files. + +// Sizes and signatures of zip file structures (central-directory, local-file-header, +// end-of-central-directory). +constexpr size_t cd_size = 46; +constexpr std::string_view cd_signature = "\x50\x4b\x01\x02"; +constexpr size_t lfh_size = 30; +constexpr std::string_view lfh_signature = "\x50\x4b\x03\x04"; +constexpr size_t eocd_size = 22; + +// Write the bytes [src, src + len) to the output file. +void append_data(const uint8_t* src, uint32_t len) { + if (fwrite(src, 1, len, output_zip.file) != len) { + printf("Error: Could not write %lu bytes to output file.\n", (unsigned long)len); + exit(1); + } + uint32_t new_output_size = output_zip.num_bytes_written + len; + if (new_output_size < output_zip.num_bytes_written) { + printf("Error: Output zip file exceeds 4 gigabytes.\n"); + exit(1); + } + output_zip.num_bytes_written = new_output_size; +} +} // namespace + +void append_cd(const uint8_t* src, uint32_t len) { + if ((output_cd.capacity - output_cd.length) < len) { + uint32_t new_capacity; + uint8_t* new_data; + + new_capacity = output_cd.capacity + (output_cd.capacity >> 1); + if (new_capacity < output_cd.length + len) new_capacity = output_cd.length + len; + new_data = (uint8_t*)realloc(output_cd.bytes, new_capacity); + if (!new_data) { + printf("Error: Could not grow central-directory buffer from %lu bytes to %lu bytes.\n", + (unsigned long)output_cd.capacity, (unsigned long)new_capacity); + exit(1); + } + output_cd.bytes = new_data; + output_cd.capacity = new_capacity; + } + memcpy(output_cd.bytes + output_cd.length, src, len); + output_cd.length += len; +} + +namespace { +// Copy a local-file-header and accompanying file data from an input file to the output file. +// The input file is [input_file, input_file + input_file_len). +// The offset within the input file of the local-file-header is given by lfh_offset. +// The central-directory entry corresponding to the file is given by cd. +void copy_file_data(const uint8_t* input_file, + size_t lfh_offset, + const uint8_t* cd, + size_t input_file_len) { + if (lfh_offset >= input_file_len || (size_t)(input_file_len - lfh_offset) < lfh_size) { + printf("Error: %s is invalid; central-directory references local-file-header at offset %llu, " + "but file is only %llu bytes.\n", + current_input_file_name, (unsigned long long)lfh_offset, + (unsigned long long)input_file_len); + exit(1); + } + + const uint8_t* lfh = input_file + lfh_offset; + if (memcmp(lfh, lfh_signature.data(), lfh_signature.size()) != 0) { + printf("Error: Expected local-file-header signature at offset %llu of %s, but instead got %02x " + "%02x %02x %02x.\n", + (unsigned long long)lfh_offset, current_input_file_name, lfh[0], lfh[1], lfh[2], lfh[3]); + exit(1); + } + + size_t data_offset = lfh_offset + lfh_size; + uint16_t name_len = read2(lfh + 26); + uint16_t extra_len = read2(lfh + 28); + uint32_t data_len = read4(cd + 20); + append_data(lfh, 6); // signature, version + // flags, compression, mod-time, mod-date, crc-32, compressed-size, uncompressed-size, name-len + append_data(cd + 8, 22); + append_data(lfh + 28, 2); // extra-len + + size_t total_variable_len = (size_t)name_len + (size_t)extra_len + (size_t)data_len; + if ((size_t)(input_file_len - data_offset) < total_variable_len) { + printf( + "Error: %s is invalid; starting at offset %llu, reading a filename of %u bytes, extra data " + "of %u bytes, and %lu bytes of compressed data would exceed file size of %llu bytes.\n", + current_input_file_name, (unsigned long long)data_offset, (unsigned)name_len, + (unsigned)extra_len, (unsigned long)data_len, (unsigned long long)input_file_len); + exit(1); + } + append_data(filename_prefix.bytes, filename_prefix.length); + append_data(input_file + data_offset, (uint32_t)total_variable_len); +} + +bool removal_entry_matches(const struct removal_entry* re, const uint8_t* full_name, uint32_t len) { + if (len < re->head_len + re->tail_len) { + return false; + } + if (memcmp(full_name, re->head, re->head_len) != 0) { + return false; + } + if (re->tail) { + for (uint32_t i = re->head_len + re->tail_len;; ++i) { + if (len == i || full_name[i] == '/') { + if (memcmp(full_name + i - re->tail_len, re->tail, re->tail_len) == 0) { + return true; + } + } + if (len == i || full_name[i - re->tail_len] == '/') { + return false; + } + } + } else { + return len == re->head_len || full_name[re->head_len] == '/'; + } +} +} // namespace + +bool should_include_filename_now(const uint8_t* name, uint32_t len) { + uint8_t* full_name = (uint8_t*)malloc(filename_prefix.length + len + 1); + memcpy(full_name, filename_prefix.bytes, filename_prefix.length); + memcpy(full_name + filename_prefix.length, name, len); + len += filename_prefix.length; + + for (int i = 0; i < removals.count; ++i) { + if (removal_entry_matches(&removals.entries[i], full_name, len)) { + free(full_name); + return false; + } + } + + uint32_t hash = 5381; + for (uint32_t i = 0; i < len; ++i) + hash = hash * 33 ^ full_name[i]; + + for (uint32_t idx = hash;; ++idx) { + hash_entry_t* e = filename_hash_table + (idx & (filename_hash_table_size - 1)); + if (e->hash == hash && e->len == len && memcmp(e->data, full_name, len) == 0) { + free(full_name); + return false; + } else if (e->data == NULL) { + e->hash = hash; + e->len = len; + e->data = full_name; + return true; + } + } +} + +// Try to find the end-of-central-directory record in a zip file. +const uint8_t* find_eocd(const uint8_t* input_file, size_t input_file_len) { + for (size_t i = eocd_size; i < 1024 + eocd_size && i <= input_file_len; ++i) { + const uint8_t* candidate = input_file + input_file_len - i; + if (memcmp(candidate, eocd_signature.data(), eocd_signature.size()) == 0) { + return candidate; + } + } + return NULL; +} + +namespace { +// Copy all appropriate files from an input zip to the output zip. +void process_input_file(const uint8_t* input_file, size_t input_file_len) { + const uint8_t* eocd = find_eocd(input_file, input_file_len); + if (!eocd) { + printf("Error: Could not find end-of-central-directory in %s.\n", current_input_file_name); + exit(1); + } + if (read2(eocd + 4) != 0 || read2(eocd + 6) != 0) { + printf("Error: %s is split over multiple disks, which is not supported.\n", + current_input_file_name); + exit(1); + } + if (!(uint16_t)~read2(eocd + 8) || !(uint16_t)~read2(eocd + 10) || !~read4(eocd + 12) || + !~read4(eocd + 16)) { + printf("Error: %s is zip64, which is not supported.\n", current_input_file_name); + exit(1); + } + uint16_t num_entries = read2(eocd + 10); + size_t cd_offset = read4(eocd + 16); + + for (uint16_t i = 0; i < num_entries; ++i) { + uint8_t cd[cd_size]; + if (cd_offset >= input_file_len || (size_t)(input_file_len - cd_offset) < sizeof(cd)) { + printf("Error: %s is invalid; central-directory %u/%u would start at offset %llu, but file " + "is only %llu bytes.\n", + current_input_file_name, (unsigned)i, (unsigned)num_entries, + (unsigned long long)cd_offset, (unsigned long long)input_file_len); + exit(1); + } + + memcpy(cd, input_file + cd_offset, sizeof(cd)); + if (memcmp(cd, cd_signature.data(), cd_signature.size()) != 0) { + printf("Error: Expected central-directory signature at offset %llu of %s, but instead got " + "%02x %02x %02x %02x.\n", + (unsigned long long)cd_offset, current_input_file_name, cd[0], cd[1], cd[2], cd[3]); + exit(1); + } + cd[8] &= 0xF7; // Clear the bit indicating that a local-file-footer follows the file data + cd_offset += sizeof(cd); + + uint16_t name_len = read2(cd + 28); + if (((uint32_t)name_len + (uint32_t)filename_prefix.length) > 0xFFFFU) { + printf("Error: Combining prefix of %.*s with filename of %.*s results in a filename which is " + "too long.\n", + (int)filename_prefix.length, (const char*)filename_prefix.bytes, (int)name_len, + (const char*)(input_file + cd_offset)); + exit(1); + } + write2(cd + 28, name_len + filename_prefix.length); + uint16_t extra_len = read2(cd + 30); + uint16_t comment_len = read2(cd + 32); + uint32_t offset = read4(cd + 42); + write4(cd + 42, output_zip.num_bytes_written); + if (!~offset || !~read4(cd + 20)) { + printf("Error: %s is zip64 (because of %.*s), which is not supported.\n", + current_input_file_name, (int)name_len, (const char*)(input_file + cd_offset)); + exit(1); + } + + size_t total_variable_len = (size_t)name_len + (size_t)extra_len + (size_t)comment_len; + if ((size_t)(input_file_len - cd_offset) < total_variable_len) { + printf("Error: %s is invalid; starting at offset %llu, reading a filename of %u bytes, extra " + "data of %u bytes, and comment of %u bytes exceed file size of %llu bytes.\n", + current_input_file_name, (unsigned long long)offset, (unsigned)name_len, + (unsigned)extra_len, (unsigned)comment_len, (unsigned long long)input_file_len); + exit(1); + } + + bool should_include = should_include_filename_now(input_file + cd_offset, name_len); + if (verbose) { + printf("%s %.*s from %s\n", should_include ? "Using" : "Skipping", (int)name_len, + (const char*)(input_file + cd_offset), current_input_file_name); + } + if (should_include) { + append_cd(cd, sizeof(cd)); + append_cd(filename_prefix.bytes, filename_prefix.length); + append_cd(input_file + cd_offset, (uint32_t)total_variable_len); + copy_file_data(input_file, offset, cd, input_file_len); + if (output_zip.num_files_written == 0xFFFFU) { + printf("Error: Too many files in output zip.\n"); + exit(1); + } + ++output_zip.num_files_written; + } + cd_offset += total_variable_len; + } +} + +// Read a file into memory and pass it to process_input_file. +void read_and_process_input_file(const char* filename) { +#ifdef _WIN32 + HANDLE file = CreateFileA(filename, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, + FILE_ATTRIBUTE_NORMAL, NULL); + if (file == INVALID_HANDLE_VALUE) { + printf("Error: Cannot open %s for reading.\n", filename); + exit(1); + } + LARGE_INTEGER size; + if (!GetFileSizeEx(file, &size)) { + printf("Error: Cannot determine size of %s.\n", filename); + exit(1); + } + if (size.HighPart != 0) { + printf("Error: Input file %s exceeds 4 gigabytes.\n", filename); + exit(1); + } + if (size.LowPart == 0) { + printf("Error: Input file %s is empty.\n", filename); + exit(1); + } + HANDLE mapping = CreateFileMappingA(file, NULL, PAGE_READONLY, 0, size.LowPart, NULL); + if (mapping == NULL) { + printf("Error: Cannot mmap %s (CreateFileMapping).\n", filename); + exit(1); + } + void* data = MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, size.LowPart); + if (data == NULL) { + printf("Error: Cannot mmap %s (MapViewOfFile).\n", filename); + exit(1); + } + process_input_file((uint8_t*)data, size.LowPart); + UnmapViewOfFile(data); + CloseHandle(mapping); + CloseHandle(file); +#else + int file = open(filename, O_RDONLY); + if (file == -1) { + printf("Error: Cannot open %s for reading.\n", filename); + exit(1); + } + struct stat st; + if (fstat(file, &st) == -1) { + printf("Error: Cannot stat %s.\n", filename); + exit(1); + } + void* data = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, file, 0); + if (data == MAP_FAILED) { + printf("Error: Cannot mmap %s.\n", filename); + exit(1); + } + process_input_file((uint8_t*)data, st.st_size); + munmap(data, st.st_size); + close(file); +#endif +} + +// Print usage information and exit. +void usage_and_exit(const char** argv) { + printf("Usage: %s [-v|--verbose] [--remove=FILE] outfile.zip [--prefix=PREFIX] infile1.zip " + "[--prefix=PREFIX] infile2.zip ...\n", + argv[0]); + exit(1); +} + +// Set filename_prefix based on a string from the command line. +void set_filename_prefix(const char* prefix) { + free(filename_prefix.bytes); + filename_prefix.bytes = NULL; + filename_prefix.length = 0; + + if (prefix == NULL) { + return; + } + if (*prefix == '/' || *prefix == '\\') { + ++prefix; + } + size_t len = strlen(prefix); + if (len == 0) { + return; + } + + filename_prefix.bytes = (uint8_t*)malloc(len + 1); + memcpy(filename_prefix.bytes, prefix, len); + for (size_t i = 0; i < len; ++i) { + if (filename_prefix.bytes[i] == '\\') filename_prefix.bytes[i] = '/'; + } + filename_prefix.bytes[len] = '/'; + filename_prefix.length = (uint16_t)(len + 1); +} + +// Set various global variables based on the command line. +void parse_command_line(int argc, const char** argv) { + int i = 1; + for (; i < argc; ++i) { + const char* arg = argv[i]; + if (strcmp(arg, "-v") == 0 || strcmp(arg, "--verbose") == 0) { + verbose = true; + } else if (strncmp(arg, "--remove=", 9) == 0) { + arg += 9; + if (*arg == '/' || *arg == '\\') ++arg; + if (removals.count == maximum_removals) { + printf("Error: Too many --remove flags.\n"); + exit(1); + } + const char* star = strchr(arg, '*'); + struct removal_entry* re = &removals.entries[removals.count++]; + if (star == NULL) { + re->head_len = (uint32_t)strlen(arg); + re->tail_len = 0; + re->head = (const uint8_t*)arg; + re->tail = NULL; + } else { + if (strchr(star + 1, '*')) { + printf("Error: At most one * is permitted per removal (%s).\n", arg); + exit(1); + } + re->head_len = (uint32_t)(star - arg); + re->tail_len = (uint32_t)strlen(star + 1); + re->head = (const uint8_t*)arg; + re->tail = (const uint8_t*)(star + 1); + } + ++removals.count; + } else { + break; + } + } + + if (i == argc) { + printf("Error: Missing output file name.\n"); + usage_and_exit(argv); + } + output_file_name = argv[i]; + ++i; + + const char* prefix = NULL; + for (; i < argc; ++i) { + const char* arg = argv[i]; + if (strncmp(arg, "--prefix=", 9) == 0) { + prefix = arg + 9; + } else { + if (input_files.count == maximum_input_files) { + printf("Error: Too many input files.\n"); + exit(1); + } + input_files.entries[input_files.count].prefix = prefix; + input_files.entries[input_files.count].name = arg; + ++input_files.count; + } + } + + if (input_files.count <= 0) { + printf("Error: Missing input file names.\n"); + usage_and_exit(argv); + } +} +} // namespace + +int zipmerge_main(int argc, const char** argv) { + parse_command_line(argc, argv); + + output_zip.file = fopen(output_file_name, "wb"); + if (!output_zip.file) { + printf("Error: Cannot open %s for writing.\n", output_file_name); + return 1; + } + + for (int i = input_files.count - 1; i >= 0; --i) { + set_filename_prefix(input_files.entries[i].prefix); + current_input_file_name = input_files.entries[i].name; + read_and_process_input_file(current_input_file_name); + } + + uint8_t eocd[eocd_size] = {0}; + memcpy(eocd, eocd_signature.data(), eocd_signature.size()); + write2(eocd + 8, output_zip.num_files_written); + write2(eocd + 10, output_zip.num_files_written); + write4(eocd + 12, output_cd.length); + write4(eocd + 16, output_zip.num_bytes_written); + append_data(output_cd.bytes, output_cd.length); + append_data(eocd, sizeof(eocd)); + fclose(output_zip.file); + return 0; +} + +void reset() { + memset(&output_zip, 0, sizeof(output_zip)); + memset(&filename_prefix, 0, sizeof(filename_prefix)); + memset(&output_cd, 0, sizeof(output_cd)); + memset(&input_files, 0, sizeof(input_files)); + memset(&filename_hash_table, 0, sizeof(filename_hash_table)); + memset(&removals, 0, sizeof(removals)); +} diff --git a/misc/bazel/internal/bin/zipmerge/zipmerge.h b/misc/bazel/internal/bin/zipmerge/zipmerge.h new file mode 100644 index 000000000000..096f080faf94 --- /dev/null +++ b/misc/bazel/internal/bin/zipmerge/zipmerge.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include +#include + +struct output_cd_t { + uint8_t* bytes; + uint32_t length; + uint32_t capacity; +}; + +inline output_cd_t output_cd{}; // An in-memory buffer in which the central-directory records for + // the output file are accumulated. + +// Read and write little-endian integers (as the only supported host platforms are little-endian, +// and all host platforms support unaligned memory accesses, these macros are currently very +// simple). +#define read2(ptr) (*(uint16_t*)(ptr)) +#define read4(ptr) (*(uint32_t*)(ptr)) +#define write2(ptr, val) (*(uint16_t*)(ptr) = (val)) +#define write4(ptr, val) (*(uint32_t*)(ptr) = (val)) + +// Add the bytes [src, src + len) to the output's central-directory. +void append_cd(const uint8_t* src, uint32_t len); + +// Test whether a given filename should be included in the output zip. +// Note that if a call returns true for a given filename, all future calls with the same filename +// will return false. +bool should_include_filename_now(const uint8_t* name, uint32_t len); + +inline constexpr std::string_view eocd_signature = "\x50\x4b\x05\x06"; +const uint8_t* find_eocd(const uint8_t* input_file, size_t input_file_len); + +int zipmerge_main(int argc, const char** argv); + +void reset(); diff --git a/misc/bazel/internal/bin/zipmerge/zipmerge_main.cpp b/misc/bazel/internal/bin/zipmerge/zipmerge_main.cpp new file mode 100644 index 000000000000..4aa0073756b0 --- /dev/null +++ b/misc/bazel/internal/bin/zipmerge/zipmerge_main.cpp @@ -0,0 +1,5 @@ +#include "misc/bazel/internal/bin/zipmerge/zipmerge.h" + +int main(int argc, const char** argv) { + return zipmerge_main(argc, argv); +} diff --git a/misc/bazel/internal/install.py b/misc/bazel/internal/install.py index f57cae04596c..858e8717f654 100644 --- a/misc/bazel/internal/install.py +++ b/misc/bazel/internal/install.py @@ -5,17 +5,20 @@ from python.runfiles import runfiles runfiles = runfiles.Create() -if not runfiles: - raise Exception("Installer should be run with `bazel run`") +assert runfiles, "Installer should be run with `bazel run`" parser = argparse.ArgumentParser() parser.add_argument("--destdir", type=pathlib.Path, required=True) parser.add_argument("--script", required=True) parser.add_argument("--build-file", required=True) +parser.add_argument("--ripunzip", required=True) +parser.add_argument("--zip-manifest", action="append", default=[], dest="zip_manifests") opts = parser.parse_args() -script = runfiles.Rlocation(opts.script) build_file = runfiles.Rlocation(opts.build_file) +script = runfiles.Rlocation(opts.script) +ripunzip = runfiles.Rlocation(opts.ripunzip) +zip_manifests = [runfiles.Rlocation(z) for z in opts.zip_manifests] destdir = pathlib.Path(build_file).parent / opts.destdir if destdir.exists(): @@ -23,3 +26,12 @@ destdir.mkdir(parents=True) subprocess.run([script, "--destdir", destdir], check=True) + +for zip_manifest in zip_manifests: + with open(zip_manifest) as manifest: + for line in manifest: + prefix, _, zip = line.partition(":") + assert zip, f"missing prefix for {prefix}, you should use prefix:zip format" + dest = destdir / prefix + dest.mkdir(parents=True, exist_ok=True) + subprocess.run([ripunzip, "unzip-file", zip, "-d", dest]) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 375ec779f06c..72448d3a53db 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -7,7 +7,6 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_fil load("@rules_pkg//pkg:pkg.bzl", "pkg_zip") load("@rules_pkg//pkg:providers.bzl", "PackageFilegroupInfo", "PackageFilesInfo") load("@rules_python//python:defs.bzl", "py_binary") -load("//:defs.bzl", "codeql_platform") def _make_internal(name): def internal(suffix = "internal"): @@ -22,27 +21,36 @@ def _get_subrule(label, suffix): return "%s/%s:%s-%s" % (path, pkg, pkg, suffix) _PackageFileWrapperInfo = provider(fields = {"pfi": "", "src": "", "arch_specific": ""}) +CodeqlZipInfo = provider(fields = {"prefix": "", "src": "", "arch_specific": ""}) CodeqlFilesInfo = provider( doc = """Wrapper around `rules_pkg` `PackageFilesInfo` carrying information about generic and arch-specific files.""", fields = { "files": "list of `_PackageFileWrapperInfo`.", + "zips": "list of `CodeqlPackageZipInfo`.", }, ) +_PLAT_DETECTION_ATTRS = { + "_windows": attr.label(default = "@platforms//os:windows"), + "_macos": attr.label(default = "@platforms//os:macos"), +} + +def _detect_plat(ctx): + if ctx.target_platform_has_constraint(ctx.attr._windows[platform_common.ConstraintValueInfo]): + return "windows64" + elif ctx.target_platform_has_constraint(ctx.attr._macos[platform_common.ConstraintValueInfo]): + return "osx64" + else: + return "linux64" + def _codeql_pkg_filegroup_impl(ctx): prefix = ctx.attr.prefix if prefix: prefix += "/" generic_prefix = prefix if ctx.attr.arch_specific: - if ctx.target_platform_has_constraint(ctx.attr._windows[platform_common.ConstraintValueInfo]): - plat = "windows64" - elif ctx.target_platform_has_constraint(ctx.attr._macos[platform_common.ConstraintValueInfo]): - plat = "osx64" - else: - plat = "linux64" - prefix = prefix + plat + "/" + prefix = prefix + _detect_plat(ctx) + "/" def transform_pfi(pfi, src, prefix = prefix, arch_specific = ctx.attr.arch_specific): return _PackageFileWrapperInfo( @@ -54,34 +62,45 @@ def _codeql_pkg_filegroup_impl(ctx): arch_specific = arch_specific, ) + def transform_pfwi(pfwi): + return transform_pfi( + pfwi.pfi, + pfwi.src, + # if it was already arch-specific the plat prefix was already added + generic_prefix if pfwi.arch_specific else prefix, + pfwi.arch_specific or ctx.attr.arch_specific, + ) + + def transform_czi(czi): + return CodeqlZipInfo( + # if it was already arch-specific the plat prefix was already added + prefix = (generic_prefix if czi.arch_specific else prefix) + czi.prefix, + src = czi.src, + arch_specific = czi.arch_specific or ctx.attr.arch_specific, + ) + files = [] + zips = [] for src in ctx.attr.srcs: if PackageFilesInfo in src: - pfi = src[PackageFilesInfo] - files.append(transform_pfi(pfi, src.label)) + files.append(transform_pfi(src[PackageFilesInfo], src.label)) elif PackageFilegroupInfo in src: pfgi = src[PackageFilegroupInfo] if pfgi.pkg_dirs or pfgi.pkg_symlinks: fail("while assembling %s found %s which contains `pkg_dirs` or `pkg_symlinks` targets" % (ctx.label, src.label) + ", which is not currently supported") files += [transform_pfi(pfi, src) for pfi, src in pfgi.pkg_files] + elif CodeqlZipInfo in src: + zips.append(transform_czi(src[CodeqlZipInfo])) else: - cfi = src[CodeqlFilesInfo] - files += [ - transform_pfi( - pfwi.pfi, - pfwi.src, - # if it was already arch specific the plat prefix was already added - generic_prefix if pfwi.arch_specific else prefix, - pfwi.arch_specific or ctx.attr.arch_specific, - ) - for pfwi in cfi.files - ] + files += [transform_pfwi(pfwi) for pfwi in src[CodeqlFilesInfo].files] + zips += [transform_czi(czi) for czi in src[CodeqlFilesInfo].zips] return [ CodeqlFilesInfo( files = files, + zips = zips, ), DefaultInfo( files = depset(transitive = [src[DefaultInfo].files for src in ctx.attr.srcs]), @@ -99,14 +118,13 @@ codeql_pkg_filegroup = rule( [PackageFilesInfo, DefaultInfo], [PackageFilegroupInfo, DefaultInfo], [CodeqlFilesInfo, DefaultInfo], + [CodeqlZipInfo, DefaultInfo], ], default = [], ), "prefix": attr.string(doc = "Prefix to add to the files", default = ""), "arch_specific": attr.bool(doc = "Whether the included files should be treated as arch-specific"), - "_windows": attr.label(default = "@platforms//os:windows"), - "_macos": attr.label(default = "@platforms//os:macos"), - }, + } | _PLAT_DETECTION_ATTRS, ) def codeql_pkg_files( @@ -169,6 +187,61 @@ def _extract_pkg_filegroup_impl(ctx): DefaultInfo(files = depset(transitive = files)), ] +def _codeql_pkg_zip_import_impl(ctx): + prefix = ctx.attr.prefix + if prefix: + prefix += "/" + if ctx.attr.arch_specific: + prefix += _detect_plat(ctx) + "/" + return [ + CodeqlZipInfo( + prefix = prefix, + src = ctx.file.src, + arch_specific = ctx.attr.arch_specific, + ), + DefaultInfo(files = depset([ctx.file.src])), + ] + +codeql_pkg_zip_import = rule( + implementation = _codeql_pkg_zip_import_impl, + doc = "Wrap a zip file to be consumed by `codeql_pkg_filegroup` and `codeql_pack` rules", + attrs = { + "src": attr.label(mandatory = True, allow_single_file = True, doc = "Zip file to wrap"), + "prefix": attr.string(doc = "Posix path prefix to nest the zip contents into"), + "arch_specific": attr.bool(doc = "Whether this is to be considered arch-specific"), + } | _PLAT_DETECTION_ATTRS, +) + +def _imported_zips_manifest_impl(ctx): + src = ctx.attr.src[CodeqlFilesInfo] + zips = [czi for czi in src.zips if czi.arch_specific == ctx.attr.arch_specific] + + # zipmerge is run in a build context, so it requries File.path pointers to find the zips + # installation runs in a run context, so it requries File.short_path to find the zips + # hence we require two separate files, regardless of the format + ctx.actions.write( + ctx.outputs.zipmerge_out, + "\n".join(["--prefix=%s %s" % (czi.prefix.rstrip("/"), czi.src.path) for czi in zips]), + ) + ctx.actions.write( + ctx.outputs.install_out, + "\n".join(["%s:%s" % (czi.prefix, czi.src.short_path) for czi in zips]), + ) + outputs = [ctx.outputs.zipmerge_out, ctx.outputs.install_out] + [czi.src for czi in zips] + return DefaultInfo( + files = depset(outputs), + ) + +_imported_zips_manifests = rule( + implementation = _imported_zips_manifest_impl, + attrs = { + "src": attr.label(providers = [CodeqlFilesInfo]), + "arch_specific": attr.bool(), + "zipmerge_out": attr.output(), + "install_out": attr.output(), + }, +) + _extrac_pkg_filegroup = rule( implementation = _extract_pkg_filegroup_impl, attrs = { @@ -219,11 +292,29 @@ def codeql_pack( visibility = ["//visibility:private"], ) pkg_zip( - name = internal(kind + "-zip"), + name = internal(kind + "-zip-base"), srcs = [internal(kind + "-zip-contents")], - package_file_name = zip_filename + "-" + (codeql_platform if kind == "arch" else kind) + ".zip", visibility = visibility, ) + _imported_zips_manifests( + name = internal(kind + "-zip-manifests"), + src = name, + zipmerge_out = internal(kind + "-zipmerge.params"), + install_out = internal(kind + "-install.params"), + arch_specific = kind == "arch", + ) + native.genrule( + name = internal(kind + "-zip"), + tools = ["//misc/bazel/internal/bin/zipmerge", internal(kind + "-zipmerge.params")], + srcs = [internal(kind + "-zip-base"), internal(kind + "-zip-manifests")], + outs = ["%s-%s.zip" % (zip_filename, kind)], + cmd = " ".join([ + "$(execpath //misc/bazel/internal/bin/zipmerge)", + "$@", + "$(execpath %s)" % internal(kind + "-zip-base"), + "$$(cat $(execpath %s))" % internal(kind + "-zipmerge.params"), + ]), + ) pkg_install( name = internal("script"), srcs = [internal("generic"), internal("arch")], @@ -239,13 +330,24 @@ def codeql_pack( name = internal("installer"), srcs = ["//misc/bazel/internal:install.py"], main = "//misc/bazel/internal:install.py", - data = [internal("build-file"), internal("script")], + data = [ + internal("build-file"), + internal("script"), + internal("generic-install.params"), + internal("generic-zip-manifests"), + internal("arch-install.params"), + internal("arch-zip-manifests"), + "//misc/bazel/internal/bin:ripunzip", + ], deps = ["@rules_python//python/runfiles"], args = [ "--build-file=$(rlocationpath %s)" % internal("build-file"), "--script=$(rlocationpath %s)" % internal("script"), "--destdir", install_dest, + "--ripunzip=$(rlocationpath //misc/bazel/internal/bin:ripunzip)", + "--zip-manifest=$(rlocationpath %s)" % internal("generic-install.params"), + "--zip-manifest=$(rlocationpath %s)" % internal("arch-install.params"), ], visibility = visibility, ) diff --git a/swift/BUILD.bazel b/swift/BUILD.bazel index 90bdb8515ac1..a23be9077e99 100644 --- a/swift/BUILD.bazel +++ b/swift/BUILD.bazel @@ -64,8 +64,7 @@ codeql_pkg_filegroup( codeql_pkg_filegroup( name = "resource-dir", - srcs = ["//swift/third_party/swift-llvm-support:swift-resource-dir"], - arch_specific = True, + srcs = ["//swift/third_party/resource-dir"], prefix = "resource-dir", ) diff --git a/swift/third_party/resource-dir/BUILD.bazel b/swift/third_party/resource-dir/BUILD.bazel new file mode 100644 index 000000000000..8474e13159bc --- /dev/null +++ b/swift/third_party/resource-dir/BUILD.bazel @@ -0,0 +1,9 @@ +load("//misc/bazel:pkg.bzl", "codeql_pkg_zip") + +codeql_pkg_zip( + name = "resource-dir", + src = select({"@platforms//os:" + os: "@swift-resource-dir-" + os for os in ("linux", "macos")}), + arch_specific = True, + target_compatible_with = select({"@platforms//os:windows": ["@platforms//:incompatible"]}), + visibility = ["//swift:__pkg__"], +) diff --git a/swift/third_party/resource-dir/resource-dir-linux.zip b/swift/third_party/resource-dir/resource-dir-linux.zip new file mode 100644 index 000000000000..dc52894ec90a --- /dev/null +++ b/swift/third_party/resource-dir/resource-dir-linux.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02bf1b93c60917b09e5de24f7a3a96e109337fedb7ee1cc0c2409d829866dbfe +size 190645227 diff --git a/swift/third_party/resource-dir/resource-dir-macos.zip b/swift/third_party/resource-dir/resource-dir-macos.zip new file mode 100644 index 000000000000..cfc595532191 --- /dev/null +++ b/swift/third_party/resource-dir/resource-dir-macos.zip @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e29feb39124731510535f8d98be80bc68b10ff0e791c909c9ff96a9b97391fa +size 483440694 diff --git a/swift/third_party/resource-dir/update.sh b/swift/third_party/resource-dir/update.sh new file mode 100644 index 000000000000..e69de29bb2d1 From 1529b58089679564dcbaa7cc3cba952f174ea5c8 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 24 May 2024 12:19:06 +0200 Subject: [PATCH 04/40] Swift: add resource dir updater --- .../BUILD.swift-toolchain-linux.bazel | 12 ++++++--- .../BUILD.swift-toolchain-macos.bazel | 13 ++++++++- swift/third_party/resource-dir/BUILD.bazel | 27 ++++++++++++++++--- swift/third_party/resource-dir/update.sh | 12 +++++++++ 4 files changed, 57 insertions(+), 7 deletions(-) mode change 100644 => 100755 swift/third_party/resource-dir/update.sh diff --git a/swift/third_party/BUILD.swift-toolchain-linux.bazel b/swift/third_party/BUILD.swift-toolchain-linux.bazel index be2c2e12a971..ef6d4d44be18 100644 --- a/swift/third_party/BUILD.swift-toolchain-linux.bazel +++ b/swift/third_party/BUILD.swift-toolchain-linux.bazel @@ -1,4 +1,5 @@ -load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files") +load("@rules_pkg//pkg:mappings.bzl", "pkg_files") +load("@rules_pkg//pkg:pkg.bzl", "pkg_zip") _strip_prefix = "usr/lib/swift" @@ -42,8 +43,13 @@ _pm_interface_files = [ for dir, interface, module in _pm_interface_files ] -pkg_filegroup( - name = "resource-dir", +pkg_zip( + name = "resource-dir-linux", srcs = [":resource-dir-original"] + [":pkg-%s" % module for _, _, module in _pm_interface_files], +) + +alias( + name = "swift_toolchain_linux", + actual = ":resource-dir-linux", visibility = ["//visibility:public"], ) diff --git a/swift/third_party/BUILD.swift-toolchain-macos.bazel b/swift/third_party/BUILD.swift-toolchain-macos.bazel index 38250f1d2104..461b292ddbf3 100644 --- a/swift/third_party/BUILD.swift-toolchain-macos.bazel +++ b/swift/third_party/BUILD.swift-toolchain-macos.bazel @@ -1,12 +1,23 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_files") +load("@rules_pkg//pkg:pkg.bzl", "pkg_zip") _strip_prefix = "usr/lib/swift" pkg_files( - name = "resource-dir", + name = "resource-dir-files", srcs = glob( ["usr/lib/swift/**/*"], ), strip_prefix = _strip_prefix, +) + +pkg_zip( + name = "resource-dir-macos", + srcs = [":resource-dir-files"], +) + +alias( + name = "swift_toolchain_macos", + actual = ":resource-dir-macos", visibility = ["//visibility:public"], ) diff --git a/swift/third_party/resource-dir/BUILD.bazel b/swift/third_party/resource-dir/BUILD.bazel index 8474e13159bc..5c8761fd72e7 100644 --- a/swift/third_party/resource-dir/BUILD.bazel +++ b/swift/third_party/resource-dir/BUILD.bazel @@ -1,9 +1,30 @@ -load("//misc/bazel:pkg.bzl", "codeql_pkg_zip") +load("//misc/bazel:pkg.bzl", "codeql_pkg_zip_import") -codeql_pkg_zip( +codeql_pkg_zip_import( name = "resource-dir", src = select({"@platforms//os:" + os: "@swift-resource-dir-" + os for os in ("linux", "macos")}), arch_specific = True, - target_compatible_with = select({"@platforms//os:windows": ["@platforms//:incompatible"]}), + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), visibility = ["//swift:__pkg__"], ) + +[ + sh_binary( + name = "update-" + os, + srcs = ["update.sh"], + args = [ + "$(rlocationpath @swift_toolchain_%s)" % os, + "$(rlocationpath resource-dir-%s.zip)" % os, + ], + data = [ + "resource-dir-%s.zip" % os, + "@swift_toolchain_" + os, + ], + target_compatible_with = ["@platforms//os:" + os], + deps = ["@bazel_tools//tools/bash/runfiles"], + ) + for os in ("linux", "macos") +] diff --git a/swift/third_party/resource-dir/update.sh b/swift/third_party/resource-dir/update.sh old mode 100644 new mode 100755 index e69de29bb2d1..dee216a61d67 --- a/swift/third_party/resource-dir/update.sh +++ b/swift/third_party/resource-dir/update.sh @@ -0,0 +1,12 @@ +# --- begin runfiles.bash initialization v3 --- +# Copy-pasted from the Bazel Bash runfiles library v3. +set -uo pipefail; set +e; f=bazel_tools/tools/bash/runfiles/runfiles.bash +source "${RUNFILES_DIR:-/dev/null}/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "${RUNFILES_MANIFEST_FILE:-/dev/null}" | cut -f2- -d' ')" 2>/dev/null || \ + source "$0.runfiles/$f" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + source "$(grep -sm1 "^$f " "$0.exe.runfiles_manifest" | cut -f2- -d' ')" 2>/dev/null || \ + { echo>&2 "ERROR: cannot find $f"; exit 1; }; f=; set -e +# --- end runfiles.bash initialization v3 --- + +cp "$(rlocation "$1")" "$(rlocation "$2")" From 8e132e90ccd419aeb1f653a1ca7c5f43a971d2f3 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 24 May 2024 12:35:17 +0200 Subject: [PATCH 05/40] Bazel: add executable attribute to `lfs_files` --- misc/bazel/lfs.bzl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/misc/bazel/lfs.bzl b/misc/bazel/lfs.bzl index 3a496ea9530c..689d2a07d67c 100644 --- a/misc/bazel/lfs.bzl +++ b/misc/bazel/lfs.bzl @@ -1,4 +1,4 @@ -def lfs_smudge(repository_ctx, srcs, extract = False, stripPrefix = None): +def lfs_smudge(repository_ctx, srcs, *, extract = False, stripPrefix = None, executable = False): python = repository_ctx.which("python3") or repository_ctx.which("python") if not python: fail("Neither python3 nor python executables found") @@ -25,7 +25,7 @@ def lfs_smudge(repository_ctx, srcs, extract = False, stripPrefix = None): repository_ctx.symlink(src, src.basename) else: repository_ctx.report_progress("trying cache for remote %s" % src.basename) - res = repository_ctx.download([], src.basename, sha256 = info, allow_fail = True) + res = repository_ctx.download([], src.basename, sha256 = info, allow_fail = True, executable = executable) if not res.success: remote.append(src) if remote: @@ -33,7 +33,7 @@ def lfs_smudge(repository_ctx, srcs, extract = False, stripPrefix = None): for src, info in zip(remote, infos): sha256, _, url = info.partition(" ") repository_ctx.report_progress("downloading remote %s" % src.basename) - repository_ctx.download(url, src.basename, sha256 = sha256) + repository_ctx.download(url, src.basename, sha256 = sha256, executable = executable) if extract: for src in srcs: repository_ctx.report_progress("extracting %s" % src.basename) @@ -62,7 +62,7 @@ def _download_lfs(repository_ctx): if not dir.is_dir: fail("`dir` not a directory in @%s" % repository_ctx.name) srcs = [f for f in dir.readdir() if not f.is_dir] - lfs_smudge(repository_ctx, srcs) + lfs_smudge(repository_ctx, srcs, executable = repository_ctx.attr.executable) # with bzlmod the name is qualified with `~` separators, and we want the base name here name = repository_ctx.name.split("~")[-1] @@ -98,5 +98,6 @@ lfs_files = repository_rule( "srcs": attr.label_list(doc = "Local paths to the LFS files to export."), "dir": attr.label(doc = "Local path to a directory containing LFS files to export. Only the direct contents " + "of the directory are exported"), + "executable": attr.bool(doc = "Whether files should be marked as executable"), }, ) From 94d6feffed730ce8a5e4fbece02e4d2cc6233c5c Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 24 May 2024 12:35:41 +0200 Subject: [PATCH 06/40] Swift: fix module --- MODULE.bazel | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index e7f24827c23e..fb77b0f4de60 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -62,16 +62,19 @@ lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files") lfs_files( name = "ripunzip-linux", srcs = ["//misc/bazel/internal/bin:linux/ripunzip"], + executable = True, ) lfs_files( name = "ripunzip-windows", srcs = ["//misc/bazel/internal/bin:windows/ripunzip.exe"], + executable = True, ) lfs_files( name = "ripunzip-macos", srcs = ["//misc/bazel/internal/bin:macos/ripunzip"], + executable = True, ) lfs_files( @@ -79,10 +82,10 @@ lfs_files( srcs = ["//swift/third_party/resource-dir:resource-dir-linux.zip"], ) -#lfs_files( -# name = "swift-resource-dir-macos", -# srcs = ["//swift/third_party/resource-dir:resource-dir-macos.zip"], -#) +lfs_files( + name = "swift-resource-dir-macos", + srcs = ["//swift/third_party/resource-dir:resource-dir-macos.zip"], +) register_toolchains( "@nodejs_toolchains//:all", From 175f0dbb005bee3bdf81796ad20da86c20358e57 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 24 May 2024 12:40:29 +0200 Subject: [PATCH 07/40] Swift: remove broken obsolete alias --- swift/third_party/swift-llvm-support/BUILD.bazel | 8 -------- 1 file changed, 8 deletions(-) diff --git a/swift/third_party/swift-llvm-support/BUILD.bazel b/swift/third_party/swift-llvm-support/BUILD.bazel index 183f9e7a7ff9..4bc1fffba949 100644 --- a/swift/third_party/swift-llvm-support/BUILD.bazel +++ b/swift/third_party/swift-llvm-support/BUILD.bazel @@ -7,11 +7,3 @@ alias( "@bazel_tools//src/conditions:darwin": "@swift_prebuilt_darwin_x86_64//:swift-llvm-support", }), ) - -alias( - name = "swift-resource-dir", - actual = select({ - "@bazel_tools//src/conditions:linux": "@swift_toolchain_linux//:resource-dir", - "@bazel_tools//src/conditions:darwin": "@swift_toolchain_macos//:resource-dir", - }), -) From e694968012ceb3e595b71420719fb3e8b9181226 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 24 May 2024 12:41:32 +0200 Subject: [PATCH 08/40] Fix change to `.gitattributes` done by mistake --- .gitattributes | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index d2cf32fa7100..d24ae61b9a83 100644 --- a/.gitattributes +++ b/.gitattributes @@ -77,7 +77,7 @@ ruby/extractor/cargo-bazel-lock.json -merge # auto-generated files for the C# build csharp/paket.lock linguist-generated=true # needs eol=crlf, as `paket` touches this file and saves it als crlf -csharp/.paket/Paket.Restore.targets linguist-generated=true eol=cr +csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf # ripunzip tool /misc/bazel/internal/bin/*/ripunzip* filter=lfs diff=lfs merge=lfs -text From dcbf42d29cb32abb8af85b0ee7e701196b5333ac Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 24 May 2024 13:06:44 +0200 Subject: [PATCH 09/40] Bazel: reorganize LFS files and add licensing information --- .gitattributes | 2 +- MODULE.bazel | 6 +- .../internal/{bin => ripunzip}/BUILD.bazel | 0 misc/bazel/internal/ripunzip/LICENSE.txt | 236 ++++++++++++++++++ misc/bazel/internal/ripunzip/README.md | 2 + .../ripunzip => ripunzip/ripunzip-linux} | 0 .../ripunzip => ripunzip/ripunzip-macos} | 0 .../ripunzip-windows.exe} | 0 .../internal/{bin => }/zipmerge/BUILD.bazel | 0 .../internal/{bin => }/zipmerge/zipmerge.cpp | 2 +- .../internal/{bin => }/zipmerge/zipmerge.h | 0 .../{bin => }/zipmerge/zipmerge_main.cpp | 2 +- misc/bazel/lfs.bzl | 17 +- misc/bazel/pkg.bzl | 8 +- swift/third_party/resource-dir/LICENSE.txt | 211 ++++++++++++++++ swift/third_party/resource-dir/README.md | 2 + 16 files changed, 470 insertions(+), 18 deletions(-) rename misc/bazel/internal/{bin => ripunzip}/BUILD.bazel (100%) create mode 100644 misc/bazel/internal/ripunzip/LICENSE.txt create mode 100644 misc/bazel/internal/ripunzip/README.md rename misc/bazel/internal/{bin/linux/ripunzip => ripunzip/ripunzip-linux} (100%) rename misc/bazel/internal/{bin/macos/ripunzip => ripunzip/ripunzip-macos} (100%) rename misc/bazel/internal/{bin/windows/ripunzip.exe => ripunzip/ripunzip-windows.exe} (100%) rename misc/bazel/internal/{bin => }/zipmerge/BUILD.bazel (100%) rename misc/bazel/internal/{bin => }/zipmerge/zipmerge.cpp (99%) rename misc/bazel/internal/{bin => }/zipmerge/zipmerge.h (100%) rename misc/bazel/internal/{bin => }/zipmerge/zipmerge_main.cpp (58%) create mode 100644 swift/third_party/resource-dir/LICENSE.txt create mode 100644 swift/third_party/resource-dir/README.md diff --git a/.gitattributes b/.gitattributes index d24ae61b9a83..2c2f788de9a0 100644 --- a/.gitattributes +++ b/.gitattributes @@ -80,7 +80,7 @@ csharp/paket.lock linguist-generated=true csharp/.paket/Paket.Restore.targets linguist-generated=true eol=crlf # ripunzip tool -/misc/bazel/internal/bin/*/ripunzip* filter=lfs diff=lfs merge=lfs -text +/misc/bazel/internal/ripunzip/ripunzip-* filter=lfs diff=lfs merge=lfs -text # swift prebuilt resources /swift/third_party/resource-dir/*.zip filter=lfs diff=lfs merge=lfs -text diff --git a/MODULE.bazel b/MODULE.bazel index fb77b0f4de60..db39b358eb16 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -61,19 +61,19 @@ lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files") lfs_files( name = "ripunzip-linux", - srcs = ["//misc/bazel/internal/bin:linux/ripunzip"], + srcs = ["//misc/bazel/internal/ripunzip:ripunzip-linux"], executable = True, ) lfs_files( name = "ripunzip-windows", - srcs = ["//misc/bazel/internal/bin:windows/ripunzip.exe"], + srcs = ["//misc/bazel/internal/ripunzip:ripunzip-windows.exe"], executable = True, ) lfs_files( name = "ripunzip-macos", - srcs = ["//misc/bazel/internal/bin:macos/ripunzip"], + srcs = ["//misc/bazel/internal/ripunzip:ripunzip-macos"], executable = True, ) diff --git a/misc/bazel/internal/bin/BUILD.bazel b/misc/bazel/internal/ripunzip/BUILD.bazel similarity index 100% rename from misc/bazel/internal/bin/BUILD.bazel rename to misc/bazel/internal/ripunzip/BUILD.bazel diff --git a/misc/bazel/internal/ripunzip/LICENSE.txt b/misc/bazel/internal/ripunzip/LICENSE.txt new file mode 100644 index 000000000000..7cb2473fa896 --- /dev/null +++ b/misc/bazel/internal/ripunzip/LICENSE.txt @@ -0,0 +1,236 @@ +This software is distributed under the terms of both the MIT license and the +Apache License (Version 2.0). + + +MIT license + +Copyright 2022 Google LLC + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. + + +Apache 2 license + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/misc/bazel/internal/ripunzip/README.md b/misc/bazel/internal/ripunzip/README.md new file mode 100644 index 000000000000..90bfbe516d7d --- /dev/null +++ b/misc/bazel/internal/ripunzip/README.md @@ -0,0 +1,2 @@ +These LFS files are distributions of [ripunzip](https://github.com/google/ripunzip), compiled locally with +`cargo install ripunzip`. A [copy](./LICENSE.txt) of the ripunzip license is included. diff --git a/misc/bazel/internal/bin/linux/ripunzip b/misc/bazel/internal/ripunzip/ripunzip-linux similarity index 100% rename from misc/bazel/internal/bin/linux/ripunzip rename to misc/bazel/internal/ripunzip/ripunzip-linux diff --git a/misc/bazel/internal/bin/macos/ripunzip b/misc/bazel/internal/ripunzip/ripunzip-macos similarity index 100% rename from misc/bazel/internal/bin/macos/ripunzip rename to misc/bazel/internal/ripunzip/ripunzip-macos diff --git a/misc/bazel/internal/bin/windows/ripunzip.exe b/misc/bazel/internal/ripunzip/ripunzip-windows.exe similarity index 100% rename from misc/bazel/internal/bin/windows/ripunzip.exe rename to misc/bazel/internal/ripunzip/ripunzip-windows.exe diff --git a/misc/bazel/internal/bin/zipmerge/BUILD.bazel b/misc/bazel/internal/zipmerge/BUILD.bazel similarity index 100% rename from misc/bazel/internal/bin/zipmerge/BUILD.bazel rename to misc/bazel/internal/zipmerge/BUILD.bazel diff --git a/misc/bazel/internal/bin/zipmerge/zipmerge.cpp b/misc/bazel/internal/zipmerge/zipmerge.cpp similarity index 99% rename from misc/bazel/internal/bin/zipmerge/zipmerge.cpp rename to misc/bazel/internal/zipmerge/zipmerge.cpp index 216c415e8581..6a663ad7bbc5 100644 --- a/misc/bazel/internal/bin/zipmerge/zipmerge.cpp +++ b/misc/bazel/internal/zipmerge/zipmerge.cpp @@ -16,7 +16,7 @@ Filename adjustment consists of optionally prepending a prefix to the filename. */ -#include "misc/bazel/internal/bin/zipmerge/zipmerge.h" +#include "misc/bazel/internal/zipmerge/zipmerge.h" #include #include diff --git a/misc/bazel/internal/bin/zipmerge/zipmerge.h b/misc/bazel/internal/zipmerge/zipmerge.h similarity index 100% rename from misc/bazel/internal/bin/zipmerge/zipmerge.h rename to misc/bazel/internal/zipmerge/zipmerge.h diff --git a/misc/bazel/internal/bin/zipmerge/zipmerge_main.cpp b/misc/bazel/internal/zipmerge/zipmerge_main.cpp similarity index 58% rename from misc/bazel/internal/bin/zipmerge/zipmerge_main.cpp rename to misc/bazel/internal/zipmerge/zipmerge_main.cpp index 4aa0073756b0..eeb9133d9c90 100644 --- a/misc/bazel/internal/bin/zipmerge/zipmerge_main.cpp +++ b/misc/bazel/internal/zipmerge/zipmerge_main.cpp @@ -1,4 +1,4 @@ -#include "misc/bazel/internal/bin/zipmerge/zipmerge.h" +#include "misc/bazel/internal/zipmerge/zipmerge.h" int main(int argc, const char** argv) { return zipmerge_main(argc, argv); diff --git a/misc/bazel/lfs.bzl b/misc/bazel/lfs.bzl index 689d2a07d67c..a068d76b2eae 100644 --- a/misc/bazel/lfs.bzl +++ b/misc/bazel/lfs.bzl @@ -66,15 +66,16 @@ def _download_lfs(repository_ctx): # with bzlmod the name is qualified with `~` separators, and we want the base name here name = repository_ctx.name.split("~")[-1] - repository_ctx.file("BUILD.bazel", """ -exports_files({files}) + basenames = [src.basename for src in srcs] + build = "exports_files(%s)\n" % repr(basenames) -filegroup( - name = "{name}", - srcs = {files}, - visibility = ["//visibility:public"], -) -""".format(name = name, files = repr([src.basename for src in srcs]))) + # add a main `name` filegroup only if it doesn't conflict with existing exported files + if name not in basenames: + build += 'filegroup(name = "%s", srcs = %s, visibility = ["//visibility:public"])\n' % ( + name, + basenames, + ) + repository_ctx.file("BUILD.bazel", build) lfs_archive = repository_rule( doc = "Export the contents from an on-demand LFS archive. The corresponding path should be added to be ignored " + diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 72448d3a53db..972ab082df3a 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -305,11 +305,11 @@ def codeql_pack( ) native.genrule( name = internal(kind + "-zip"), - tools = ["//misc/bazel/internal/bin/zipmerge", internal(kind + "-zipmerge.params")], + tools = ["//misc/bazel/internal/zipmerge", internal(kind + "-zipmerge.params")], srcs = [internal(kind + "-zip-base"), internal(kind + "-zip-manifests")], outs = ["%s-%s.zip" % (zip_filename, kind)], cmd = " ".join([ - "$(execpath //misc/bazel/internal/bin/zipmerge)", + "$(execpath //misc/bazel/internal/zipmerge)", "$@", "$(execpath %s)" % internal(kind + "-zip-base"), "$$(cat $(execpath %s))" % internal(kind + "-zipmerge.params"), @@ -337,7 +337,7 @@ def codeql_pack( internal("generic-zip-manifests"), internal("arch-install.params"), internal("arch-zip-manifests"), - "//misc/bazel/internal/bin:ripunzip", + "//misc/bazel/internal/ripunzip", ], deps = ["@rules_python//python/runfiles"], args = [ @@ -345,7 +345,7 @@ def codeql_pack( "--script=$(rlocationpath %s)" % internal("script"), "--destdir", install_dest, - "--ripunzip=$(rlocationpath //misc/bazel/internal/bin:ripunzip)", + "--ripunzip=$(rlocationpath //misc/bazel/internal/ripunzip)", "--zip-manifest=$(rlocationpath %s)" % internal("generic-install.params"), "--zip-manifest=$(rlocationpath %s)" % internal("arch-install.params"), ], diff --git a/swift/third_party/resource-dir/LICENSE.txt b/swift/third_party/resource-dir/LICENSE.txt new file mode 100644 index 000000000000..61b0c78195f2 --- /dev/null +++ b/swift/third_party/resource-dir/LICENSE.txt @@ -0,0 +1,211 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +## Runtime Library Exception to the Apache 2.0 License: ## + + + As an exception, if you use this Software to compile your source code and + portions of this Software are embedded into the binary product as a result, + you may redistribute such product without providing attribution as would + otherwise be required by Sections 4(a), 4(b) and 4(d) of the License. diff --git a/swift/third_party/resource-dir/README.md b/swift/third_party/resource-dir/README.md new file mode 100644 index 000000000000..38873b4a54ff --- /dev/null +++ b/swift/third_party/resource-dir/README.md @@ -0,0 +1,2 @@ +These LFS files are redistributed parts of the [Swift toolchains](https://www.swift.org/download/). +A [copy](./LICENSE.txt) of the [swift](https://github.com/apple/swift) license is included. From fa2c626e49f5b708b291e5a9c12ea762b46892e9 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 24 May 2024 13:46:00 +0200 Subject: [PATCH 10/40] Bazel: add fat macOS ripunzip binary --- misc/bazel/internal/ripunzip/ripunzip-macos | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/bazel/internal/ripunzip/ripunzip-macos b/misc/bazel/internal/ripunzip/ripunzip-macos index 6ac67e6f0161..d832ddf4f078 100755 --- a/misc/bazel/internal/ripunzip/ripunzip-macos +++ b/misc/bazel/internal/ripunzip/ripunzip-macos @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:91e21ae4c75fd79dee4afc16aa79cef5a404080a4784360923edd23b2b2e988d -size 5316808 +oid sha256:334def0f75bb12442e989230e6d7c5830c78c684278dc84d8baf4ff81e7929a4 +size 10854600 From ea01ae65347aca6d0310ca0d5844b44adb8bea35 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 24 May 2024 14:14:50 +0200 Subject: [PATCH 11/40] Swift: fix integration test log upload --- swift/actions/run-integration-tests/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/swift/actions/run-integration-tests/action.yml b/swift/actions/run-integration-tests/action.yml index fe5a20b02bd8..0efd2371b960 100644 --- a/swift/actions/run-integration-tests/action.yml +++ b/swift/actions/run-integration-tests/action.yml @@ -27,5 +27,5 @@ runs: with: name: swift-integration-tests-logs-${{ runner.os }} path: | - swift/ql/integration-tests/**/db/log + swift/ql/integration-tests/**/*db/log retention-days: 1 From e990d75a8f7a2d0b4e4ade9e85888f4a9304f790 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 24 May 2024 15:23:51 +0200 Subject: [PATCH 12/40] Bazel: use codeql platform as arch zip filename --- misc/bazel/pkg.bzl | 102 +++++++++++++++++++++++++++++---------------- 1 file changed, 66 insertions(+), 36 deletions(-) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 972ab082df3a..567b6ab44e4d 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -216,32 +216,66 @@ def _imported_zips_manifest_impl(ctx): src = ctx.attr.src[CodeqlFilesInfo] zips = [czi for czi in src.zips if czi.arch_specific == ctx.attr.arch_specific] - # zipmerge is run in a build context, so it requries File.path pointers to find the zips - # installation runs in a run context, so it requries File.short_path to find the zips - # hence we require two separate files, regardless of the format + output = ctx.actions.declare_file(ctx.label.name + ".params") ctx.actions.write( - ctx.outputs.zipmerge_out, - "\n".join(["--prefix=%s %s" % (czi.prefix.rstrip("/"), czi.src.path) for czi in zips]), - ) - ctx.actions.write( - ctx.outputs.install_out, + output, "\n".join(["%s:%s" % (czi.prefix, czi.src.short_path) for czi in zips]), ) - outputs = [ctx.outputs.zipmerge_out, ctx.outputs.install_out] + [czi.src for czi in zips] return DefaultInfo( - files = depset(outputs), + files = depset([output]), + runfiles = ctx.runfiles([czi.src for czi in zips]), ) -_imported_zips_manifests = rule( +_imported_zips_manifest = rule( implementation = _imported_zips_manifest_impl, attrs = { "src": attr.label(providers = [CodeqlFilesInfo]), "arch_specific": attr.bool(), - "zipmerge_out": attr.output(), - "install_out": attr.output(), + "zip_prefix": attr.string(), }, ) +def _zipmerge_impl(ctx): + src = ctx.attr.src[CodeqlFilesInfo] + zip_infos = [czi for czi in src.zips if czi.arch_specific == ctx.attr.arch_specific] + zips = depset([ctx.file.base_zip] + [czi.src for czi in zip_infos]) + filename = ctx.attr.zip_name + "-" + if ctx.attr.arch_specific: + filename += _detect_plat(ctx) + else: + filename += "generic" + filename += ".zip" + output = ctx.actions.declare_file(filename) + args = [output.path, ctx.file.base_zip.path] + for info in zip_infos: + args += [ + "--prefix=%s/%s" % (ctx.attr.zip_prefix, info.prefix.rstrip("/")), + info.src.path, + ] + + ctx.actions.run( + outputs = [output], + executable = ctx.executable._zipmerge, + inputs = zips, + arguments = args, + ) + + return [ + DefaultInfo(files = depset([output])), + ] + +_zipmerge = rule( + implementation = _zipmerge_impl, + attrs = { + "src": attr.label(providers = [CodeqlFilesInfo]), + "base_zip": attr.label(allow_single_file = True), + "zip_name": attr.string(), + "arch_specific": attr.bool(), + "zip_prefix": attr.string(), + "_zipmerge": attr.label(default = "//misc/bazel/internal/zipmerge", executable = True, cfg = "exec"), + } | _PLAT_DETECTION_ATTRS, +) + _extrac_pkg_filegroup = rule( implementation = _extract_pkg_filegroup_impl, attrs = { @@ -291,30 +325,28 @@ def codeql_pack( prefix = zip_prefix, visibility = ["//visibility:private"], ) + _imported_zips_manifest( + name = internal(kind + "-zip-manifest"), + src = name, + arch_specific = kind == "arch", + zip_prefix = zip_prefix, + visibility = ["//visibility:private"], + ) pkg_zip( name = internal(kind + "-zip-base"), srcs = [internal(kind + "-zip-contents")], - visibility = visibility, + visibility = ["//visibility:private"], ) - _imported_zips_manifests( - name = internal(kind + "-zip-manifests"), + _zipmerge( + name = internal(kind + "-zip"), + base_zip = internal(kind + "-zip-base"), + zip_name = zip_filename, + zip_prefix = zip_prefix, src = name, - zipmerge_out = internal(kind + "-zipmerge.params"), - install_out = internal(kind + "-install.params"), arch_specific = kind == "arch", + visibility = visibility, ) - native.genrule( - name = internal(kind + "-zip"), - tools = ["//misc/bazel/internal/zipmerge", internal(kind + "-zipmerge.params")], - srcs = [internal(kind + "-zip-base"), internal(kind + "-zip-manifests")], - outs = ["%s-%s.zip" % (zip_filename, kind)], - cmd = " ".join([ - "$(execpath //misc/bazel/internal/zipmerge)", - "$@", - "$(execpath %s)" % internal(kind + "-zip-base"), - "$$(cat $(execpath %s))" % internal(kind + "-zipmerge.params"), - ]), - ) + pkg_install( name = internal("script"), srcs = [internal("generic"), internal("arch")], @@ -333,10 +365,8 @@ def codeql_pack( data = [ internal("build-file"), internal("script"), - internal("generic-install.params"), - internal("generic-zip-manifests"), - internal("arch-install.params"), - internal("arch-zip-manifests"), + internal("generic-zip-manifest"), + internal("arch-zip-manifest"), "//misc/bazel/internal/ripunzip", ], deps = ["@rules_python//python/runfiles"], @@ -346,8 +376,8 @@ def codeql_pack( "--destdir", install_dest, "--ripunzip=$(rlocationpath //misc/bazel/internal/ripunzip)", - "--zip-manifest=$(rlocationpath %s)" % internal("generic-install.params"), - "--zip-manifest=$(rlocationpath %s)" % internal("arch-install.params"), + "--zip-manifest=$(rlocationpath %s)" % internal("generic-zip-manifest"), + "--zip-manifest=$(rlocationpath %s)" % internal("arch-zip-manifest"), ], visibility = visibility, ) From f35f077c76c8f5abdd506ded48c6f7ea769a1d16 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 24 May 2024 15:24:15 +0200 Subject: [PATCH 13/40] Swift: cleanup tools scripts in pack --- swift/tools/BUILD.bazel | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/swift/tools/BUILD.bazel b/swift/tools/BUILD.bazel index ae04a587a944..777b96490685 100644 --- a/swift/tools/BUILD.bazel +++ b/swift/tools/BUILD.bazel @@ -6,16 +6,6 @@ sh_binary( visibility = ["//swift/tools/test/qltest:__pkg__"], ) -sh_binary( - name = "autobuild", - srcs = ["autobuild.sh"], -) - -sh_binary( - name = "identify-environment", - srcs = ["identify-environment.sh"], -) - codeql_pkg_files( name = "tools", srcs = [ @@ -23,9 +13,9 @@ codeql_pkg_files( "tracing-config.lua", ], exes = [ - ":autobuild", - ":identify-environment", - ":qltest", + "autobuild.sh", + "identify-environment.sh", + "qltest.sh", ], visibility = ["//swift:__pkg__"], ) From b9064c5446ab60cb075ea73cacfbb62b829fc438 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 24 May 2024 15:50:16 +0200 Subject: [PATCH 14/40] Bazel: fail install on `ripunzip` failing --- misc/bazel/internal/install.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/bazel/internal/install.py b/misc/bazel/internal/install.py index 858e8717f654..9032f0831bc3 100644 --- a/misc/bazel/internal/install.py +++ b/misc/bazel/internal/install.py @@ -34,4 +34,4 @@ assert zip, f"missing prefix for {prefix}, you should use prefix:zip format" dest = destdir / prefix dest.mkdir(parents=True, exist_ok=True) - subprocess.run([ripunzip, "unzip-file", zip, "-d", dest]) + subprocess.run([ripunzip, "unzip-file", zip, "-d", dest], check=True) From 5d4b61c365d95a2d8bf29ea41b350e22e85f22dd Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 24 May 2024 17:43:52 +0200 Subject: [PATCH 15/40] Bazel: replace prebuilt ripunzip from workflow --- misc/bazel/internal/ripunzip/README.md | 4 ++-- misc/bazel/internal/ripunzip/ripunzip-linux | 4 ++-- misc/bazel/internal/ripunzip/ripunzip-macos | 4 ++-- misc/bazel/internal/ripunzip/ripunzip-windows.exe | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) mode change 100644 => 100755 misc/bazel/internal/ripunzip/ripunzip-windows.exe diff --git a/misc/bazel/internal/ripunzip/README.md b/misc/bazel/internal/ripunzip/README.md index 90bfbe516d7d..4312f00a9b0b 100644 --- a/misc/bazel/internal/ripunzip/README.md +++ b/misc/bazel/internal/ripunzip/README.md @@ -1,2 +1,2 @@ -These LFS files are distributions of [ripunzip](https://github.com/google/ripunzip), compiled locally with -`cargo install ripunzip`. A [copy](./LICENSE.txt) of the ripunzip license is included. +These LFS files are distributions of [ripunzip](https://github.com/google/ripunzip), compiled with this [workflow](https://github.com/github/codeql/actions/workflows/build-ripunzip.yml). +A [copy](./LICENSE.txt) of the ripunzip license is included. diff --git a/misc/bazel/internal/ripunzip/ripunzip-linux b/misc/bazel/internal/ripunzip/ripunzip-linux index 151e21ddff3f..356063894609 100755 --- a/misc/bazel/internal/ripunzip/ripunzip-linux +++ b/misc/bazel/internal/ripunzip/ripunzip-linux @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:609d90e14e1cdd0351eee3d45a6e09e36cc44b3289e912a19945a49f4fdc416d -size 5775960 +oid sha256:b5e444b6efcb11e899ff932dc5846927dd78578d0889386d82aa21133e077fde +size 12423064 diff --git a/misc/bazel/internal/ripunzip/ripunzip-macos b/misc/bazel/internal/ripunzip/ripunzip-macos index d832ddf4f078..d80eeea06670 100755 --- a/misc/bazel/internal/ripunzip/ripunzip-macos +++ b/misc/bazel/internal/ripunzip/ripunzip-macos @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:334def0f75bb12442e989230e6d7c5830c78c684278dc84d8baf4ff81e7929a4 -size 10854600 +oid sha256:e8ff604d47ec88c4a795d307dee9454771589e8bd0b9747c6f49d2a59081f829 +size 10632454 diff --git a/misc/bazel/internal/ripunzip/ripunzip-windows.exe b/misc/bazel/internal/ripunzip/ripunzip-windows.exe old mode 100644 new mode 100755 index bae1e9566149..44727f650dbd --- a/misc/bazel/internal/ripunzip/ripunzip-windows.exe +++ b/misc/bazel/internal/ripunzip/ripunzip-windows.exe @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:7623346ec339ab7bf4179297dc129dcedeed932c49e594e6cfddfd1c9c2ff11c -size 4527104 +oid sha256:6e6b68c668a84d1232335524f9ca15dff61f7365ec16d57caa9763fda145f33d +size 4548096 From 0b7a4257d76b01f218560af4af5609d74491bf2b Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 27 May 2024 10:31:19 +0200 Subject: [PATCH 16/40] Bazel: use `{CODEQL_PLATFORM}` as discriminant between arch and generic contents --- misc/bazel/pkg.bzl | 303 +++++++-------------- swift/BUILD.bazel | 19 +- swift/extractor/BUILD.bazel | 1 - swift/third_party/resource-dir/BUILD.bazel | 7 +- 4 files changed, 107 insertions(+), 223 deletions(-) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 567b6ab44e4d..cff037a55851 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -14,28 +14,19 @@ def _make_internal(name): return internal -def _get_subrule(label, suffix): - if ":" in label or "/" not in label: - return "%s-%s" % (label, suffix) - path, _, pkg = label.rpartition("/") - return "%s/%s:%s-%s" % (path, pkg, pkg, suffix) - -_PackageFileWrapperInfo = provider(fields = {"pfi": "", "src": "", "arch_specific": ""}) -CodeqlZipInfo = provider(fields = {"prefix": "", "src": "", "arch_specific": ""}) - -CodeqlFilesInfo = provider( - doc = """Wrapper around `rules_pkg` `PackageFilesInfo` carrying information about generic and arch-specific files.""", - fields = { - "files": "list of `_PackageFileWrapperInfo`.", - "zips": "list of `CodeqlPackageZipInfo`.", - }, -) - _PLAT_DETECTION_ATTRS = { "_windows": attr.label(default = "@platforms//os:windows"), "_macos": attr.label(default = "@platforms//os:macos"), } +_PLAT_PLACEHOLDER = "{CODEQL_PLATFORM}" + +def _process_path(path, plat): + if _PLAT_PLACEHOLDER in path: + path = path.replace(_PLAT_PLACEHOLDER, plat) + return ("arch", path) + return ("generic", path) + def _detect_plat(ctx): if ctx.target_platform_has_constraint(ctx.attr._windows[platform_common.ConstraintValueInfo]): return "windows64" @@ -44,107 +35,20 @@ def _detect_plat(ctx): else: return "linux64" -def _codeql_pkg_filegroup_impl(ctx): - prefix = ctx.attr.prefix - if prefix: - prefix += "/" - generic_prefix = prefix - if ctx.attr.arch_specific: - prefix = prefix + _detect_plat(ctx) + "/" - - def transform_pfi(pfi, src, prefix = prefix, arch_specific = ctx.attr.arch_specific): - return _PackageFileWrapperInfo( - pfi = PackageFilesInfo( - attributes = pfi.attributes, - dest_src_map = {prefix + d: s for d, s in pfi.dest_src_map.items()}, - ), - src = src, - arch_specific = arch_specific, - ) - - def transform_pfwi(pfwi): - return transform_pfi( - pfwi.pfi, - pfwi.src, - # if it was already arch-specific the plat prefix was already added - generic_prefix if pfwi.arch_specific else prefix, - pfwi.arch_specific or ctx.attr.arch_specific, - ) - - def transform_czi(czi): - return CodeqlZipInfo( - # if it was already arch-specific the plat prefix was already added - prefix = (generic_prefix if czi.arch_specific else prefix) + czi.prefix, - src = czi.src, - arch_specific = czi.arch_specific or ctx.attr.arch_specific, - ) - - files = [] - zips = [] - - for src in ctx.attr.srcs: - if PackageFilesInfo in src: - files.append(transform_pfi(src[PackageFilesInfo], src.label)) - elif PackageFilegroupInfo in src: - pfgi = src[PackageFilegroupInfo] - if pfgi.pkg_dirs or pfgi.pkg_symlinks: - fail("while assembling %s found %s which contains `pkg_dirs` or `pkg_symlinks` targets" % - (ctx.label, src.label) + ", which is not currently supported") - files += [transform_pfi(pfi, src) for pfi, src in pfgi.pkg_files] - elif CodeqlZipInfo in src: - zips.append(transform_czi(src[CodeqlZipInfo])) - else: - files += [transform_pfwi(pfwi) for pfwi in src[CodeqlFilesInfo].files] - zips += [transform_czi(czi) for czi in src[CodeqlFilesInfo].zips] - - return [ - CodeqlFilesInfo( - files = files, - zips = zips, - ), - DefaultInfo( - files = depset(transitive = [src[DefaultInfo].files for src in ctx.attr.srcs]), - ), - ] - -codeql_pkg_filegroup = rule( - implementation = _codeql_pkg_filegroup_impl, - doc = """CodeQL specific packaging mapping. No `pkg_mkdirs` or `pkg_symlink` rules are supported, either directly - or transitively. Only `pkg_files` and `pkg_filegroup` thereof are allowed.""", - attrs = { - "srcs": attr.label_list( - doc = "List of arch-agnostic `pkg_files`, `pkg_filegroup` or `codeql_pkg_filegroup` targets", - providers = [ - [PackageFilesInfo, DefaultInfo], - [PackageFilegroupInfo, DefaultInfo], - [CodeqlFilesInfo, DefaultInfo], - [CodeqlZipInfo, DefaultInfo], - ], - default = [], - ), - "prefix": attr.string(doc = "Prefix to add to the files", default = ""), - "arch_specific": attr.bool(doc = "Whether the included files should be treated as arch-specific"), - } | _PLAT_DETECTION_ATTRS, -) - def codeql_pkg_files( *, name, srcs = None, exes = None, - arch_specific = False, - prefix = None, visibility = None, **kwargs): + """ Wrapper around `pkg_files` adding a distinction between `srcs` and `exes`, where the + latter will get executable permissions. """ - Wrapper around `pkg_files`. Added functionality: - * `exes` will get their file attributes set to be executable. This is important only for POSIX files, there's no - need to mark windows executables as such - If `exes` and `srcs` are both used, the resulting rule is a `pkg_filegroup` one. - """ + internal = _make_internal(name) if "attributes" in kwargs: - fail("codeql_pkg_files does not support `attributes`. Use `exes` to mark executable files.") + fail("do not use attributes with codeql_pkg_* rules. Use `exes` to mark executable files.") internal_srcs = [] if srcs and exes: pkg_files( @@ -160,103 +64,98 @@ def codeql_pkg_files( attributes = pkg_attributes(mode = "755"), **kwargs ) - internal_srcs = [internal("srcs"), internal("exes")] + pkg_filegroup( + name = name, + srcs = [internal("srcs"), internal("exes")], + visibility = visibility, + ) else: pkg_files( - name = internal(), + name = name, srcs = srcs or exes, visibility = visibility, attributes = pkg_attributes(mode = "755") if exes else None, **kwargs ) - internal_srcs = [internal()] - codeql_pkg_filegroup( - name = name, - srcs = internal_srcs, - arch_specific = arch_specific, - prefix = prefix, - visibility = visibility, - ) def _extract_pkg_filegroup_impl(ctx): - src = ctx.attr.src[CodeqlFilesInfo] - pfi_lbls = [(pfwi.pfi, pfwi.src) for pfwi in src.files if pfwi.arch_specific == ctx.attr.arch_specific] - files = [depset(pfi.dest_src_map.values()) for pfi, _ in pfi_lbls] + src = ctx.attr.src[PackageFilegroupInfo] + plat = _detect_plat(ctx) + + if src.pkg_dirs or src.pkg_symlinks: + fail("`pkg_dirs` and `pkg_symlinks` are not supported for codeql packaging rules") + + pkg_files = [] + for pfi, origin in src.pkg_files: + dest_src_map = {} + for dest, file in pfi.dest_src_map.items(): + file_kind, dest = _process_path(dest, plat) + if file_kind == ctx.attr.kind: + dest_src_map[dest] = file + if dest_src_map: + pkg_files.append((PackageFilesInfo(dest_src_map = dest_src_map, attributes = pfi.attributes), origin)) + + files = [depset(pfi.dest_src_map.values()) for pfi, _ in pkg_files] return [ - PackageFilegroupInfo(pkg_files = pfi_lbls, pkg_dirs = [], pkg_symlinks = []), + PackageFilegroupInfo(pkg_files = pkg_files, pkg_dirs = [], pkg_symlinks = []), DefaultInfo(files = depset(transitive = files)), ] -def _codeql_pkg_zip_import_impl(ctx): - prefix = ctx.attr.prefix - if prefix: - prefix += "/" - if ctx.attr.arch_specific: - prefix += _detect_plat(ctx) + "/" - return [ - CodeqlZipInfo( - prefix = prefix, - src = ctx.file.src, - arch_specific = ctx.attr.arch_specific, - ), - DefaultInfo(files = depset([ctx.file.src])), - ] - -codeql_pkg_zip_import = rule( - implementation = _codeql_pkg_zip_import_impl, - doc = "Wrap a zip file to be consumed by `codeql_pkg_filegroup` and `codeql_pack` rules", +_extrac_pkg_filegroup = rule( + implementation = _extract_pkg_filegroup_impl, attrs = { - "src": attr.label(mandatory = True, allow_single_file = True, doc = "Zip file to wrap"), - "prefix": attr.string(doc = "Posix path prefix to nest the zip contents into"), - "arch_specific": attr.bool(doc = "Whether this is to be considered arch-specific"), + "src": attr.label(providers = [PackageFilegroupInfo, DefaultInfo]), + "kind": attr.string(doc = "generic or arch", values = ["generic", "arch"]), } | _PLAT_DETECTION_ATTRS, ) def _imported_zips_manifest_impl(ctx): - src = ctx.attr.src[CodeqlFilesInfo] - zips = [czi for czi in src.zips if czi.arch_specific == ctx.attr.arch_specific] + plat = _detect_plat(ctx) + + manifest = [] + files = [] + for zip, prefix in ctx.attr.zips.items(): + zip_kind, prefix = _process_path(prefix, plat) + if zip_kind == ctx.attr.kind: + zip_files = zip.files.to_list() + manifest += ["%s:%s" % (prefix, f.short_path) for f in zip_files] + files += zip_files output = ctx.actions.declare_file(ctx.label.name + ".params") ctx.actions.write( output, - "\n".join(["%s:%s" % (czi.prefix, czi.src.short_path) for czi in zips]), + "\n".join(manifest), ) return DefaultInfo( files = depset([output]), - runfiles = ctx.runfiles([czi.src for czi in zips]), + runfiles = ctx.runfiles(files), ) _imported_zips_manifest = rule( implementation = _imported_zips_manifest_impl, attrs = { - "src": attr.label(providers = [CodeqlFilesInfo]), - "arch_specific": attr.bool(), - "zip_prefix": attr.string(), - }, + "zips": attr.label_keyed_string_dict(allow_files = True), + "kind": attr.string(doc = "generic or arch", values = ["generic", "arch"]), + } | _PLAT_DETECTION_ATTRS, ) def _zipmerge_impl(ctx): - src = ctx.attr.src[CodeqlFilesInfo] - zip_infos = [czi for czi in src.zips if czi.arch_specific == ctx.attr.arch_specific] - zips = depset([ctx.file.base_zip] + [czi.src for czi in zip_infos]) + zips = [] filename = ctx.attr.zip_name + "-" - if ctx.attr.arch_specific: - filename += _detect_plat(ctx) - else: - filename += "generic" - filename += ".zip" + plat = _detect_plat(ctx) + filename = "%s-%s.zip" % (ctx.attr.zip_name, plat if ctx.attr.kind == "arch" else "generic") output = ctx.actions.declare_file(filename) - args = [output.path, ctx.file.base_zip.path] - for info in zip_infos: - args += [ - "--prefix=%s/%s" % (ctx.attr.zip_prefix, info.prefix.rstrip("/")), - info.src.path, - ] - + args = [output.path, "--prefix=%s" % ctx.attr.zip_prefix, ctx.file.base.path] + for zip, prefix in ctx.attr.zips.items(): + zip_kind, prefix = _process_path(prefix, plat) + if zip_kind == ctx.attr.kind: + args.append("--prefix=%s/%s" % (ctx.attr.zip_prefix, prefix.rstrip("/"))) + args += [f.path for f in zip.files.to_list()] + zips.append(zip.files) ctx.actions.run( outputs = [output], executable = ctx.executable._zipmerge, - inputs = zips, + inputs = depset([ctx.file.base], transitive = zips), arguments = args, ) @@ -267,85 +166,75 @@ def _zipmerge_impl(ctx): _zipmerge = rule( implementation = _zipmerge_impl, attrs = { - "src": attr.label(providers = [CodeqlFilesInfo]), - "base_zip": attr.label(allow_single_file = True), + "base": attr.label(allow_single_file = True), + "zips": attr.label_keyed_string_dict(allow_files = True), "zip_name": attr.string(), - "arch_specific": attr.bool(), + "kind": attr.string(doc = "generic or arch", values = ["generic", "arch"]), "zip_prefix": attr.string(), "_zipmerge": attr.label(default = "//misc/bazel/internal/zipmerge", executable = True, cfg = "exec"), } | _PLAT_DETECTION_ATTRS, ) -_extrac_pkg_filegroup = rule( - implementation = _extract_pkg_filegroup_impl, - attrs = { - "src": attr.label(providers = [CodeqlFilesInfo, DefaultInfo]), - "arch_specific": attr.bool(), - }, -) - def codeql_pack( *, name, srcs = None, - zip_prefix = None, + zips = None, zip_filename = "extractor", visibility = None, install_dest = "extractor-pack", **kwargs): """ - Define a codeql pack. This accepts `pkg_files`, `pkg_filegroup` or their `codeql_*` counterparts as `srcs`. + Define a codeql pack. This macro accepts `pkg_files`, `pkg_filegroup` or their `codeql_*` counterparts as `srcs`. + `zips` is a map from prefixes to `.zip` files to import. * defines a `-generic-zip` target creating a `-generic.zip` archive with the generic bits, - prefixed with `zip_prefix` (`name` by default) + prefixed with `name` * defines a `-arch-zip` target creating a `-.zip` archive with the arch-specific bits, prefixed with `zip_prefix` (`name` by default) * defines a runnable `-installer` target that will install the pack in `install_dest`, relative to where the rule is used. The install destination can be overridden appending `-- --destdir=...` to the `bazel run` invocation. This installation does not use the `zip_prefix`. + + The distinction between arch-specific and generic contents is made based on whether the paths (including possible + prefixes added by rules) contain the special `{CODEQL_PLATFORM}` placeholder, which in case it is present will also + be replaced by the appropriate platform (`linux64`, `windows64` or `osx64`). """ internal = _make_internal(name) - zip_prefix = zip_prefix or name zip_filename = zip_filename or name - codeql_pkg_filegroup( - name = name, + zips = zips or {} + pkg_filegroup( + name = internal("base"), srcs = srcs, - visibility = visibility, + visibility = ["//visibility:private"], **kwargs ) for kind in ("generic", "arch"): _extrac_pkg_filegroup( name = internal(kind), - src = name, - arch_specific = kind == "arch", - visibility = ["//visibility:private"], - ) - pkg_filegroup( - name = internal(kind + "-zip-contents"), - srcs = [internal(kind)], - prefix = zip_prefix, - visibility = ["//visibility:private"], - ) - _imported_zips_manifest( - name = internal(kind + "-zip-manifest"), - src = name, - arch_specific = kind == "arch", - zip_prefix = zip_prefix, + src = internal("base"), + kind = kind, visibility = ["//visibility:private"], ) pkg_zip( name = internal(kind + "-zip-base"), - srcs = [internal(kind + "-zip-contents")], + srcs = [internal(kind)], visibility = ["//visibility:private"], ) _zipmerge( name = internal(kind + "-zip"), - base_zip = internal(kind + "-zip-base"), + base = internal(kind + "-zip-base"), + zips = zips, zip_name = zip_filename, - zip_prefix = zip_prefix, - src = name, - arch_specific = kind == "arch", + zip_prefix = name, + kind = kind, visibility = visibility, ) + _imported_zips_manifest( + name = internal(kind + "-zip-manifest"), + zips = zips, + kind = kind, + visibility = ["//visibility:private"], + ) pkg_install( name = internal("script"), @@ -381,6 +270,10 @@ def codeql_pack( ], visibility = visibility, ) + native.filegroup( + name = name, + srcs = [internal("generic-zip"), internal("arch-zip")], + ) strip_prefix = _strip_prefix diff --git a/swift/BUILD.bazel b/swift/BUILD.bazel index a23be9077e99..2ceecf079c50 100644 --- a/swift/BUILD.bazel +++ b/swift/BUILD.bazel @@ -1,7 +1,7 @@ +load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup") load( "//misc/bazel:pkg.bzl", "codeql_pack", - "codeql_pkg_filegroup", "codeql_pkg_files", "codeql_pkg_runfiles", ) @@ -31,11 +31,10 @@ codeql_pkg_files( codeql_pkg_runfiles( name = "autobuilder", - arch_specific = True, exes = ["//swift/swift-autobuilder"], ) -codeql_pkg_filegroup( +pkg_filegroup( name = "tools-arch", srcs = select({ "@platforms//os:macos": [ @@ -50,10 +49,10 @@ codeql_pkg_filegroup( ":autobuilder-incompatible-os", ], }), - arch_specific = True, + prefix = "{CODEQL_PLATFORM}", ) -codeql_pkg_filegroup( +pkg_filegroup( name = "tools", srcs = [ ":tools-arch", @@ -62,12 +61,6 @@ codeql_pkg_filegroup( prefix = "tools", ) -codeql_pkg_filegroup( - name = "resource-dir", - srcs = ["//swift/third_party/resource-dir"], - prefix = "resource-dir", -) - codeql_pkg_files( name = "root-files", srcs = [ @@ -80,12 +73,14 @@ codeql_pkg_files( codeql_pack( name = "swift", srcs = [ - ":resource-dir", ":root-files", ":tools", "//swift/downgrades", ], visibility = ["//visibility:public"], + zips = { + "//swift/third_party/resource-dir": "resource-dir/{CODEQL_PLATFORM}", + }, ) alias( diff --git a/swift/extractor/BUILD.bazel b/swift/extractor/BUILD.bazel index 02b1a91e3f9a..8290aec41216 100644 --- a/swift/extractor/BUILD.bazel +++ b/swift/extractor/BUILD.bazel @@ -31,7 +31,6 @@ sh_binary( codeql_pkg_runfiles( name = "pkg", - arch_specific = True, excludes = ["extractor.sh"], # script gets copied as "extractor", no need for the original .sh file exes = [":extractor"], visibility = ["//swift:__pkg__"], diff --git a/swift/third_party/resource-dir/BUILD.bazel b/swift/third_party/resource-dir/BUILD.bazel index 5c8761fd72e7..0b4fdbf24df6 100644 --- a/swift/third_party/resource-dir/BUILD.bazel +++ b/swift/third_party/resource-dir/BUILD.bazel @@ -1,9 +1,6 @@ -load("//misc/bazel:pkg.bzl", "codeql_pkg_zip_import") - -codeql_pkg_zip_import( +alias( name = "resource-dir", - src = select({"@platforms//os:" + os: "@swift-resource-dir-" + os for os in ("linux", "macos")}), - arch_specific = True, + actual = select({"@platforms//os:" + os: "@swift-resource-dir-" + os for os in ("linux", "macos")}), target_compatible_with = select({ "@platforms//os:windows": ["@platforms//:incompatible"], "//conditions:default": [], From 546d644765413360df6c8ca71099a463ed120b14 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 27 May 2024 10:46:50 +0200 Subject: [PATCH 17/40] Swift: do not use `codeql_pkg_files` needlessly --- swift/downgrades/BUILD.bazel | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/swift/downgrades/BUILD.bazel b/swift/downgrades/BUILD.bazel index 7f5f6fe4dd6e..5f643aea185d 100644 --- a/swift/downgrades/BUILD.bazel +++ b/swift/downgrades/BUILD.bazel @@ -1,6 +1,6 @@ -load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix") +load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") -codeql_pkg_files( +pkg_files( name = "downgrades", srcs = glob( ["**"], From 6bbad226722b14d9821dacf15025333aaa841d74 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 27 May 2024 16:05:38 +0200 Subject: [PATCH 18/40] Codegen: make codegen work on windows --- misc/codegen/loaders/dbschemeloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/codegen/loaders/dbschemeloader.py b/misc/codegen/loaders/dbschemeloader.py index 51e362362a77..f6fbab50499c 100644 --- a/misc/codegen/loaders/dbschemeloader.py +++ b/misc/codegen/loaders/dbschemeloader.py @@ -33,7 +33,7 @@ def _get_table(match): name=match["table"], columns=[_get_column(f) for f in _Re.field.finditer(match["tablebody"])], keyset=keyset, - dir=pathlib.PosixPath(match["tabledir"]) if match["tabledir"] else None, + dir=pathlib.PurePosixPath(match["tabledir"]) if match["tabledir"] else None, ) From 2f53c0e6408860628191f986adee5117e06913c3 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 27 May 2024 16:09:42 +0200 Subject: [PATCH 19/40] Bazel: fix `codeql_pack` installation on Windows --- .bazelrc | 3 +++ misc/bazel/internal/install.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.bazelrc b/.bazelrc index b034ae8bb869..3035b0beb395 100644 --- a/.bazelrc +++ b/.bazelrc @@ -19,6 +19,9 @@ build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor --host_cxxopt=/std:c startup --windows_enable_symlinks common --enable_runfiles +# with the above, we can avoid building python zips which is the default on windows as that's expensive +build --nobuild_python_zip + common --registry=file:///%workspace%/misc/bazel/registry common --registry=https://bcr.bazel.build diff --git a/misc/bazel/internal/install.py b/misc/bazel/internal/install.py index 9032f0831bc3..efcd07bfbc8c 100644 --- a/misc/bazel/internal/install.py +++ b/misc/bazel/internal/install.py @@ -19,7 +19,7 @@ script = runfiles.Rlocation(opts.script) ripunzip = runfiles.Rlocation(opts.ripunzip) zip_manifests = [runfiles.Rlocation(z) for z in opts.zip_manifests] -destdir = pathlib.Path(build_file).parent / opts.destdir +destdir = pathlib.Path(build_file).resolve().parent / opts.destdir if destdir.exists(): shutil.rmtree(destdir) From 2f95944244ec7251cb30e85751d9f6db1273f3aa Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 27 May 2024 16:15:03 +0200 Subject: [PATCH 20/40] Bazel: add documentation to `install.py` --- misc/bazel/internal/install.py | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/misc/bazel/internal/install.py b/misc/bazel/internal/install.py index efcd07bfbc8c..d609b4fe7810 100644 --- a/misc/bazel/internal/install.py +++ b/misc/bazel/internal/install.py @@ -1,3 +1,12 @@ +""" +Helper script for installing `codeql_pack` targets. + +This mainly wraps around a `pkg_install` script from `rules_pkg` adding: +* resolving destination directory with respect to a provided `--build-file` +* clean-up of target destination directory before a reinstall +* installing imported zip files using a provided `--ripunzip` +""" + import argparse import pathlib import shutil @@ -7,12 +16,17 @@ runfiles = runfiles.Create() assert runfiles, "Installer should be run with `bazel run`" -parser = argparse.ArgumentParser() -parser.add_argument("--destdir", type=pathlib.Path, required=True) -parser.add_argument("--script", required=True) -parser.add_argument("--build-file", required=True) -parser.add_argument("--ripunzip", required=True) -parser.add_argument("--zip-manifest", action="append", default=[], dest="zip_manifests") +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument("--destdir", type=pathlib.Path, required=True, + help="Desination directory, relative to `--build-file`") +parser.add_argument("--script", required=True, + help="The wrapped `pkg_install` installation script rlocation") +parser.add_argument("--build-file", required=True, + help="BUILD.bazel rlocation relative to which the installation should take place") +parser.add_argument("--ripunzip", required=True, + help="ripunzip executable rlocation") +parser.add_argument("--zip-manifest", action="append", default=[], dest="zip_manifests", + help="The rlocation of a file containing newline-separated `prefix:zip_file` entries") opts = parser.parse_args() build_file = runfiles.Rlocation(opts.build_file) From cde71a915b693f3995f94828c6d7f1725a0e0e99 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 27 May 2024 17:00:02 +0200 Subject: [PATCH 21/40] Bazel: address review comments --- misc/bazel/internal/install.py | 23 ++++---- misc/bazel/pkg.bzl | 98 ++++++++++++++++++++-------------- 2 files changed, 70 insertions(+), 51 deletions(-) diff --git a/misc/bazel/internal/install.py b/misc/bazel/internal/install.py index d609b4fe7810..fafa901de7ea 100644 --- a/misc/bazel/internal/install.py +++ b/misc/bazel/internal/install.py @@ -19,20 +19,20 @@ parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--destdir", type=pathlib.Path, required=True, help="Desination directory, relative to `--build-file`") -parser.add_argument("--script", required=True, +parser.add_argument("--pkg-install-script", required=True, help="The wrapped `pkg_install` installation script rlocation") parser.add_argument("--build-file", required=True, help="BUILD.bazel rlocation relative to which the installation should take place") parser.add_argument("--ripunzip", required=True, help="ripunzip executable rlocation") -parser.add_argument("--zip-manifest", action="append", default=[], dest="zip_manifests", +parser.add_argument("--zip-manifest", required=True, help="The rlocation of a file containing newline-separated `prefix:zip_file` entries") opts = parser.parse_args() build_file = runfiles.Rlocation(opts.build_file) -script = runfiles.Rlocation(opts.script) +script = runfiles.Rlocation(opts.pkg_install_script) ripunzip = runfiles.Rlocation(opts.ripunzip) -zip_manifests = [runfiles.Rlocation(z) for z in opts.zip_manifests] +zip_manifest = runfiles.Rlocation(opts.zip_manifest) destdir = pathlib.Path(build_file).resolve().parent / opts.destdir if destdir.exists(): @@ -41,11 +41,10 @@ destdir.mkdir(parents=True) subprocess.run([script, "--destdir", destdir], check=True) -for zip_manifest in zip_manifests: - with open(zip_manifest) as manifest: - for line in manifest: - prefix, _, zip = line.partition(":") - assert zip, f"missing prefix for {prefix}, you should use prefix:zip format" - dest = destdir / prefix - dest.mkdir(parents=True, exist_ok=True) - subprocess.run([ripunzip, "unzip-file", zip, "-d", dest], check=True) +with open(zip_manifest) as manifest: + for line in manifest: + prefix, _, zip = line.partition(":") + assert zip, f"missing prefix for {prefix}, you should use prefix:zip format" + dest = destdir / prefix + dest.mkdir(parents=True, exist_ok=True) + subprocess.run([ripunzip, "unzip-file", zip, "-d", dest], check=True) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 71a6d35c1946..5eb5d0354c6d 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -21,15 +21,15 @@ _PLAT_DETECTION_ATTRS = { _PLAT_PLACEHOLDER = "{CODEQL_PLATFORM}" -def _process_path(path, plat): +def _process_path(path, platform): if _PLAT_PLACEHOLDER in path: - path = path.replace(_PLAT_PLACEHOLDER, plat) + path = path.replace(_PLAT_PLACEHOLDER, platform) return ("arch", path) return ("generic", path) -def _detect_plat(ctx): +def _detect_platform(ctx): if ctx.target_platform_has_constraint(ctx.attr._windows[platform_common.ConstraintValueInfo]): - return "windows64" + return "win64" elif ctx.target_platform_has_constraint(ctx.attr._macos[platform_common.ConstraintValueInfo]): return "osx64" else: @@ -80,7 +80,7 @@ def codeql_pkg_files( def _extract_pkg_filegroup_impl(ctx): src = ctx.attr.src[PackageFilegroupInfo] - plat = _detect_plat(ctx) + platform = _detect_platform(ctx) if src.pkg_dirs or src.pkg_symlinks: fail("`pkg_dirs` and `pkg_symlinks` are not supported for codeql packaging rules") @@ -89,7 +89,7 @@ def _extract_pkg_filegroup_impl(ctx): for pfi, origin in src.pkg_files: dest_src_map = {} for dest, file in pfi.dest_src_map.items(): - file_kind, dest = _process_path(dest, plat) + file_kind, dest = _process_path(dest, platform) if file_kind == ctx.attr.kind: dest_src_map[dest] = file if dest_src_map: @@ -101,25 +101,33 @@ def _extract_pkg_filegroup_impl(ctx): DefaultInfo(files = depset(transitive = files)), ] -_extrac_pkg_filegroup = rule( +_extract_pkg_filegroup = rule( implementation = _extract_pkg_filegroup_impl, + doc = """ + This internal rule extracts the arch or generic part of a `PackageFilegroupInfo` source, returning a + `PackageFilegroupInfo` that is a subset of the provided `src`, while expanding `{CODEQL_PLATFORM}` in + destination paths to the relevant codeql platform (linux64, win64 or osx64). + The distinction between generic and arch contents is given on a per-file basis depending on the install path + containing {CODEQL_PLATFORM}, which will typically have been added by a `prefix` attribute to a `pkg_*` rule. + No `pkg_dirs` or `pkg_symlink` must have been used for assembling the source mapping information: we could + easily add support for that, but we don't require it for now. + """, attrs = { "src": attr.label(providers = [PackageFilegroupInfo, DefaultInfo]), - "kind": attr.string(doc = "generic or arch", values = ["generic", "arch"]), + "kind": attr.string(doc = "What part to extract", values = ["generic", "arch"]), } | _PLAT_DETECTION_ATTRS, ) def _imported_zips_manifest_impl(ctx): - plat = _detect_plat(ctx) + platform = _detect_platform(ctx) manifest = [] files = [] for zip, prefix in ctx.attr.zips.items(): - zip_kind, prefix = _process_path(prefix, plat) - if zip_kind == ctx.attr.kind: - zip_files = zip.files.to_list() - manifest += ["%s:%s" % (prefix, f.short_path) for f in zip_files] - files += zip_files + _, prefix = _process_path(prefix, platform) + zip_files = zip.files.to_list() + manifest += ["%s:%s" % (prefix, f.short_path) for f in zip_files] + files += zip_files output = ctx.actions.declare_file(ctx.label.name + ".params") ctx.actions.write( @@ -133,21 +141,24 @@ def _imported_zips_manifest_impl(ctx): _imported_zips_manifest = rule( implementation = _imported_zips_manifest_impl, + doc = """ + This internal rule prints a zip manifest file that `misc/bazel/internal/install.py` understands. + {CODEQL_PLATFORM} can be used as zip prefixes and will be expanded to the relevant codeql platform. + """, attrs = { - "zips": attr.label_keyed_string_dict(allow_files = True), - "kind": attr.string(doc = "generic or arch", values = ["generic", "arch"]), + "zips": attr.label_keyed_string_dict(doc = "mapping from zip files to install prefixes", allow_files = True), } | _PLAT_DETECTION_ATTRS, ) def _zipmerge_impl(ctx): zips = [] filename = ctx.attr.zip_name + "-" - plat = _detect_plat(ctx) - filename = "%s-%s.zip" % (ctx.attr.zip_name, plat if ctx.attr.kind == "arch" else "generic") + platform = _detect_platform(ctx) + filename = "%s-%s.zip" % (ctx.attr.zip_name, platform if ctx.attr.kind == "arch" else "generic") output = ctx.actions.declare_file(filename) args = [output.path, "--prefix=%s" % ctx.attr.zip_prefix, ctx.file.base.path] for zip, prefix in ctx.attr.zips.items(): - zip_kind, prefix = _process_path(prefix, plat) + zip_kind, prefix = _process_path(prefix, platform) if zip_kind == ctx.attr.kind: args.append("--prefix=%s/%s" % (ctx.attr.zip_prefix, prefix.rstrip("/"))) args += [f.path for f in zip.files.to_list()] @@ -165,12 +176,24 @@ def _zipmerge_impl(ctx): _zipmerge = rule( implementation = _zipmerge_impl, + doc = """ + This internal rule merges a `base` zip file with the ones indicated by the `zips` mapping where the prefix + indicates a matching kind between arch and generic. An imported zip file will be considered arch-specific + if its prefix contains `{CODEQL_PLATFORM}` (and this prefix will have that expanded to the appropriate + platform). + + The output filename will be either `{zip_name}-generic.zip` or `{zip_name}-{CODEQL_PLATFORM}.zip`, depending on + the requested `kind`. + """, attrs = { - "base": attr.label(allow_single_file = True), - "zips": attr.label_keyed_string_dict(allow_files = True), - "zip_name": attr.string(), - "kind": attr.string(doc = "generic or arch", values = ["generic", "arch"]), - "zip_prefix": attr.string(), + "base": attr.label( + doc = "Base zip file to which zips from `zips` will be merged with", + allow_single_file = True, + ), + "zips": attr.label_keyed_string_dict(doc = "mapping from zip files to install prefixes", allow_files = True), + "zip_name": attr.string(doc = "Prefix to use for the output file name"), + "kind": attr.string(doc = "Which zip kind to consider", values = ["generic", "arch"]), + "zip_prefix": attr.string(doc = "Prefix posix path to add to the zip contents in the archive"), "_zipmerge": attr.label(default = "//misc/bazel/internal/zipmerge", executable = True, cfg = "exec"), } | _PLAT_DETECTION_ATTRS, ) @@ -190,14 +213,14 @@ def codeql_pack( * defines a `-generic-zip` target creating a `-generic.zip` archive with the generic bits, prefixed with `name` * defines a `-arch-zip` target creating a `-.zip` archive with the - arch-specific bits, prefixed with `zip_prefix` (`name` by default) + arch-specific bits, prefixed with `name` * defines a runnable `-installer` target that will install the pack in `install_dest`, relative to where the rule is used. The install destination can be overridden appending `-- --destdir=...` to the `bazel run` - invocation. This installation does not use the `zip_prefix`. + invocation. This installation _does not_ prefix the contents with `name`. The distinction between arch-specific and generic contents is made based on whether the paths (including possible prefixes added by rules) contain the special `{CODEQL_PLATFORM}` placeholder, which in case it is present will also - be replaced by the appropriate platform (`linux64`, `windows64` or `osx64`). + be replaced by the appropriate platform (`linux64`, `win64` or `osx64`). """ internal = _make_internal(name) zip_filename = zip_filename or name @@ -209,7 +232,7 @@ def codeql_pack( **kwargs ) for kind in ("generic", "arch"): - _extrac_pkg_filegroup( + _extract_pkg_filegroup( name = internal(kind), src = internal("base"), kind = kind, @@ -229,12 +252,11 @@ def codeql_pack( kind = kind, visibility = visibility, ) - _imported_zips_manifest( - name = internal(kind + "-zip-manifest"), - zips = zips, - kind = kind, - visibility = ["//visibility:private"], - ) + _imported_zips_manifest( + name = internal("zip-manifest"), + zips = zips, + visibility = ["//visibility:private"], + ) pkg_install( name = internal("script"), @@ -254,19 +276,17 @@ def codeql_pack( data = [ internal("build-file"), internal("script"), - internal("generic-zip-manifest"), - internal("arch-zip-manifest"), + internal("zip-manifest"), "//misc/bazel/internal/ripunzip", ], deps = ["@rules_python//python/runfiles"], args = [ "--build-file=$(rlocationpath %s)" % internal("build-file"), - "--script=$(rlocationpath %s)" % internal("script"), + "--pkg-install-script=$(rlocationpath %s)" % internal("script"), "--destdir", install_dest, "--ripunzip=$(rlocationpath //misc/bazel/internal/ripunzip)", - "--zip-manifest=$(rlocationpath %s)" % internal("generic-zip-manifest"), - "--zip-manifest=$(rlocationpath %s)" % internal("arch-zip-manifest"), + "--zip-manifest=$(rlocationpath %s)" % internal("zip-manifest"), ], visibility = visibility, ) From 392ef09d196259c9b20633a9082c08f6d41d3aa8 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 27 May 2024 17:28:33 +0200 Subject: [PATCH 22/40] Zipmerge: make lib public for internal testing --- misc/bazel/internal/zipmerge/BUILD.bazel | 3 +++ 1 file changed, 3 insertions(+) diff --git a/misc/bazel/internal/zipmerge/BUILD.bazel b/misc/bazel/internal/zipmerge/BUILD.bazel index 6aa11d4fe505..332d74ecab55 100644 --- a/misc/bazel/internal/zipmerge/BUILD.bazel +++ b/misc/bazel/internal/zipmerge/BUILD.bazel @@ -4,6 +4,9 @@ cc_library( "zipmerge.cpp", ], hdrs = ["zipmerge.h"], + visibility = ["//visibility:public"], + # this is to make internal repo be able to keep on testing this code + # before we fully port tests here ) cc_binary( From f7bfe435c8e44bd11e7a8c067d25772fc8b9f953 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 11:20:32 +0200 Subject: [PATCH 23/40] Swift: fix windows build again --- swift/BUILD.bazel | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/swift/BUILD.bazel b/swift/BUILD.bazel index 2ceecf079c50..09f7e22ffe04 100644 --- a/swift/BUILD.bazel +++ b/swift/BUILD.bazel @@ -78,9 +78,12 @@ codeql_pack( "//swift/downgrades", ], visibility = ["//visibility:public"], - zips = { - "//swift/third_party/resource-dir": "resource-dir/{CODEQL_PLATFORM}", - }, + zips = select({ + "@platforms//os:windows": {}, + "//conditions:default": { + "//swift/third_party/resource-dir": "resource-dir/{CODEQL_PLATFORM}", + }, + }), ) alias( From a8543d4a8876e190fae2841494164b7565b17edd Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 11:59:19 +0200 Subject: [PATCH 24/40] Zipmerge: port tests from internal repo --- .github/workflows/zipmerge-test.yml | 23 +++ MODULE.bazel | 1 + misc/bazel/internal/zipmerge/BUILD.bazel | 13 +- .../zipmerge/test-files/CPython-partial.zip | Bin 0 -> 3909 bytes .../internal/zipmerge/test-files/CPython.zip | Bin 0 -> 9481 bytes .../zipmerge/test-files/almost-minimal.zip | Bin 0 -> 163 bytes .../internal/zipmerge/test-files/empty.zip | Bin 0 -> 22 bytes .../zipmerge/test-files/minimal-x3.zip | Bin 0 -> 414 bytes .../internal/zipmerge/test-files/minimal.zip | Bin 0 -> 150 bytes .../slf4j-api-classes-with-footers.jar | Bin 0 -> 341 bytes .../slf4j-api-classes-without-footers.jar | Bin 0 -> 309 bytes .../bazel/internal/zipmerge/zipmerge_test.cpp | 155 ++++++++++++++++++ 12 files changed, 188 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/zipmerge-test.yml create mode 100644 misc/bazel/internal/zipmerge/test-files/CPython-partial.zip create mode 100644 misc/bazel/internal/zipmerge/test-files/CPython.zip create mode 100644 misc/bazel/internal/zipmerge/test-files/almost-minimal.zip create mode 100644 misc/bazel/internal/zipmerge/test-files/empty.zip create mode 100644 misc/bazel/internal/zipmerge/test-files/minimal-x3.zip create mode 100644 misc/bazel/internal/zipmerge/test-files/minimal.zip create mode 100644 misc/bazel/internal/zipmerge/test-files/slf4j-api-classes-with-footers.jar create mode 100644 misc/bazel/internal/zipmerge/test-files/slf4j-api-classes-without-footers.jar create mode 100644 misc/bazel/internal/zipmerge/zipmerge_test.cpp diff --git a/.github/workflows/zipmerge-test.yml b/.github/workflows/zipmerge-test.yml new file mode 100644 index 000000000000..8492b045c048 --- /dev/null +++ b/.github/workflows/zipmerge-test.yml @@ -0,0 +1,23 @@ +name: "Test zipmerge code" + +on: + pull_request: + paths: + - "misc/bazel/internal/zipmerge/**" + - "MODULE.bazel" + - ".bazelrc*" + branches: + - main + - "rc/*" + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - run: | + bazel test //misc/bazel/internal/zipmerge:test diff --git a/MODULE.bazel b/MODULE.bazel index f9ffd739b467..dff694ce0d86 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -24,6 +24,7 @@ bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") bazel_dep(name = "fmt", version = "10.0.0") bazel_dep(name = "gazelle", version = "0.36.0") bazel_dep(name = "rules_dotnet", version = "0.15.1") +bazel_dep(name = "googletest", version = "1.14.0.bcr.1") bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True) diff --git a/misc/bazel/internal/zipmerge/BUILD.bazel b/misc/bazel/internal/zipmerge/BUILD.bazel index 332d74ecab55..bb944cc03432 100644 --- a/misc/bazel/internal/zipmerge/BUILD.bazel +++ b/misc/bazel/internal/zipmerge/BUILD.bazel @@ -4,9 +4,6 @@ cc_library( "zipmerge.cpp", ], hdrs = ["zipmerge.h"], - visibility = ["//visibility:public"], - # this is to make internal repo be able to keep on testing this code - # before we fully port tests here ) cc_binary( @@ -20,4 +17,12 @@ cc_binary( ], ) -#TODO port tests from internal repo +cc_test( + name = "test", + size = "small", + deps = [ + ":lib", + "@bazel_tools//tools/cpp/runfiles", + "@googletest//:gtest_main", + ], +) diff --git a/misc/bazel/internal/zipmerge/test-files/CPython-partial.zip b/misc/bazel/internal/zipmerge/test-files/CPython-partial.zip new file mode 100644 index 0000000000000000000000000000000000000000..a5a30ec3f90b6ba00898f0b123b7d14592da00e4 GIT binary patch literal 3909 zcmZvf2{=@38^=fXHOnAbvu|0lWl5HhZOG1e6Jt^s#K=;JB>ND?*vS^6NKy>hvJ}~u zVeH#r>{~@XExwoD&biL~&zb9)=Xd?@=REg4x1lZ}5fJd>7zvKl_<8gDgX~b%4SnFZ zTwTNse~%*jIZ9!QKkFh10N}|90Q~sv(I{1S2SZzT52)B3XXic(uxpP5&|eD^Ai<*6 zszAZ&%Ei%E8Lk^!Sq-wBbPkjbvtzarJV=9ZDn_T}Y_(}{W4hVqtd#@i*aX(p!E|;9d#_IgO+FCMi>RA|1$Uvu)j1nJSzCv`W`GLlnu;DB0 zd%_Mqx^0&msZ$$n`7*wm)9|EYcAX-oCd7?f`;u`R$=Hh<)7>}tW>8A*{|}hqo>|_D z`LE!<9A|NZ(E7U%mHj7Ar!Vp!ob}LgZ{hL2XC@PYsueYUS7w=ClzN9rxksp@h2%Tu z*87;K3P~*#{?j$tEgg-DhihaD7x<{5hV6jmoX5ne)(jabRGeJwTO4`D0{%n?u^+BX6u`!_0wVW{v ztEB3=mc*~*rXP}FDc#Ocs@4x9McZ#!DWN9oH@lEB;pbn8GUIM|#(J#4QzI@zos(e> z@;fv$YT|-B(?MnAdjj7UhO0&U5AY_go_W4*grHf!HHc+%EAzn{joyP|ucqjTEa1FX zww%uwD4zK&Z1X5B-F;QGP_b|ymY(5a0Bw(r&D+=$fk)+gU#!mQu2fe=BjP3GY28Ce# zcD6_!;s?QkWZo<5(D5ocLPFK0I!X(=+f&WGUaooDl#2E9MwnlIIVg3XQS9AYfDATzD^{Zwkr8Zc!Rxt>2FUhaE zqntEjsv~^d6gPGkoE~ZMU9&SAn+LHpYU&GD3f;E2>W5qawWtfsPGnH}2B4mLevH8h zzbJo>DCvq(?200m--}ZPHQS(e-N$zL7!4K3Fzf^yxRJ41P~+`vnPBNnVwr>ty$wAx zt(=-i{~VlM&`2n$%gsAa)p8}e&VaU#DKR6HADq78z0yqx+G>Zu z@voXD{Vv+D>_V}cU6BJZu$!^hKaPcUYe;kr%2BK9)cj`}nMQ=ssg6w(ZuA9#Fvd zpz+O0voK2OiG|*b4xJ_)FzLoU10ZcS(tVwMH>7pJJir{%5OCx=H@a&S{4$3@_j3aPIUwWe+f92Vx zw4N+|RveYEP@^AB2>&R5d0L;F(Klo_bbuy%E254KLdoJ1h4A`vb5-2p}lBp;iT8rHeRdlZfoUAl>)o{_d!wM5d006L_tTdi5s0-Z0#|>)YZtHef zaZC~IjtvsD!7C)S4KZ=B$PJ=LOgoQ9Pf??p`mQTJt>86t!c_b5QO?crmP>84FV_^V zQ!#QxC6kjjP0-o34ABKAG-L|YUk$Mco3USR^?IIm%V`yv!}>-miyDJ4phC{kx9%6W zhi*MYG-G}Ga6?H zHo`K4uNwYQ?ZXr@<#d7^PP|}6p_MLu<W2eKKq5M2_mcc4vkL@?4uL%r3 z8m_5Vx?Wl&UJ9w4*ncay_?nmBCEsWJ^vU>aaiVvN594y`Fh1D9S!+TejyRcf#ZRZzxq_oeHC16oXW|jEtkfk{Af+gqw9#K-vpicd z6Db9&rX;!~yvA}nZI7V+yY4N>h3J?HArm)}cRlsQt(J{}tSL>x2p4Qz>Ap9xJibP|C_%2PdxZHBlHiLdoOYP_ZPgC=|RLb-bo3j#~@bq}(;`4Z7 zB6jY(6^vE<%QQR+nC*uFAn^KdKh;AkPW5;T!hp{mX7{P-dsf>Dt5y=;4yUq99z>WI1j52)@4-btP)jbSxLLy z{<*KkItwP1ztYKy6I5O3^LduBuqTH3N;4)sEo_?V@xtwV$4B9=1%Lx4=V;Q=;x!RX%U<5(ALpwh;O}9QS(A{z<}F#zR!EWPpy-{}O7^Y+1K*Xk- zc3)};)x7_`d64k{Q8Cq?n4u+<(;49v80TnI8^W2;K8SYt5ZbHnK;cp+uidOR68J5; zM&tIDW>aY06`}89KtY=|qOtZy)--+7^BTPBgi|rHyKO@rOX}c351qa5Mtv}*B}#Kc zU4m1WfA$1_>nC^*=xvv3GAbrKvY+>7z&^S?9ytp_=p z`Nur`hu7aalH-)zrC+lKr=Qe6DJSdCaZ1c#|8tD;XEizw`=G$|(?SKqN&ORcGMOKT zy(Ik?_Se*Y9Jl-87iPfrr2dIJnNyGBP=~GGk+&;mE@{4<803fO#PyYjE7Kt_h literal 0 HcmV?d00001 diff --git a/misc/bazel/internal/zipmerge/test-files/CPython.zip b/misc/bazel/internal/zipmerge/test-files/CPython.zip new file mode 100644 index 0000000000000000000000000000000000000000..2cc0dd85e3986f4f1dd524c3916e2c098437af5b GIT binary patch literal 9481 zcmZvi1y~%*^0yav_uwu;!{Q!1XmGa$f6$ax;dKJ+OR18(+d9YR=is@ zS=_JyfD%3*)-|YzCT(? z(O2EMItZPL*-;%pwINz=@DMYl3JLQ)3+PrPM!=GJt@G0lLruIZSqVpE ziP!m^!+(E6jbfrW0yxPHf=ScC%1A{fB_qBj$ycWJf_}EvJ4$0+71mC!X-TY($J^pU zKEF9ko1$T+04F7XpHxI`ppT$nyP{`K020xmp>*E8-}URwiu(#fOZahiXCmV>J!W43 zT{lr#!GP-Wuxpa!I;|@{;P`0+4uZ}Sb^Ev}OS8>)i50xk$nv_)+YGFUwQ?7E_ThEj z>1@h0+F=R>AR0GpNeg@0jpm+DQagUoOFm-aXHpvJYYp6<)XyoRxxlCs6xt^uszD4v z_|a{xbnI}eo9>+DgEI}l7M%ZE?;S>II*=h1l4n9biqz%NL9q=#QW{aT@|*X7%Edhn z>(4k1aC>Fki^~?mq!tUut&^er2v4BLkWyZ(rft|k&VLYclY;vE<`MLV22riBTaSfyBh;JHQJdCNj~yX{+z z@JRdceW`_U_W4-F)+yiJ4oZIa(>X}9ZvgMPUVQqw%%wLXEDL+zC5OxHUhr97GQGXk zndewgtDwal0kqW8S90jue!8K1Ve!47ACD&}w*+(iy3=O^bglB17)4|i^I$W`XJ5@t z9Tfjc6vX4 z5u*>GW4CT^gkfu`aOelZwjfT{xgzxhGqO`jk8A$HlN!d3g#jg2xclF>;?6>aOA$kKm1ldrg?!D# z%XGH4kGbK)+^dpNthFI|XQ3$Pzmb`|wh2vPIet99E#`VU2_=6$0&+b5+PnFQTc6<0 zWJO{kd@jn`G+J+HEo4%hbz+_ixuR`c=_?hCF-0Csq-yp~?XV+h0uW<7?v+t&;uCD* zV1v0~(TeC}8Ml#>Wpg}VZ&q2DbG@4;wl(qkE=%wePUgy>*{v)#t^0FN{b?UV5`T<_ z?9Mw32@LFH0}lmk^c>txz>UsK!x9%dCeGC?XX+-NHNJ@K$F(BBa`VAWR2i889T*kf zag|e0$=Dtgf*6%@j!k_qCmd@d_Z_ThSko)b#dq`h43;8aMxW}(JMO#UZ#~otj_xyd zYv0}W336myss5#eRokSoc0-G<9F85N0T9K~xk*#wu)eFgTT9qMH@n z{}i^pJ1W*X{1Q3<_cmx15~US)MvpJMj!MmPvE>sr(%v3b4g2NDNn7y+G7ia?6a={5 z4Gg3H1q}cA?o8S)sX&d89pjS$mr}5q5?wz*E^s@Qq!9GW!7AvdBLR6~{L{QZVKg7Mh zj4UB?)L_t&SW;>8(VtGp366Ow^wcz^L~n=mnb1&TX!;9>1Q}lf-_ys6Jo zRDl{co*L;6YpG~#3fykc{*2ExY3p6tOI#pd;zRsfd_)zDEKER-%C=7SAQOjwB4na4 zg6J!Z-iFlHJuWRCw@OwSEDs~DzA&q~WbZvsOui;KN6qg@V7mLTRT4%am-!iNir6X7 z#1)BWZ_!FXESa$*9HEJBFTDRg2V|@h=eXY(n~*zU7mi+Tg_b-$u5(HjuI!dHu0!KK zdX%(LvVWP-r0uv-Iz=UyTm22gk#3%Q{Kq|kN)TGsjYB0`rk#X{gPyEUK}a9DRq8Wu zDyHiU<;rf&l;Dueycfiue)26E?H0WFclom8{$D@;aZo&R39*Y6Iq($eQ&44zb^i@w z7xwu(k#jYR#L@5WnP1jdS8c>4$hOb+9;CJB(nL_SUnBcwoJ7hJZ2lr6fpm=5vATxOc}TM)}_(MzEi~^@PvPYy@;KUGx>71PXA=LZYP}Gx&;YZ zfZ?3L9Q+6|@>9kP^d=&zj$XwMwrQ#hW-393EP%MUI z;58$@nxjMzTNPMmKDwCRJk@~fT6!P^=9(|l!Qd4z$GwH_43p-#NfZ5IqCRFj>8Dof zieh_r4Sg^0EXt>Q>c_o-)a}w@tr#5hsVEES$!s!PqE?aO`!S#;;l*xjqW-%*Iqibv zBPSrp+JCjKMhCc!r8SPjqAkZ<-I#kU4lY+n|3XV@7^%@-$NK@T zbcvLuovP9EawicLAyL>anv04_T94CO%M@E9XgBOQ#eO%RrQJX6YdPczd9ca5~>gnpHIVIEa zq`ErE%Oq8$5M^kG_83-kEa;X#Wh9cY9U*ck1mhMeCO_CRGDY$HWtNP)g}Y6+J9dSD zN)u2WtoIh2*ks!*uZ9h+1KdcLx*~vy&$P6IiYgFBS@TZejB7;{nXOsCE?qKoOERf3 z$2az|jVOMVGN;-I$d%2_^Lfkvv-24d{}PkL%)?e0GTmX3LG(iZ`N4Es>t0=Z;!RFC zeThXC+i{YrN$M^&lC-)TR^0{?DzwgeEQqI*WA5paNEo?R_l>!@ zg%M$+gnmQC>&#uaX5V83Iv7V+Y&1YeVIG85;Q~ja+E*T^G6EO!rwdI?%7=RMsw(KPCdee?I!`lbS6Q@%45&`Z+V-(C$NJNL!c}Vc6ILXQvNV@e=#!;r)ru^Ac+6k2^*GrbT(CI+OT88~t4rusA zesiDG1NR|{fK#-b9B954UJ6@j{YkxpJ_fGWh;aBO$L3!(r`(G@EaSZK%~fD|NJ10e z;>&bkxL!>j3}8UR2#Eca*aD6@t01Pmfs-2XH*yncktmuyGi49naKq#Kr@v0duMTeyNyL#&7h#4 z_SEWDIC)-%RqjaiIopZ`xTAnwBF$~bQn(D(L6ae4ZwZ76q!@H3U^19*;^q)gqhjLJ zlkKb|GZC%=#Xk9&U*CZT#NaSS3dkElWV`vNnIACB+3ZST^>tR zmAe?X|6C&SKt~3`{H{h_gW6@f#eUUDgyVlAd2+U=GIDKHon&`b{4KihT!WGVYJS)c z2*m|Nb5I{$uw>YTZK1ABcrVOg{n2%YI)~wkiCDD4FX{b4>~S;6S{lXV+K5PzxceLq z8t7YER;`n;oFI`f!q??B20lU236>93vMd@_3Kbe)6ibd$#2*1`m)1_Y-2{j$4}2!mZZ{lB|Q*I2rR1 zn)<--pEJ%4qtMC8SNjYsxSpGD{O6!HW%B?rK$o=sW^=qcsC8B5`>U&;qf?_$ ztD)9@u%4&$yJ_4Y?Xw@;l+bLwun4xbPe~5Ogfe!p&jU`|QS)2ho7wNo)SOX8Fe_=% zKk2Gh0jU}t;um_8*=fJwK{@K#Mzm(3e9A~v(_bG%(o#H0moRig4)Or$-?sY z8P#G4-a_NZx}kICg&-&a%4pH93!vf6>YxSizG zdIYgF;51wn2(CK~-Z?ihBK8yHb2i(N1}AVksLFoFDSLMGngc0cbR3e`M>B_mqb(2s z{;(c~;VMeBKXHV$RcWCAI>AN#QR9S}EF~L~CQa5hnK!kE`IAIYXSG=ve_?K9`zJ)H zhJqArzPt3T%5nADYs*H>5p!blZ{*J@Wp*5e0m8gta;g3fd4Yyw1Pk8&R$Fhr4<(OrF&$&ng-+oTXCq&ccbsEEn7;f9V_p4BB;uj$H@-N99u6rzF^Le*>bB0siUCEc@>;gPE@{%;-nPKr7fu49S$oO zF=25_>;|*)3lrp|`fomm^6O26_`(!X^Vl%WW#3*C!R7f+7Tmqx9@Wr$;#6cO+xjQMWe6U8gtc`mey>8mMsW3b||8u zQNg?v*NW-X=!qJ!0_VN7^X+M!c)W0L37+5fibk?lu<7_KWp7K`SBdjY!jm_VKYICC4xin5yJ2K{aID`C+Zzik%%`r9|eWPmA6sh_M!k0VoYRB1MinN(F&2OGo?IRb=^zP5O ziE8gO)3xeM(ft6rxetyEAcbjVXG2v#(t>M7R651GxMbePEagJ6!rY835R2wd(%>7N z<(&}#T}WpL`Cm1!Mmlv9t zB>bx$qH>aMLFMS5G^cEBuwNvj$nT^v*8vz?y(XB)+f?SBZ65MbZt*vKDv8|y$pMQ=ik`B z=Y+@}z!UA+1R*JqbDYoIe2NO)tPRmSm^c8ImG`Sn%eK;W?DUlnOcrm$u!M~MGHa_` zpc~0}oHYZ3n-^1uTZ$5%`cXMHKi_Zr8i3A;Y`J>wV9rC-hKKDN~d}UhQLrsq) zc=;?M@&ScT776V|Wb3$c*;mDwboc|xwC~Ofb9V^{N>wU?;=EurIfuy{2NY_FhzIzU z=zw+ptS5x0UK6z8lkGQC`k3U6;?fj7vrDkkPSdm5U1M7O_m?qorFdGbe3$M4Q zv&W_z;!HjYZSN+0na+L0B{*nSYcds$#aEdz+&a||*zYW%?-?Z6u{z{AJBbeqWe2Ah zp6C^2G~Kp?Sns9U^zsRst^FiJ%Pv0w1bBaRec=*(IwyC!RgKSk;}gSofWn^A1M=bmx)_2M0pR|~&q*K~O`kv3h z2UjaDUT?P+}KQisn6wU5`KxiDi8I+bbH=p)O0XQIFC>6 z!LGy|RnsjsF$5^1H1Gym;Mwx{eAwXC5c0Ox-upMm&qRfzm4VrEv*vn{k#1~EZ2?7S zX{F_%FRQ~2p&m~2av`Ncs!^ux`xiy%aknT@0Sv>>`C0mux}W_L?{YpeG(nd+m6hC^ z*ltgrhC^t~d}hMUEHSM!)(1*$aM}0$e7iDCrV43*n_hAJ?qiY5&w6h|wK9zUl$9E5 z4TCDX!hw#&ghV7u&G)GO8U3Y#0S}%?Q&KQg2rwipYd~pjlzPmVwKxtKf2&}b05_C~ z)xK~_=A<0RKErh`W@$0i&^c99^;#kkd|(~xB&w+^6D!iRl7NJngWtVFJXg~Lo2M4H ztvV9~V7-S(&ie7^T+Bz@6p1J#5nR^&l!p-C@e?y=J7&FzRg44SJIphr#|D7rMm*Mu z9_>!;ApXO=-T@e<%Fxc)^L%81{^%-=Br?qd6VGzxvW7wPe0~4a`1|Hjs(gNB*RHL} z{{4pfo*(zciV~ls??WMim#guxFS?UJEMAV5>Q#crg>FhkNsGU$AD9;Ch|5+LCKzqm zs65R8acy}xduI9F!_Y;(=}`3{mhX{ghBT;30j5DuanXV&FhptsH+I<=`+*xJJ3fm( z+lF!f;DDmIa`~-_%WQXpk8Z8Br^_TWGzNnOiCHXV>PKOe-Z0KUxyr%-uKM%zf+)Sn z;MRJV_qN6eOJsOZyrvX)ws>q6c31ltv@~{ zw-vMm$^VGzmY&fMB)o+QtVC7Y8%aJ(R{g=goulY5-!FW30-7+oIn24yBdxU^Bti{> z({SzPiy&cGHSmRUykE^^vy4 zTSrG+g27$kT>O$y$vS>5syRlu?5CAzkNOM}+VJ?F2OX*~*{4$hLTFb(UM#y|b`kXP zJoncodR56o<6telz>C$q^8j>i@MoN9q_p$CoNdrLY+DlWUL%9}3s%|)3k_*db#moK z4U~&l&9{c|ewAE(f9fhhd!tgR9|H8982l05JGmf&jXSMKGuLv%t^OhEQnLFWF#5%|la+4xvZzA6yMFn#$lmum_=WA9_H!gMY4c2}vim(x3F6Nr z3-+6g?_u|M^m11w0D^1Z5tfB3Dx{=0V#mn1HkO#}z9PCF9Xt>$!M>WlevRyK)TCDN z(D1X+rpcEb<*q^b>XQE=pBHgys%WH{}Hvp)5!OWSW-uXChXu$1t>xaP1~A@{7|M}9k`41;YspC-w!nt`u_ ztsWN1=dXZLE$>vg+R_p7DkiPC+S)scrrfmkn)sssz3;7%S01ShkA-^zUW}oUeao~f zE*KYa*>b~RtjT_-^zB+B%A#|=^qAhlWb z-wtCd;l7LfY)7BJ37m{lfj=}&gx;`i(m60NA{B-%qhFYI(iorRzD8dW$7;fU3*{k> zeuB}UKC=(r{4ogy^zaiq3y=f!S~M2wz2`Rs6|e~b>* zl3lDuDJ^6*c2GEs2A;x%xA?WAu+(l;)xVxdkn(i8#1aeBpH;?vU72B}o0mtd1!ojW znAfh>!x2G;loDh}P0i=f!R9#S(x%7ReA}OL%voxhJda+ z%{-?Yh{+DBSbS5|9`ZsMkI`_;KyJ}5|C+W z3^U^mG&X%7z0yt^gBPSfJQ`ygIX0?bTHs?Xyl;9!^J7l;3N?JpPVXlSaz>gULpT3W z0)O+eMJXG5p}UbT;}%25>0rn+!k^1~gRG5#u`jE$mQVly$!{x{FG8uIwTYvNy^@Ke zlfBJ9E0~{@IxV(X(OU^0(L@!LSP$^9od--4TjM#eUa5Gb1uw=#(E#%f+Fc|Ph6j2wo}nreT9X|TKfWe! z$;6^NG;n$bv_|Fx`g+yaz?CfFEwUp-`ZFW63N@%*<&v zUhJeLx!x8&aM1duQ;nQX$l9!ym3%kpl~>C=UbHFr1A~(VqWR5?TVclcN9M&JsC&%Q zGnXWkqqXJSbj zq^DjkqshMn2}OHbI}>|Hx8DQ_xyVk79y0X6Q{U6GuLV^8M_Aa1=Jc&hjt8~##xe1) zaKMwh$Li|Ugf08STp74Vj;z}nNz9O>&m9kQ@rkui zY7ZohADM5p`4&7a_W{wSlSjJm?Xu59M4PzzwM1`lrx^WDW^6kBbLh2^9zw93Dw8;s zDAZOI=iT6f!;z@=yu-W4pGq^!|0~VNMU91a zlc5K_M1u7y&Ob`9v+L~*nuoTtOhqpZh=JqPHw5dowpY^HErpjOTtX{q@z{!;&B~Nr zq!+TNcbw#`MF=%j@KC%|z3FcjK>Pi&RxN&~I__Ce*xy7R+To7P8Of(4w_we@Z9PNf zTumIwYY!Kk!h+KML}UBXz8vCN^V;M^b%n>$Mo5&I4vnR%E_;@50;*mm(g<}izk877 z|6Bodl~WWguTu*EfKilm?Kx|fq)eHy9nZc6;M#UMCGpy{Xgda7fK)k$o~EY0)P(?Yb79Iz#WJ9WgT5` z{yzS1&;P6F;#bdq^+WTr|F05^-zj-V|LN5~{ZIRE%AbPU@06&Qf{))Q|CZl=hkfJ4 z{o92)tUv9)VSlPuzr!lw{tNq`TGsElyNdr{<_!O||Hl0(BK?j_dZ~5!<=a2Fed@lmFrT;{CG!=KPsje&;B?Xo0`{`QM4= zcOItHe@5W>{eQT>?7w+`&V9f0$X^P9{y*>k&V;|i*c1K(`vv`F{|)D3>1)~`EEQ3mE^ECT=tFRz-Hv@Asa*U|q2SGFe5 literal 0 HcmV?d00001 diff --git a/misc/bazel/internal/zipmerge/test-files/almost-minimal.zip b/misc/bazel/internal/zipmerge/test-files/almost-minimal.zip new file mode 100644 index 0000000000000000000000000000000000000000..44541199b782c45c8679e4f33065b0c5ddf09b1e GIT binary patch literal 163 zcmWIWW@h1H0D(holU${aJhR{hvO$;wh|@B2QuRtIN-|P&a`F|*^NVs)c)0?+8JX-E zaA{HjY6AfUFo`gN3o63MAi=Qw>OafJQWqE}@9;E3rUSfL*+AMDfiMb48-vXN05Pv3 AN&o-= literal 0 HcmV?d00001 diff --git a/misc/bazel/internal/zipmerge/test-files/empty.zip b/misc/bazel/internal/zipmerge/test-files/empty.zip new file mode 100644 index 0000000000000000000000000000000000000000..15cb0ecb3e219d1701294bfdf0fe3f5cb5d208e7 GIT binary patch literal 22 NcmWIWW@Tf*000g10H*)| literal 0 HcmV?d00001 diff --git a/misc/bazel/internal/zipmerge/test-files/minimal-x3.zip b/misc/bazel/internal/zipmerge/test-files/minimal-x3.zip new file mode 100644 index 0000000000000000000000000000000000000000..a9854656fb0414e710db62782e18ea4931b75855 GIT binary patch literal 414 zcmWIWW@h1H0D;x5lU%_JC;<{p(of6GN!2TN<|ABAso5M1$nFjRnho=|&WQ-SJGfYLBsTnrEqMg|E6 duJ~N$M^YCU(dhthRyL3}Mj(s=(rO?M0{|`L6|VpQ literal 0 HcmV?d00001 diff --git a/misc/bazel/internal/zipmerge/test-files/slf4j-api-classes-with-footers.jar b/misc/bazel/internal/zipmerge/test-files/slf4j-api-classes-with-footers.jar new file mode 100644 index 0000000000000000000000000000000000000000..b3723a437c2289aa05c0000facb0615aa7791d59 GIT binary patch literal 341 zcmWIWW@Zs#;Nak3Sdl!(jR6U8GO#fCx`sIFdiuHP|2xINz|0Wf&CUT*!30$nfK#&w zPz7AGucM!*n`>~0p0C?y-!rFuymj?1@_OrPojY@WbCAIm;|EWR^t^m^Jbf>gu43Vg zcp-UY_6)6zAL6QN&zERfJQo*zBCa}(VQTiLqK`$4U`MPxc0tM+XoU+92Y53wi7=r0 u0p>uEA5a0@Q6L|oYehB|84f!vFyKEk!c` literal 0 HcmV?d00001 diff --git a/misc/bazel/internal/zipmerge/test-files/slf4j-api-classes-without-footers.jar b/misc/bazel/internal/zipmerge/test-files/slf4j-api-classes-without-footers.jar new file mode 100644 index 0000000000000000000000000000000000000000..498cc8966326fa7e0539d8b31381bf44a0f81274 GIT binary patch literal 309 zcmWIWW@Zs#VBp|jSdl!(jR6RlKm-tQGO#fCx`sIFdiuHP|2xINz|0VUqIl)83sTNN zg)Tr0Gyj@-BlPJ4nQ*O0?ho*0Wdo^V0>WA#y$Hl%004TLLyrIe literal 0 HcmV?d00001 diff --git a/misc/bazel/internal/zipmerge/zipmerge_test.cpp b/misc/bazel/internal/zipmerge/zipmerge_test.cpp new file mode 100644 index 000000000000..4ac907d60d2d --- /dev/null +++ b/misc/bazel/internal/zipmerge/zipmerge_test.cpp @@ -0,0 +1,155 @@ +#include "misc/bazel/internal/zipmerge/zipmerge.h" + +#include +#include +#include +#include + +#include +#include +#include "tools/cpp/runfiles/runfiles.h" + +using bazel::tools::cpp::runfiles::Runfiles; +using namespace std::string_literals; + +namespace codeql_testing { + +TEST(Zipmerge, ReadAndWrite) { + char buf[7] = {0}; + write2(buf + 1, 0xF2F1U); + write4(buf + 3, 0xF6F5F4F3UL); + EXPECT_STREQ(buf, "\x00\xF1\xF2\xF3\xF4\xF5\xF6"); + EXPECT_EQ(read2(buf + 1), 0xF2F1U); + EXPECT_EQ(read4(buf + 3), 0xF6F5F4F3UL); +} + +TEST(Zipmerge, AppendCd) { + output_cd.length = 0; + append_cd((const uint8_t*)"a", 1); + append_cd((const uint8_t*)"bcd", 3); + append_cd((const uint8_t*)"efghijklmno", 11); + EXPECT_EQ(output_cd.length, 15); + std::string_view bytes{reinterpret_cast(output_cd.bytes), 15}; + EXPECT_EQ(bytes, "abcdefghijklmno"); +} + +TEST(Zipmerge, ShouldIncludeFilenameNow) { + EXPECT_TRUE(should_include_filename_now((const uint8_t*)"x", 1)); + EXPECT_FALSE(should_include_filename_now((const uint8_t*)"x", 1)); + EXPECT_TRUE(should_include_filename_now((const uint8_t*)"y", 1)); + EXPECT_TRUE(should_include_filename_now((const uint8_t*)"yy", 2)); + EXPECT_FALSE(should_include_filename_now((const uint8_t*)"x", 1)); + EXPECT_FALSE(should_include_filename_now((const uint8_t*)"yy", 2)); +} + +TEST(Zipmerge, FindEocd) { + uint8_t buf[500] = {0}; + auto i = 0u; + for (auto& b : buf) { + b = i % 256; + } + memcpy(buf + 17, eocd_signature.data(), eocd_signature.size()); + memcpy(buf + 101, eocd_signature.data(), eocd_signature.size()); + EXPECT_EQ(find_eocd(buf, sizeof(buf)), buf + 101); +} + +const size_t num_hash_bytes = 128 / 8; +const size_t num_hash_uint64s = 2; + +std::string read_file(std::string_view filename) { + std::ifstream f(filename, std::ios::binary); + EXPECT_TRUE(f) << "Could not open '" << filename << "' (" << std::strerror(errno) << ")"; + if (!f) { + return 0; + } + std::stringstream contents; + contents << f.rdbuf(); + return contents.str(); +} + +std::string get_file(const char* name) { + static auto runfiles = []{ +std::string error; + auto ret = Runfiles::CreateForTest(&error); + EXPECT_TRUE(ret) << error; +return ret; +}(); + return runfiles->Rlocation("_main/misc/bazel/internal/zipmerge/test-files/"s + name); +} + +void expect_same_file(const char* actual, const char* expected) { + auto expected_file = get_file(expected); + auto actual_contents = read_file(actual); + unlink(actual); // If tests start failing, you might want to comment out this unlink in order to + // inspect the output. + ASSERT_EQ(actual_contents, read_file(expected_file)) + << "contents of " << actual << " do not match contents of " << expected_file; +} + +template +const char* zipmerge(Args*... inputs) { + reset(); + const char* output = nullptr; + std::vector args{"self"}; + std::array flags{{inputs...}}; + auto i = 0u; + for (; i < flags.size() && std::string_view{flags[i]}.starts_with("-"); ++i) { + args.push_back(flags[i]); + } + output = flags[i]; + args.push_back(output); + ++i; + for (; i < flags.size(); ++i) { + args.push_back(std::string_view{flags[i]}.starts_with("-") ? flags[i] : get_file(flags[i])); + } + EXPECT_EQ(zipmerge_main(args.size(), args.data()), 0); + return output; +} + +TEST(Zipmerge, Identity) { + expect_same_file(zipmerge("out.zip", "CPython.zip"), "CPython.zip"); +} + +TEST(Zipmerge, Idempotent) { + expect_same_file(zipmerge("out.zip", "CPython.zip", "CPython.zip", "CPython.zip"), "CPython.zip"); +} + +TEST(Zipmerge, RemoveEverything) { + expect_same_file(zipmerge("--remove=CPython", "out.zip", "CPython.zip"), "empty.zip"); +} + +TEST(Zipmerge, RemoveEverythingWildcard) { + expect_same_file(zipmerge("--remove=*on", "out.zip", "CPython.zip"), "empty.zip"); +} + +TEST(Zipmerge, RemovePrefixedPaths) { + expect_same_file(zipmerge("--remove=My/CPython", "out.zip", "--prefix=My", "CPython.zip"), + "empty.zip"); +} +TEST(Zipmerge, RemoveSome) { + expect_same_file(zipmerge("--remove=CPython/Extensions.qll", "--remove=CPython/ReturnTypeTrap.ql", + "out.zip", "CPython.zip"), + "CPython-partial.zip"); +} + +TEST(Zipmerge, RemoveSomeWildcard) { + expect_same_file(zipmerge("--remove=CPython/E*.qll", "--remove=CPython/R*", "--remove=CP*l", + "out.zip", "CPython.zip"), + "CPython-partial.zip"); +} + +TEST(Zipmerge, Prefix) { + expect_same_file( + zipmerge("out.zip", "minimal.zip", "--prefix=a", "minimal.zip", "--prefix=b", "minimal.zip"), + "minimal-x3.zip"); +} + +TEST(Zipmerge, InputFileOrder) { + expect_same_file(zipmerge("out.zip", "minimal.zip", "almost-minimal.zip"), "almost-minimal.zip"); +} + +TEST(Zipmerge, LocalFileFooters) { + expect_same_file(zipmerge("out.jar", "slf4j-api-classes-with-footers.jar"), + "slf4j-api-classes-without-footers.jar"); +} +} // namespace codeql_testing From 6d798410ce2c737dc58b05d2d34dcb8268da9c38 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 12:51:52 +0200 Subject: [PATCH 25/40] Bazel: add `--no-cleanup` to installer script --- misc/bazel/internal/install.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/misc/bazel/internal/install.py b/misc/bazel/internal/install.py index fafa901de7ea..bee745db87be 100644 --- a/misc/bazel/internal/install.py +++ b/misc/bazel/internal/install.py @@ -27,6 +27,8 @@ help="ripunzip executable rlocation") parser.add_argument("--zip-manifest", required=True, help="The rlocation of a file containing newline-separated `prefix:zip_file` entries") +parser.add_argument("--cleanup", action=argparse.BooleanOptionalAction, default=True, + help="Whether to wipe the destination directory before installing (true by default)") opts = parser.parse_args() build_file = runfiles.Rlocation(opts.build_file) @@ -35,10 +37,10 @@ zip_manifest = runfiles.Rlocation(opts.zip_manifest) destdir = pathlib.Path(build_file).resolve().parent / opts.destdir -if destdir.exists(): +if destdir.exists() and opts.cleanup: shutil.rmtree(destdir) -destdir.mkdir(parents=True) +destdir.mkdir(parents=True, exist_ok=True) subprocess.run([script, "--destdir", destdir], check=True) with open(zip_manifest) as manifest: From 76fbb522d25cf85fe573b9342c010b10e938298e Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 14:10:31 +0200 Subject: [PATCH 26/40] Bazel: use pack name for zip file name --- misc/bazel/pkg.bzl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 5eb5d0354c6d..549ab4651bb9 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -203,7 +203,7 @@ def codeql_pack( name, srcs = None, zips = None, - zip_filename = "extractor", + zip_filename = None, visibility = None, install_dest = "extractor-pack", **kwargs): @@ -248,7 +248,7 @@ def codeql_pack( base = internal(kind + "-zip-base"), zips = zips, zip_name = zip_filename, - zip_prefix = name, + zip_prefix = name, # this is prefixing the zip contents with the pack name kind = kind, visibility = visibility, ) From 6b971617e7d483a9a581701f96226240f361b029 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 14:17:05 +0200 Subject: [PATCH 27/40] Bazel: rename `_process_path` to `_expand_path`, and make its use clearer --- misc/bazel/pkg.bzl | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 549ab4651bb9..3ab8f316cfe6 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -21,7 +21,7 @@ _PLAT_DETECTION_ATTRS = { _PLAT_PLACEHOLDER = "{CODEQL_PLATFORM}" -def _process_path(path, platform): +def _expand_path(path, platform): if _PLAT_PLACEHOLDER in path: path = path.replace(_PLAT_PLACEHOLDER, platform) return ("arch", path) @@ -89,9 +89,9 @@ def _extract_pkg_filegroup_impl(ctx): for pfi, origin in src.pkg_files: dest_src_map = {} for dest, file in pfi.dest_src_map.items(): - file_kind, dest = _process_path(dest, platform) + file_kind, expanded_dest = _expand_path(dest, platform) if file_kind == ctx.attr.kind: - dest_src_map[dest] = file + dest_src_map[expanded_dest] = file if dest_src_map: pkg_files.append((PackageFilesInfo(dest_src_map = dest_src_map, attributes = pfi.attributes), origin)) @@ -124,9 +124,10 @@ def _imported_zips_manifest_impl(ctx): manifest = [] files = [] for zip, prefix in ctx.attr.zips.items(): - _, prefix = _process_path(prefix, platform) + # we don't care about the kind here, as we're taking all zips together + _, expanded_prefix = _expand_path(prefix, platform) zip_files = zip.files.to_list() - manifest += ["%s:%s" % (prefix, f.short_path) for f in zip_files] + manifest += ["%s:%s" % (expanded_prefix, f.short_path) for f in zip_files] files += zip_files output = ctx.actions.declare_file(ctx.label.name + ".params") @@ -158,9 +159,9 @@ def _zipmerge_impl(ctx): output = ctx.actions.declare_file(filename) args = [output.path, "--prefix=%s" % ctx.attr.zip_prefix, ctx.file.base.path] for zip, prefix in ctx.attr.zips.items(): - zip_kind, prefix = _process_path(prefix, platform) + zip_kind, expanded_prefix = _expand_path(prefix, platform) if zip_kind == ctx.attr.kind: - args.append("--prefix=%s/%s" % (ctx.attr.zip_prefix, prefix.rstrip("/"))) + args.append("--prefix=%s/%s" % (ctx.attr.zip_prefix, expanded_prefix.rstrip("/"))) args += [f.path for f in zip.files.to_list()] zips.append(zip.files) ctx.actions.run( From e2206e62d67eac314a3a1602010aa5273dc98298 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 14:38:31 +0200 Subject: [PATCH 28/40] Bazel: restrict `codeql_pack` `zips` to `.zip` files --- misc/bazel/pkg.bzl | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 3ab8f316cfe6..a504db06ab37 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -147,7 +147,10 @@ _imported_zips_manifest = rule( {CODEQL_PLATFORM} can be used as zip prefixes and will be expanded to the relevant codeql platform. """, attrs = { - "zips": attr.label_keyed_string_dict(doc = "mapping from zip files to install prefixes", allow_files = True), + "zips": attr.label_keyed_string_dict( + doc = "mapping from zip files to install prefixes", + allow_files = [".zip"], + ), } | _PLAT_DETECTION_ATTRS, ) @@ -189,9 +192,12 @@ _zipmerge = rule( attrs = { "base": attr.label( doc = "Base zip file to which zips from `zips` will be merged with", - allow_single_file = True, + allow_single_file = [".zip"], + ), + "zips": attr.label_keyed_string_dict( + doc = "mapping from zip files to install prefixes", + allow_files = [".zip"], ), - "zips": attr.label_keyed_string_dict(doc = "mapping from zip files to install prefixes", allow_files = True), "zip_name": attr.string(doc = "Prefix to use for the output file name"), "kind": attr.string(doc = "Which zip kind to consider", values = ["generic", "arch"]), "zip_prefix": attr.string(doc = "Prefix posix path to add to the zip contents in the archive"), From 00ed00e1e5db2de35204e986b72d299563af3bb1 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 14:54:06 +0200 Subject: [PATCH 29/40] Bazel: avoid unneeded operations if no imported zips are present --- misc/bazel/internal/install.py | 27 ++++++----- misc/bazel/pkg.bzl | 83 ++++++++++++++++++++++------------ 2 files changed, 70 insertions(+), 40 deletions(-) diff --git a/misc/bazel/internal/install.py b/misc/bazel/internal/install.py index bee745db87be..f1c1a410e0c6 100644 --- a/misc/bazel/internal/install.py +++ b/misc/bazel/internal/install.py @@ -23,18 +23,18 @@ help="The wrapped `pkg_install` installation script rlocation") parser.add_argument("--build-file", required=True, help="BUILD.bazel rlocation relative to which the installation should take place") -parser.add_argument("--ripunzip", required=True, - help="ripunzip executable rlocation") -parser.add_argument("--zip-manifest", required=True, +parser.add_argument("--ripunzip", + help="ripunzip executable rlocation. Must be provided if `--zip-manifest` is.") +parser.add_argument("--zip-manifest", help="The rlocation of a file containing newline-separated `prefix:zip_file` entries") parser.add_argument("--cleanup", action=argparse.BooleanOptionalAction, default=True, help="Whether to wipe the destination directory before installing (true by default)") opts = parser.parse_args() +if opts.zip_manifest and not opts.ripunzip: + parser.error("Provide `--ripunzip` when specifying `--zip-manifest`") build_file = runfiles.Rlocation(opts.build_file) script = runfiles.Rlocation(opts.pkg_install_script) -ripunzip = runfiles.Rlocation(opts.ripunzip) -zip_manifest = runfiles.Rlocation(opts.zip_manifest) destdir = pathlib.Path(build_file).resolve().parent / opts.destdir if destdir.exists() and opts.cleanup: @@ -43,10 +43,13 @@ destdir.mkdir(parents=True, exist_ok=True) subprocess.run([script, "--destdir", destdir], check=True) -with open(zip_manifest) as manifest: - for line in manifest: - prefix, _, zip = line.partition(":") - assert zip, f"missing prefix for {prefix}, you should use prefix:zip format" - dest = destdir / prefix - dest.mkdir(parents=True, exist_ok=True) - subprocess.run([ripunzip, "unzip-file", zip, "-d", dest], check=True) +if opts.zip_manifest: + ripunzip = runfiles.Rlocation(opts.ripunzip) + zip_manifest = runfiles.Rlocation(opts.zip_manifest) + with open(zip_manifest) as manifest: + for line in manifest: + prefix, _, zip = line.partition(":") + assert zip, f"missing prefix for {prefix}, you should use prefix:zip format" + dest = destdir / prefix + dest.mkdir(parents=True, exist_ok=True) + subprocess.run([ripunzip, "unzip-file", zip, "-d", dest], check=True) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index a504db06ab37..53735d009bc2 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -27,13 +27,28 @@ def _expand_path(path, platform): return ("arch", path) return ("generic", path) -def _detect_platform(ctx): - if ctx.target_platform_has_constraint(ctx.attr._windows[platform_common.ConstraintValueInfo]): - return "win64" - elif ctx.target_platform_has_constraint(ctx.attr._macos[platform_common.ConstraintValueInfo]): - return "osx64" +def _platform_select( + ctx = None, + *, + linux, + windows, + macos): + if ctx: + if ctx.target_platform_has_constraint(ctx.attr._windows[platform_common.ConstraintValueInfo]): + return windows + elif ctx.target_platform_has_constraint(ctx.attr._macos[platform_common.ConstraintValueInfo]): + return macos + else: + return linux else: - return "linux64" + return select({ + "@platforms//os:linux": linux, + "@platforms//os:macos": macos, + "@platforms//os:windows": windows, + }) + +def _detect_platform(ctx = None): + return _platform_select(ctx, linux = "linux64", macos = "osx64", windows = "win64") def codeql_pkg_files( *, @@ -160,11 +175,11 @@ def _zipmerge_impl(ctx): platform = _detect_platform(ctx) filename = "%s-%s.zip" % (ctx.attr.zip_name, platform if ctx.attr.kind == "arch" else "generic") output = ctx.actions.declare_file(filename) - args = [output.path, "--prefix=%s" % ctx.attr.zip_prefix, ctx.file.base.path] + args = [output.path, "--prefix=%s" % ctx.attr.prefix, ctx.file.base.path] for zip, prefix in ctx.attr.zips.items(): zip_kind, expanded_prefix = _expand_path(prefix, platform) if zip_kind == ctx.attr.kind: - args.append("--prefix=%s/%s" % (ctx.attr.zip_prefix, expanded_prefix.rstrip("/"))) + args.append("--prefix=%s/%s" % (ctx.attr.prefix, expanded_prefix.rstrip("/"))) args += [f.path for f in zip.files.to_list()] zips.append(zip.files) ctx.actions.run( @@ -200,7 +215,7 @@ _zipmerge = rule( ), "zip_name": attr.string(doc = "Prefix to use for the output file name"), "kind": attr.string(doc = "Which zip kind to consider", values = ["generic", "arch"]), - "zip_prefix": attr.string(doc = "Prefix posix path to add to the zip contents in the archive"), + "prefix": attr.string(doc = "Prefix posix path to add to the zip contents in the archive"), "_zipmerge": attr.label(default = "//misc/bazel/internal/zipmerge", executable = True, cfg = "exec"), } | _PLAT_DETECTION_ATTRS, ) @@ -245,25 +260,35 @@ def codeql_pack( kind = kind, visibility = ["//visibility:private"], ) - pkg_zip( - name = internal(kind + "-zip-base"), - srcs = [internal(kind)], - visibility = ["//visibility:private"], - ) - _zipmerge( - name = internal(kind + "-zip"), - base = internal(kind + "-zip-base"), + if zips: + pkg_zip( + name = internal(kind + "-zip-base"), + srcs = [internal(kind)], + visibility = ["//visibility:private"], + ) + _zipmerge( + name = internal(kind + "-zip"), + base = internal(kind + "-zip-base"), + zips = zips, + zip_name = zip_filename, + kind = kind, + prefix = name, + visibility = visibility, + ) + else: + pkg_zip( + name = internal(kind + "-zip"), + srcs = [internal(kind)], + visibility = ["//visibility:private"], + package_dir = name, + package_file_name = name + "-" + (_detect_platform() if kind == "arch" else "generic") + ".zip", + ) + if zips: + _imported_zips_manifest( + name = internal("zip-manifest"), zips = zips, - zip_name = zip_filename, - zip_prefix = name, # this is prefixing the zip contents with the pack name - kind = kind, - visibility = visibility, + visibility = ["//visibility:private"], ) - _imported_zips_manifest( - name = internal("zip-manifest"), - zips = zips, - visibility = ["//visibility:private"], - ) pkg_install( name = internal("script"), @@ -283,18 +308,20 @@ def codeql_pack( data = [ internal("build-file"), internal("script"), + ] + ([ internal("zip-manifest"), "//misc/bazel/internal/ripunzip", - ], + ] if zips else []), deps = ["@rules_python//python/runfiles"], args = [ "--build-file=$(rlocationpath %s)" % internal("build-file"), "--pkg-install-script=$(rlocationpath %s)" % internal("script"), "--destdir", install_dest, + ] + ([ "--ripunzip=$(rlocationpath //misc/bazel/internal/ripunzip)", "--zip-manifest=$(rlocationpath %s)" % internal("zip-manifest"), - ], + ] if zips else []), visibility = visibility, ) native.filegroup( From 9c1efb9f0e9bfb5fe9ab1d569482c9610d5489d6 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 15:09:15 +0200 Subject: [PATCH 30/40] Bazel: expose `compression_level` in `codeql_pack` --- misc/bazel/pkg.bzl | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 53735d009bc2..0a33374fa780 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -228,6 +228,7 @@ def codeql_pack( zip_filename = None, visibility = None, install_dest = "extractor-pack", + compression_level = None, **kwargs): """ Define a codeql pack. This macro accepts `pkg_files`, `pkg_filegroup` or their `codeql_*` counterparts as `srcs`. @@ -243,6 +244,9 @@ def codeql_pack( The distinction between arch-specific and generic contents is made based on whether the paths (including possible prefixes added by rules) contain the special `{CODEQL_PLATFORM}` placeholder, which in case it is present will also be replaced by the appropriate platform (`linux64`, `win64` or `osx64`). + + `compression_level` can be used to tweak the compression level used when creating archives. Consider that this + does not affect the contents of `zips`, only `srcs`. """ internal = _make_internal(name) zip_filename = zip_filename or name @@ -265,6 +269,7 @@ def codeql_pack( name = internal(kind + "-zip-base"), srcs = [internal(kind)], visibility = ["//visibility:private"], + compression_level = compression_level, ) _zipmerge( name = internal(kind + "-zip"), @@ -282,6 +287,7 @@ def codeql_pack( visibility = ["//visibility:private"], package_dir = name, package_file_name = name + "-" + (_detect_platform() if kind == "arch" else "generic") + ".zip", + compression_level = compression_level, ) if zips: _imported_zips_manifest( From 67d622fa9deba423b5e70e20073180fd8773800d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 15:44:53 +0200 Subject: [PATCH 31/40] Bazel: actually run the zipmerge tests --- misc/bazel/internal/zipmerge/.clang-format | 7 ++++++ misc/bazel/internal/zipmerge/BUILD.bazel | 2 ++ .../bazel/internal/zipmerge/zipmerge_test.cpp | 25 ++++++++++--------- 3 files changed, 22 insertions(+), 12 deletions(-) create mode 100644 misc/bazel/internal/zipmerge/.clang-format diff --git a/misc/bazel/internal/zipmerge/.clang-format b/misc/bazel/internal/zipmerge/.clang-format new file mode 100644 index 000000000000..ca0a3afd986d --- /dev/null +++ b/misc/bazel/internal/zipmerge/.clang-format @@ -0,0 +1,7 @@ +BasedOnStyle: Chromium +ColumnLimit: 100 +IndentWidth: 2 +SortIncludes: false +AllowShortIfStatementsOnASingleLine: WithoutElse +AlwaysBreakBeforeMultilineStrings: false +Standard: c++20 diff --git a/misc/bazel/internal/zipmerge/BUILD.bazel b/misc/bazel/internal/zipmerge/BUILD.bazel index bb944cc03432..d46bf74cc1e2 100644 --- a/misc/bazel/internal/zipmerge/BUILD.bazel +++ b/misc/bazel/internal/zipmerge/BUILD.bazel @@ -20,6 +20,8 @@ cc_binary( cc_test( name = "test", size = "small", + srcs = ["zipmerge_test.cpp"], + data = glob(["test-files/*"]), deps = [ ":lib", "@bazel_tools//tools/cpp/runfiles", diff --git a/misc/bazel/internal/zipmerge/zipmerge_test.cpp b/misc/bazel/internal/zipmerge/zipmerge_test.cpp index 4ac907d60d2d..46bd55e7ffa3 100644 --- a/misc/bazel/internal/zipmerge/zipmerge_test.cpp +++ b/misc/bazel/internal/zipmerge/zipmerge_test.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -53,10 +54,7 @@ TEST(Zipmerge, FindEocd) { EXPECT_EQ(find_eocd(buf, sizeof(buf)), buf + 101); } -const size_t num_hash_bytes = 128 / 8; -const size_t num_hash_uint64s = 2; - -std::string read_file(std::string_view filename) { +std::string read_file(const std::string& filename) { std::ifstream f(filename, std::ios::binary); EXPECT_TRUE(f) << "Could not open '" << filename << "' (" << std::strerror(errno) << ")"; if (!f) { @@ -68,12 +66,12 @@ std::string read_file(std::string_view filename) { } std::string get_file(const char* name) { - static auto runfiles = []{ -std::string error; - auto ret = Runfiles::CreateForTest(&error); - EXPECT_TRUE(ret) << error; -return ret; -}(); + static auto runfiles = [] { + std::string error; + auto ret = Runfiles::CreateForTest(&error); + EXPECT_TRUE(ret) << error; + return ret; + }(); return runfiles->Rlocation("_main/misc/bazel/internal/zipmerge/test-files/"s + name); } @@ -90,7 +88,7 @@ template const char* zipmerge(Args*... inputs) { reset(); const char* output = nullptr; - std::vector args{"self"}; + std::vector args{"self"}; std::array flags{{inputs...}}; auto i = 0u; for (; i < flags.size() && std::string_view{flags[i]}.starts_with("-"); ++i) { @@ -102,7 +100,10 @@ const char* zipmerge(Args*... inputs) { for (; i < flags.size(); ++i) { args.push_back(std::string_view{flags[i]}.starts_with("-") ? flags[i] : get_file(flags[i])); } - EXPECT_EQ(zipmerge_main(args.size(), args.data()), 0); + std::vector argv; + std::transform(args.begin(), args.end(), std::back_inserter(argv), + [](const std::string& s) { return s.c_str(); }); + EXPECT_EQ(zipmerge_main(argv.size(), argv.data()), 0); return output; } From c3ccf4d5a339adf5bd4886052135ae0ece108526 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 15:47:24 +0200 Subject: [PATCH 32/40] Zipmerge: substitute `CPython` archives with dummy ones --- .../zipmerge/test-files/CPython-partial.zip | Bin 3909 -> 0 bytes .../internal/zipmerge/test-files/CPython.zip | Bin 9481 -> 0 bytes .../zipmerge/test-files/directory-partial.zip | Bin 0 -> 490 bytes .../zipmerge/test-files/directory.zip | Bin 0 -> 810 bytes .../bazel/internal/zipmerge/zipmerge_test.cpp | 23 +++++++++--------- 5 files changed, 12 insertions(+), 11 deletions(-) delete mode 100644 misc/bazel/internal/zipmerge/test-files/CPython-partial.zip delete mode 100644 misc/bazel/internal/zipmerge/test-files/CPython.zip create mode 100644 misc/bazel/internal/zipmerge/test-files/directory-partial.zip create mode 100644 misc/bazel/internal/zipmerge/test-files/directory.zip diff --git a/misc/bazel/internal/zipmerge/test-files/CPython-partial.zip b/misc/bazel/internal/zipmerge/test-files/CPython-partial.zip deleted file mode 100644 index a5a30ec3f90b6ba00898f0b123b7d14592da00e4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3909 zcmZvf2{=@38^=fXHOnAbvu|0lWl5HhZOG1e6Jt^s#K=;JB>ND?*vS^6NKy>hvJ}~u zVeH#r>{~@XExwoD&biL~&zb9)=Xd?@=REg4x1lZ}5fJd>7zvKl_<8gDgX~b%4SnFZ zTwTNse~%*jIZ9!QKkFh10N}|90Q~sv(I{1S2SZzT52)B3XXic(uxpP5&|eD^Ai<*6 zszAZ&%Ei%E8Lk^!Sq-wBbPkjbvtzarJV=9ZDn_T}Y_(}{W4hVqtd#@i*aX(p!E|;9d#_IgO+FCMi>RA|1$Uvu)j1nJSzCv`W`GLlnu;DB0 zd%_Mqx^0&msZ$$n`7*wm)9|EYcAX-oCd7?f`;u`R$=Hh<)7>}tW>8A*{|}hqo>|_D z`LE!<9A|NZ(E7U%mHj7Ar!Vp!ob}LgZ{hL2XC@PYsueYUS7w=ClzN9rxksp@h2%Tu z*87;K3P~*#{?j$tEgg-DhihaD7x<{5hV6jmoX5ne)(jabRGeJwTO4`D0{%n?u^+BX6u`!_0wVW{v ztEB3=mc*~*rXP}FDc#Ocs@4x9McZ#!DWN9oH@lEB;pbn8GUIM|#(J#4QzI@zos(e> z@;fv$YT|-B(?MnAdjj7UhO0&U5AY_go_W4*grHf!HHc+%EAzn{joyP|ucqjTEa1FX zww%uwD4zK&Z1X5B-F;QGP_b|ymY(5a0Bw(r&D+=$fk)+gU#!mQu2fe=BjP3GY28Ce# zcD6_!;s?QkWZo<5(D5ocLPFK0I!X(=+f&WGUaooDl#2E9MwnlIIVg3XQS9AYfDATzD^{Zwkr8Zc!Rxt>2FUhaE zqntEjsv~^d6gPGkoE~ZMU9&SAn+LHpYU&GD3f;E2>W5qawWtfsPGnH}2B4mLevH8h zzbJo>DCvq(?200m--}ZPHQS(e-N$zL7!4K3Fzf^yxRJ41P~+`vnPBNnVwr>ty$wAx zt(=-i{~VlM&`2n$%gsAa)p8}e&VaU#DKR6HADq78z0yqx+G>Zu z@voXD{Vv+D>_V}cU6BJZu$!^hKaPcUYe;kr%2BK9)cj`}nMQ=ssg6w(ZuA9#Fvd zpz+O0voK2OiG|*b4xJ_)FzLoU10ZcS(tVwMH>7pJJir{%5OCx=H@a&S{4$3@_j3aPIUwWe+f92Vx zw4N+|RveYEP@^AB2>&R5d0L;F(Klo_bbuy%E254KLdoJ1h4A`vb5-2p}lBp;iT8rHeRdlZfoUAl>)o{_d!wM5d006L_tTdi5s0-Z0#|>)YZtHef zaZC~IjtvsD!7C)S4KZ=B$PJ=LOgoQ9Pf??p`mQTJt>86t!c_b5QO?crmP>84FV_^V zQ!#QxC6kjjP0-o34ABKAG-L|YUk$Mco3USR^?IIm%V`yv!}>-miyDJ4phC{kx9%6W zhi*MYG-G}Ga6?H zHo`K4uNwYQ?ZXr@<#d7^PP|}6p_MLu<W2eKKq5M2_mcc4vkL@?4uL%r3 z8m_5Vx?Wl&UJ9w4*ncay_?nmBCEsWJ^vU>aaiVvN594y`Fh1D9S!+TejyRcf#ZRZzxq_oeHC16oXW|jEtkfk{Af+gqw9#K-vpicd z6Db9&rX;!~yvA}nZI7V+yY4N>h3J?HArm)}cRlsQt(J{}tSL>x2p4Qz>Ap9xJibP|C_%2PdxZHBlHiLdoOYP_ZPgC=|RLb-bo3j#~@bq}(;`4Z7 zB6jY(6^vE<%QQR+nC*uFAn^KdKh;AkPW5;T!hp{mX7{P-dsf>Dt5y=;4yUq99z>WI1j52)@4-btP)jbSxLLy z{<*KkItwP1ztYKy6I5O3^LduBuqTH3N;4)sEo_?V@xtwV$4B9=1%Lx4=V;Q=;x!RX%U<5(ALpwh;O}9QS(A{z<}F#zR!EWPpy-{}O7^Y+1K*Xk- zc3)};)x7_`d64k{Q8Cq?n4u+<(;49v80TnI8^W2;K8SYt5ZbHnK;cp+uidOR68J5; zM&tIDW>aY06`}89KtY=|qOtZy)--+7^BTPBgi|rHyKO@rOX}c351qa5Mtv}*B}#Kc zU4m1WfA$1_>nC^*=xvv3GAbrKvY+>7z&^S?9ytp_=p z`Nur`hu7aalH-)zrC+lKr=Qe6DJSdCaZ1c#|8tD;XEizw`=G$|(?SKqN&ORcGMOKT zy(Ik?_Se*Y9Jl-87iPfrr2dIJnNyGBP=~GGk+&;mE@{4<803fO#PyYjE7Kt_h diff --git a/misc/bazel/internal/zipmerge/test-files/CPython.zip b/misc/bazel/internal/zipmerge/test-files/CPython.zip deleted file mode 100644 index 2cc0dd85e3986f4f1dd524c3916e2c098437af5b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9481 zcmZvi1y~%*^0yav_uwu;!{Q!1XmGa$f6$ax;dKJ+OR18(+d9YR=is@ zS=_JyfD%3*)-|YzCT(? z(O2EMItZPL*-;%pwINz=@DMYl3JLQ)3+PrPM!=GJt@G0lLruIZSqVpE ziP!m^!+(E6jbfrW0yxPHf=ScC%1A{fB_qBj$ycWJf_}EvJ4$0+71mC!X-TY($J^pU zKEF9ko1$T+04F7XpHxI`ppT$nyP{`K020xmp>*E8-}URwiu(#fOZahiXCmV>J!W43 zT{lr#!GP-Wuxpa!I;|@{;P`0+4uZ}Sb^Ev}OS8>)i50xk$nv_)+YGFUwQ?7E_ThEj z>1@h0+F=R>AR0GpNeg@0jpm+DQagUoOFm-aXHpvJYYp6<)XyoRxxlCs6xt^uszD4v z_|a{xbnI}eo9>+DgEI}l7M%ZE?;S>II*=h1l4n9biqz%NL9q=#QW{aT@|*X7%Edhn z>(4k1aC>Fki^~?mq!tUut&^er2v4BLkWyZ(rft|k&VLYclY;vE<`MLV22riBTaSfyBh;JHQJdCNj~yX{+z z@JRdceW`_U_W4-F)+yiJ4oZIa(>X}9ZvgMPUVQqw%%wLXEDL+zC5OxHUhr97GQGXk zndewgtDwal0kqW8S90jue!8K1Ve!47ACD&}w*+(iy3=O^bglB17)4|i^I$W`XJ5@t z9Tfjc6vX4 z5u*>GW4CT^gkfu`aOelZwjfT{xgzxhGqO`jk8A$HlN!d3g#jg2xclF>;?6>aOA$kKm1ldrg?!D# z%XGH4kGbK)+^dpNthFI|XQ3$Pzmb`|wh2vPIet99E#`VU2_=6$0&+b5+PnFQTc6<0 zWJO{kd@jn`G+J+HEo4%hbz+_ixuR`c=_?hCF-0Csq-yp~?XV+h0uW<7?v+t&;uCD* zV1v0~(TeC}8Ml#>Wpg}VZ&q2DbG@4;wl(qkE=%wePUgy>*{v)#t^0FN{b?UV5`T<_ z?9Mw32@LFH0}lmk^c>txz>UsK!x9%dCeGC?XX+-NHNJ@K$F(BBa`VAWR2i889T*kf zag|e0$=Dtgf*6%@j!k_qCmd@d_Z_ThSko)b#dq`h43;8aMxW}(JMO#UZ#~otj_xyd zYv0}W336myss5#eRokSoc0-G<9F85N0T9K~xk*#wu)eFgTT9qMH@n z{}i^pJ1W*X{1Q3<_cmx15~US)MvpJMj!MmPvE>sr(%v3b4g2NDNn7y+G7ia?6a={5 z4Gg3H1q}cA?o8S)sX&d89pjS$mr}5q5?wz*E^s@Qq!9GW!7AvdBLR6~{L{QZVKg7Mh zj4UB?)L_t&SW;>8(VtGp366Ow^wcz^L~n=mnb1&TX!;9>1Q}lf-_ys6Jo zRDl{co*L;6YpG~#3fykc{*2ExY3p6tOI#pd;zRsfd_)zDEKER-%C=7SAQOjwB4na4 zg6J!Z-iFlHJuWRCw@OwSEDs~DzA&q~WbZvsOui;KN6qg@V7mLTRT4%am-!iNir6X7 z#1)BWZ_!FXESa$*9HEJBFTDRg2V|@h=eXY(n~*zU7mi+Tg_b-$u5(HjuI!dHu0!KK zdX%(LvVWP-r0uv-Iz=UyTm22gk#3%Q{Kq|kN)TGsjYB0`rk#X{gPyEUK}a9DRq8Wu zDyHiU<;rf&l;Dueycfiue)26E?H0WFclom8{$D@;aZo&R39*Y6Iq($eQ&44zb^i@w z7xwu(k#jYR#L@5WnP1jdS8c>4$hOb+9;CJB(nL_SUnBcwoJ7hJZ2lr6fpm=5vATxOc}TM)}_(MzEi~^@PvPYy@;KUGx>71PXA=LZYP}Gx&;YZ zfZ?3L9Q+6|@>9kP^d=&zj$XwMwrQ#hW-393EP%MUI z;58$@nxjMzTNPMmKDwCRJk@~fT6!P^=9(|l!Qd4z$GwH_43p-#NfZ5IqCRFj>8Dof zieh_r4Sg^0EXt>Q>c_o-)a}w@tr#5hsVEES$!s!PqE?aO`!S#;;l*xjqW-%*Iqibv zBPSrp+JCjKMhCc!r8SPjqAkZ<-I#kU4lY+n|3XV@7^%@-$NK@T zbcvLuovP9EawicLAyL>anv04_T94CO%M@E9XgBOQ#eO%RrQJX6YdPczd9ca5~>gnpHIVIEa zq`ErE%Oq8$5M^kG_83-kEa;X#Wh9cY9U*ck1mhMeCO_CRGDY$HWtNP)g}Y6+J9dSD zN)u2WtoIh2*ks!*uZ9h+1KdcLx*~vy&$P6IiYgFBS@TZejB7;{nXOsCE?qKoOERf3 z$2az|jVOMVGN;-I$d%2_^Lfkvv-24d{}PkL%)?e0GTmX3LG(iZ`N4Es>t0=Z;!RFC zeThXC+i{YrN$M^&lC-)TR^0{?DzwgeEQqI*WA5paNEo?R_l>!@ zg%M$+gnmQC>&#uaX5V83Iv7V+Y&1YeVIG85;Q~ja+E*T^G6EO!rwdI?%7=RMsw(KPCdee?I!`lbS6Q@%45&`Z+V-(C$NJNL!c}Vc6ILXQvNV@e=#!;r)ru^Ac+6k2^*GrbT(CI+OT88~t4rusA zesiDG1NR|{fK#-b9B954UJ6@j{YkxpJ_fGWh;aBO$L3!(r`(G@EaSZK%~fD|NJ10e z;>&bkxL!>j3}8UR2#Eca*aD6@t01Pmfs-2XH*yncktmuyGi49naKq#Kr@v0duMTeyNyL#&7h#4 z_SEWDIC)-%RqjaiIopZ`xTAnwBF$~bQn(D(L6ae4ZwZ76q!@H3U^19*;^q)gqhjLJ zlkKb|GZC%=#Xk9&U*CZT#NaSS3dkElWV`vNnIACB+3ZST^>tR zmAe?X|6C&SKt~3`{H{h_gW6@f#eUUDgyVlAd2+U=GIDKHon&`b{4KihT!WGVYJS)c z2*m|Nb5I{$uw>YTZK1ABcrVOg{n2%YI)~wkiCDD4FX{b4>~S;6S{lXV+K5PzxceLq z8t7YER;`n;oFI`f!q??B20lU236>93vMd@_3Kbe)6ibd$#2*1`m)1_Y-2{j$4}2!mZZ{lB|Q*I2rR1 zn)<--pEJ%4qtMC8SNjYsxSpGD{O6!HW%B?rK$o=sW^=qcsC8B5`>U&;qf?_$ ztD)9@u%4&$yJ_4Y?Xw@;l+bLwun4xbPe~5Ogfe!p&jU`|QS)2ho7wNo)SOX8Fe_=% zKk2Gh0jU}t;um_8*=fJwK{@K#Mzm(3e9A~v(_bG%(o#H0moRig4)Or$-?sY z8P#G4-a_NZx}kICg&-&a%4pH93!vf6>YxSizG zdIYgF;51wn2(CK~-Z?ihBK8yHb2i(N1}AVksLFoFDSLMGngc0cbR3e`M>B_mqb(2s z{;(c~;VMeBKXHV$RcWCAI>AN#QR9S}EF~L~CQa5hnK!kE`IAIYXSG=ve_?K9`zJ)H zhJqArzPt3T%5nADYs*H>5p!blZ{*J@Wp*5e0m8gta;g3fd4Yyw1Pk8&R$Fhr4<(OrF&$&ng-+oTXCq&ccbsEEn7;f9V_p4BB;uj$H@-N99u6rzF^Le*>bB0siUCEc@>;gPE@{%;-nPKr7fu49S$oO zF=25_>;|*)3lrp|`fomm^6O26_`(!X^Vl%WW#3*C!R7f+7Tmqx9@Wr$;#6cO+xjQMWe6U8gtc`mey>8mMsW3b||8u zQNg?v*NW-X=!qJ!0_VN7^X+M!c)W0L37+5fibk?lu<7_KWp7K`SBdjY!jm_VKYICC4xin5yJ2K{aID`C+Zzik%%`r9|eWPmA6sh_M!k0VoYRB1MinN(F&2OGo?IRb=^zP5O ziE8gO)3xeM(ft6rxetyEAcbjVXG2v#(t>M7R651GxMbePEagJ6!rY835R2wd(%>7N z<(&}#T}WpL`Cm1!Mmlv9t zB>bx$qH>aMLFMS5G^cEBuwNvj$nT^v*8vz?y(XB)+f?SBZ65MbZt*vKDv8|y$pMQ=ik`B z=Y+@}z!UA+1R*JqbDYoIe2NO)tPRmSm^c8ImG`Sn%eK;W?DUlnOcrm$u!M~MGHa_` zpc~0}oHYZ3n-^1uTZ$5%`cXMHKi_Zr8i3A;Y`J>wV9rC-hKKDN~d}UhQLrsq) zc=;?M@&ScT776V|Wb3$c*;mDwboc|xwC~Ofb9V^{N>wU?;=EurIfuy{2NY_FhzIzU z=zw+ptS5x0UK6z8lkGQC`k3U6;?fj7vrDkkPSdm5U1M7O_m?qorFdGbe3$M4Q zv&W_z;!HjYZSN+0na+L0B{*nSYcds$#aEdz+&a||*zYW%?-?Z6u{z{AJBbeqWe2Ah zp6C^2G~Kp?Sns9U^zsRst^FiJ%Pv0w1bBaRec=*(IwyC!RgKSk;}gSofWn^A1M=bmx)_2M0pR|~&q*K~O`kv3h z2UjaDUT?P+}KQisn6wU5`KxiDi8I+bbH=p)O0XQIFC>6 z!LGy|RnsjsF$5^1H1Gym;Mwx{eAwXC5c0Ox-upMm&qRfzm4VrEv*vn{k#1~EZ2?7S zX{F_%FRQ~2p&m~2av`Ncs!^ux`xiy%aknT@0Sv>>`C0mux}W_L?{YpeG(nd+m6hC^ z*ltgrhC^t~d}hMUEHSM!)(1*$aM}0$e7iDCrV43*n_hAJ?qiY5&w6h|wK9zUl$9E5 z4TCDX!hw#&ghV7u&G)GO8U3Y#0S}%?Q&KQg2rwipYd~pjlzPmVwKxtKf2&}b05_C~ z)xK~_=A<0RKErh`W@$0i&^c99^;#kkd|(~xB&w+^6D!iRl7NJngWtVFJXg~Lo2M4H ztvV9~V7-S(&ie7^T+Bz@6p1J#5nR^&l!p-C@e?y=J7&FzRg44SJIphr#|D7rMm*Mu z9_>!;ApXO=-T@e<%Fxc)^L%81{^%-=Br?qd6VGzxvW7wPe0~4a`1|Hjs(gNB*RHL} z{{4pfo*(zciV~ls??WMim#guxFS?UJEMAV5>Q#crg>FhkNsGU$AD9;Ch|5+LCKzqm zs65R8acy}xduI9F!_Y;(=}`3{mhX{ghBT;30j5DuanXV&FhptsH+I<=`+*xJJ3fm( z+lF!f;DDmIa`~-_%WQXpk8Z8Br^_TWGzNnOiCHXV>PKOe-Z0KUxyr%-uKM%zf+)Sn z;MRJV_qN6eOJsOZyrvX)ws>q6c31ltv@~{ zw-vMm$^VGzmY&fMB)o+QtVC7Y8%aJ(R{g=goulY5-!FW30-7+oIn24yBdxU^Bti{> z({SzPiy&cGHSmRUykE^^vy4 zTSrG+g27$kT>O$y$vS>5syRlu?5CAzkNOM}+VJ?F2OX*~*{4$hLTFb(UM#y|b`kXP zJoncodR56o<6telz>C$q^8j>i@MoN9q_p$CoNdrLY+DlWUL%9}3s%|)3k_*db#moK z4U~&l&9{c|ewAE(f9fhhd!tgR9|H8982l05JGmf&jXSMKGuLv%t^OhEQnLFWF#5%|la+4xvZzA6yMFn#$lmum_=WA9_H!gMY4c2}vim(x3F6Nr z3-+6g?_u|M^m11w0D^1Z5tfB3Dx{=0V#mn1HkO#}z9PCF9Xt>$!M>WlevRyK)TCDN z(D1X+rpcEb<*q^b>XQE=pBHgys%WH{}Hvp)5!OWSW-uXChXu$1t>xaP1~A@{7|M}9k`41;YspC-w!nt`u_ ztsWN1=dXZLE$>vg+R_p7DkiPC+S)scrrfmkn)sssz3;7%S01ShkA-^zUW}oUeao~f zE*KYa*>b~RtjT_-^zB+B%A#|=^qAhlWb z-wtCd;l7LfY)7BJ37m{lfj=}&gx;`i(m60NA{B-%qhFYI(iorRzD8dW$7;fU3*{k> zeuB}UKC=(r{4ogy^zaiq3y=f!S~M2wz2`Rs6|e~b>* zl3lDuDJ^6*c2GEs2A;x%xA?WAu+(l;)xVxdkn(i8#1aeBpH;?vU72B}o0mtd1!ojW znAfh>!x2G;loDh}P0i=f!R9#S(x%7ReA}OL%voxhJda+ z%{-?Yh{+DBSbS5|9`ZsMkI`_;KyJ}5|C+W z3^U^mG&X%7z0yt^gBPSfJQ`ygIX0?bTHs?Xyl;9!^J7l;3N?JpPVXlSaz>gULpT3W z0)O+eMJXG5p}UbT;}%25>0rn+!k^1~gRG5#u`jE$mQVly$!{x{FG8uIwTYvNy^@Ke zlfBJ9E0~{@IxV(X(OU^0(L@!LSP$^9od--4TjM#eUa5Gb1uw=#(E#%f+Fc|Ph6j2wo}nreT9X|TKfWe! z$;6^NG;n$bv_|Fx`g+yaz?CfFEwUp-`ZFW63N@%*<&v zUhJeLx!x8&aM1duQ;nQX$l9!ym3%kpl~>C=UbHFr1A~(VqWR5?TVclcN9M&JsC&%Q zGnXWkqqXJSbj zq^DjkqshMn2}OHbI}>|Hx8DQ_xyVk79y0X6Q{U6GuLV^8M_Aa1=Jc&hjt8~##xe1) zaKMwh$Li|Ugf08STp74Vj;z}nNz9O>&m9kQ@rkui zY7ZohADM5p`4&7a_W{wSlSjJm?Xu59M4PzzwM1`lrx^WDW^6kBbLh2^9zw93Dw8;s zDAZOI=iT6f!;z@=yu-W4pGq^!|0~VNMU91a zlc5K_M1u7y&Ob`9v+L~*nuoTtOhqpZh=JqPHw5dowpY^HErpjOTtX{q@z{!;&B~Nr zq!+TNcbw#`MF=%j@KC%|z3FcjK>Pi&RxN&~I__Ce*xy7R+To7P8Of(4w_we@Z9PNf zTumIwYY!Kk!h+KML}UBXz8vCN^V;M^b%n>$Mo5&I4vnR%E_;@50;*mm(g<}izk877 z|6Bodl~WWguTu*EfKilm?Kx|fq)eHy9nZc6;M#UMCGpy{Xgda7fK)k$o~EY0)P(?Yb79Iz#WJ9WgT5` z{yzS1&;P6F;#bdq^+WTr|F05^-zj-V|LN5~{ZIRE%AbPU@06&Qf{))Q|CZl=hkfJ4 z{o92)tUv9)VSlPuzr!lw{tNq`TGsElyNdr{<_!O||Hl0(BK?j_dZ~5!<=a2Fed@lmFrT;{CG!=KPsje&;B?Xo0`{`QM4= zcOItHe@5W>{eQT>?7w+`&V9f0$X^P9{y*>k&V;|i*c1K(`vv`F{|)D3>1)~`EEQ3mE^ECT=tFRz-Hv@Asa*U|q2SGFe5 diff --git a/misc/bazel/internal/zipmerge/test-files/directory-partial.zip b/misc/bazel/internal/zipmerge/test-files/directory-partial.zip new file mode 100644 index 0000000000000000000000000000000000000000..2c65f48180a69025daba1b626e8ffa2ead68b218 GIT binary patch literal 490 zcmWIWW@h1H0D-O5dm_LLD8a=b!;q3$l$u~<$K%^iqKE4UdLSza(R zFo1~wxEbX@GuRity2}JK1BCfe%t+KLsVD&(bm?|z+BG0XGboV@VHC(YPN8!bU^glS zZWPE(9++-Q0UE`~B*%=)ZxT>n3NZY21Tm3)$_nu*n&Xho!}KS_JVu~NTN+(Z%!B$G lXeh|ncnrn#F|wikD277)kISX3Y(Uiv%s}`HNZ$f+7yv3IYG(id literal 0 HcmV?d00001 diff --git a/misc/bazel/internal/zipmerge/test-files/directory.zip b/misc/bazel/internal/zipmerge/test-files/directory.zip new file mode 100644 index 0000000000000000000000000000000000000000..046f0a963b9f8aff3c57631a5088195020bb86dd GIT binary patch literal 810 zcmWIWW@h1H0D-O5dm_LLD8a=b!;q3$l$u~<$K%^iqKE4UdLSza(R zFo1~wxEa+zGg=SddCvqi1BCfe%t+QNsVD&(bnA9#+HD|4Gbou0VN^NLDE5V~?qWA8 z5pL8apfT4ljY>o^3gk1V(76k+8d2?#->@5%1UCxgqMMkm zN&*_i$Rx*%E5s$BVK2b&*Ac`-4u4ii_@f0GvU!-H4>6AsXwsHO7ZmfLaRD?G6c>05 z#f%4JL;Fz-g~kfdP*AMkF%&aKkPW?sVkk6z2zwthZjcRC1%?75IH55GG!#qP3GilR S13H0$6$qCAwH1H@kO2UYce_#m literal 0 HcmV?d00001 diff --git a/misc/bazel/internal/zipmerge/zipmerge_test.cpp b/misc/bazel/internal/zipmerge/zipmerge_test.cpp index 46bd55e7ffa3..5bb464284c73 100644 --- a/misc/bazel/internal/zipmerge/zipmerge_test.cpp +++ b/misc/bazel/internal/zipmerge/zipmerge_test.cpp @@ -108,35 +108,36 @@ const char* zipmerge(Args*... inputs) { } TEST(Zipmerge, Identity) { - expect_same_file(zipmerge("out.zip", "CPython.zip"), "CPython.zip"); + expect_same_file(zipmerge("out.zip", "directory.zip"), "directory.zip"); } TEST(Zipmerge, Idempotent) { - expect_same_file(zipmerge("out.zip", "CPython.zip", "CPython.zip", "CPython.zip"), "CPython.zip"); + expect_same_file(zipmerge("out.zip", "directory.zip", "directory.zip", "directory.zip"), + "directory.zip"); } TEST(Zipmerge, RemoveEverything) { - expect_same_file(zipmerge("--remove=CPython", "out.zip", "CPython.zip"), "empty.zip"); + expect_same_file(zipmerge("--remove=directory", "out.zip", "directory.zip"), "empty.zip"); } TEST(Zipmerge, RemoveEverythingWildcard) { - expect_same_file(zipmerge("--remove=*on", "out.zip", "CPython.zip"), "empty.zip"); + expect_same_file(zipmerge("--remove=*ory", "out.zip", "directory.zip"), "empty.zip"); } TEST(Zipmerge, RemovePrefixedPaths) { - expect_same_file(zipmerge("--remove=My/CPython", "out.zip", "--prefix=My", "CPython.zip"), + expect_same_file(zipmerge("--remove=My/directory", "out.zip", "--prefix=My", "directory.zip"), "empty.zip"); } TEST(Zipmerge, RemoveSome) { - expect_same_file(zipmerge("--remove=CPython/Extensions.qll", "--remove=CPython/ReturnTypeTrap.ql", - "out.zip", "CPython.zip"), - "CPython-partial.zip"); + expect_same_file( + zipmerge("--remove=directory/b.txt", "--remove=directory/c.txt", "out.zip", "directory.zip"), + "directory-partial.zip"); } TEST(Zipmerge, RemoveSomeWildcard) { - expect_same_file(zipmerge("--remove=CPython/E*.qll", "--remove=CPython/R*", "--remove=CP*l", - "out.zip", "CPython.zip"), - "CPython-partial.zip"); + expect_same_file(zipmerge("--remove=directory/b*t", "--remove=directory/c*", "--remove=dir*t", + "out.zip", "directory.zip"), + "directory-partial.zip"); } TEST(Zipmerge, Prefix) { From 5eb12b85030edb827034b9f9b73684afcfce113b Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 16:15:20 +0200 Subject: [PATCH 33/40] Zipmerge: substitute stripped down slf4j jars with dummy ones --- .../internal/zipmerge/test-files/footers.jar | Bin 0 -> 333 bytes .../internal/zipmerge/test-files/no-footers.jar | Bin 0 -> 301 bytes .../test-files/slf4j-api-classes-with-footers.jar | Bin 341 -> 0 bytes .../slf4j-api-classes-without-footers.jar | Bin 309 -> 0 bytes misc/bazel/internal/zipmerge/zipmerge_test.cpp | 3 +-- 5 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 misc/bazel/internal/zipmerge/test-files/footers.jar create mode 100644 misc/bazel/internal/zipmerge/test-files/no-footers.jar delete mode 100644 misc/bazel/internal/zipmerge/test-files/slf4j-api-classes-with-footers.jar delete mode 100644 misc/bazel/internal/zipmerge/test-files/slf4j-api-classes-without-footers.jar diff --git a/misc/bazel/internal/zipmerge/test-files/footers.jar b/misc/bazel/internal/zipmerge/test-files/footers.jar new file mode 100644 index 0000000000000000000000000000000000000000..0ed06d829770e681d356e7c6628b5e13b58532c2 GIT binary patch literal 333 zcmWIWW@Zs#;Nak3&}iHf!GHuf8CV#6T|*poJ^kGD|D9rBU}gyLX6FE@V1gyI_iXj>o-@MdHZVL){|%wZt6qXM{N kKt4g&ifk;%2(A^$9|7L1Y#>!kKv)Z;cY!zz04p3pcmMzZ literal 0 HcmV?d00001 diff --git a/misc/bazel/internal/zipmerge/test-files/no-footers.jar b/misc/bazel/internal/zipmerge/test-files/no-footers.jar new file mode 100644 index 0000000000000000000000000000000000000000..781e3bd5d72dc56ec640cea31d7d0d1af35ba2e2 GIT binary patch literal 301 zcmWIWW@Zs#VBp|j&}iHf!2kqIAOZ+D8CV#6T|*poJ^kGD|D9rBU}gwFQT)i$>yI^1 zp)C*tjX)^&b@cOea}5sB^L6{|d*-x{x31nrUT;+)Bpeg literal 0 HcmV?d00001 diff --git a/misc/bazel/internal/zipmerge/test-files/slf4j-api-classes-with-footers.jar b/misc/bazel/internal/zipmerge/test-files/slf4j-api-classes-with-footers.jar deleted file mode 100644 index b3723a437c2289aa05c0000facb0615aa7791d59..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 341 zcmWIWW@Zs#;Nak3Sdl!(jR6U8GO#fCx`sIFdiuHP|2xINz|0Wf&CUT*!30$nfK#&w zPz7AGucM!*n`>~0p0C?y-!rFuymj?1@_OrPojY@WbCAIm;|EWR^t^m^Jbf>gu43Vg zcp-UY_6)6zAL6QN&zERfJQo*zBCa}(VQTiLqK`$4U`MPxc0tM+XoU+92Y53wi7=r0 u0p>uEA5a0@Q6L|oYehB|84f!vFyKEk!c` diff --git a/misc/bazel/internal/zipmerge/test-files/slf4j-api-classes-without-footers.jar b/misc/bazel/internal/zipmerge/test-files/slf4j-api-classes-without-footers.jar deleted file mode 100644 index 498cc8966326fa7e0539d8b31381bf44a0f81274..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 309 zcmWIWW@Zs#VBp|jSdl!(jR6RlKm-tQGO#fCx`sIFdiuHP|2xINz|0VUqIl)83sTNN zg)Tr0Gyj@-BlPJ4nQ*O0?ho*0Wdo^V0>WA#y$Hl%004TLLyrIe diff --git a/misc/bazel/internal/zipmerge/zipmerge_test.cpp b/misc/bazel/internal/zipmerge/zipmerge_test.cpp index 5bb464284c73..f58660d4e1db 100644 --- a/misc/bazel/internal/zipmerge/zipmerge_test.cpp +++ b/misc/bazel/internal/zipmerge/zipmerge_test.cpp @@ -151,7 +151,6 @@ TEST(Zipmerge, InputFileOrder) { } TEST(Zipmerge, LocalFileFooters) { - expect_same_file(zipmerge("out.jar", "slf4j-api-classes-with-footers.jar"), - "slf4j-api-classes-without-footers.jar"); + expect_same_file(zipmerge("out.jar", "footers.jar"), "no-footers.jar"); } } // namespace codeql_testing From de484773f0803a89ac5eba87c264c8884f430c26 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 16:29:41 +0200 Subject: [PATCH 34/40] Zipmerge: print test outputs on CI --- .github/workflows/zipmerge-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/zipmerge-test.yml b/.github/workflows/zipmerge-test.yml index 8492b045c048..edae93a90a00 100644 --- a/.github/workflows/zipmerge-test.yml +++ b/.github/workflows/zipmerge-test.yml @@ -20,4 +20,4 @@ jobs: steps: - uses: actions/checkout@v4 - run: | - bazel test //misc/bazel/internal/zipmerge:test + bazel test //misc/bazel/internal/zipmerge:test --test_output=all From 45f1fdfaffe304e07dbd53eb046f1fe5117f828e Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 17:24:20 +0200 Subject: [PATCH 35/40] Bazel: extract pack filtering logic out of `_zipmerge` --- misc/bazel/pkg.bzl | 168 ++++++++++++++++++++++++++++++--------------- 1 file changed, 113 insertions(+), 55 deletions(-) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 0a33374fa780..7ecf4b3af785 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -9,8 +9,9 @@ load("@rules_pkg//pkg:providers.bzl", "PackageFilegroupInfo", "PackageFilesInfo" load("@rules_python//python:defs.bzl", "py_binary") def _make_internal(name): - def internal(suffix = "internal"): - return "%s-%s" % (name, suffix) + def internal(suffix = "internal", *args): + args = (name, suffix) + args + return "-".join(args) return internal @@ -133,17 +134,63 @@ _extract_pkg_filegroup = rule( } | _PLAT_DETECTION_ATTRS, ) -def _imported_zips_manifest_impl(ctx): +_ZipInfo = provider(fields = {"zips_to_prefixes": "mapping of zip files to prefixes"}) + +def _zip_info_impl(ctx): + zips = {} + for zip_target, prefix in ctx.attr.srcs.items(): + for zip in zip_target.files.to_list(): + zips[zip] = prefix + return [ + _ZipInfo(zips_to_prefixes = zips), + ] + +_zip_info = rule( + implementation = _zip_info_impl, + doc = """ + This internal rule simply instantiates a _ZipInfo provider out of `zips`. + """, + attrs = { + "srcs": attr.label_keyed_string_dict( + doc = "mapping from zip files to install prefixes", + allow_files = [".zip"], + ), + }, +) + +def _zip_info_filter_impl(ctx): platform = _detect_platform(ctx) + filtered_zips = {} + for zip_info in ctx.attr.srcs: + for zip, prefix in zip_info[_ZipInfo].zips_to_prefixes.items(): + zip_kind, expanded_prefix = _expand_path(prefix, platform) + if zip_kind == ctx.attr.kind: + filtered_zips[zip] = expanded_prefix + return [ + _ZipInfo(zips_to_prefixes = filtered_zips), + ] + +_zip_info_filter = rule( + implementation = _zip_info_filter_impl, + doc = """ + This internal rule transforms a _ZipInfo provider so that: + * only zips matching `kind` are included + * a kind of a zip is given by its prefix: if it contains {CODEQL_PLATFORM} it is arch, otherwise it's generic + * in the former case, {CODEQL_PLATFORM} is expanded + """, + attrs = { + "srcs": attr.label_list(doc = "_ZipInfos to transform", providers = [_ZipInfo]), + "kind": attr.string(doc = "Which zip kind to consider", values = ["generic", "arch"]), + } | _PLAT_DETECTION_ATTRS, +) +def _imported_zips_manifest_impl(ctx): manifest = [] files = [] - for zip, prefix in ctx.attr.zips.items(): - # we don't care about the kind here, as we're taking all zips together - _, expanded_prefix = _expand_path(prefix, platform) - zip_files = zip.files.to_list() - manifest += ["%s:%s" % (expanded_prefix, f.short_path) for f in zip_files] - files += zip_files + for zip_info in ctx.attr.srcs: + zip_info = zip_info[_ZipInfo] + manifest += ["%s:%s" % (p, z.short_path) for z, p in zip_info.zips_to_prefixes.items()] + files += list(zip_info.zips_to_prefixes) output = ctx.actions.declare_file(ctx.label.name + ".params") ctx.actions.write( @@ -162,30 +209,39 @@ _imported_zips_manifest = rule( {CODEQL_PLATFORM} can be used as zip prefixes and will be expanded to the relevant codeql platform. """, attrs = { - "zips": attr.label_keyed_string_dict( - doc = "mapping from zip files to install prefixes", - allow_files = [".zip"], + "srcs": attr.label_list( + doc = "mappings from zip files to install prefixes in _ZipInfo format", + providers = [_ZipInfo], ), - } | _PLAT_DETECTION_ATTRS, + }, ) def _zipmerge_impl(ctx): zips = [] - filename = ctx.attr.zip_name + "-" - platform = _detect_platform(ctx) - filename = "%s-%s.zip" % (ctx.attr.zip_name, platform if ctx.attr.kind == "arch" else "generic") - output = ctx.actions.declare_file(filename) - args = [output.path, "--prefix=%s" % ctx.attr.prefix, ctx.file.base.path] - for zip, prefix in ctx.attr.zips.items(): - zip_kind, expanded_prefix = _expand_path(prefix, platform) - if zip_kind == ctx.attr.kind: - args.append("--prefix=%s/%s" % (ctx.attr.prefix, expanded_prefix.rstrip("/"))) - args += [f.path for f in zip.files.to_list()] - zips.append(zip.files) + transitive_zips = [] + output = ctx.actions.declare_file(ctx.attr.out) + args = [output.path] + for zip_target in ctx.attr.srcs: + if _ZipInfo in zip_target: + zip_info = zip_target[_ZipInfo] + for zip, prefix in zip_info.zips_to_prefixes.items(): + args += [ + "--prefix=%s/%s" % (ctx.attr.prefix, prefix.rstrip("/")), + zip.path, + ] + zips.append(zip) + else: + zips = zip_target.files.to_list() + for zip in zips: + if zip.extension != "zip": + fail("%s file found while expecting a .zip file " % zip.short_path) + args.append("--prefix=%s" % ctx.attr.prefix) + args += [z.path for z in zips] + transitive_zips.append(zip_target.files) ctx.actions.run( outputs = [output], executable = ctx.executable._zipmerge, - inputs = depset([ctx.file.base], transitive = zips), + inputs = depset(zips, transitive = transitive_zips), arguments = args, ) @@ -196,30 +252,22 @@ def _zipmerge_impl(ctx): _zipmerge = rule( implementation = _zipmerge_impl, doc = """ - This internal rule merges a `base` zip file with the ones indicated by the `zips` mapping where the prefix - indicates a matching kind between arch and generic. An imported zip file will be considered arch-specific - if its prefix contains `{CODEQL_PLATFORM}` (and this prefix will have that expanded to the appropriate - platform). - - The output filename will be either `{zip_name}-generic.zip` or `{zip_name}-{CODEQL_PLATFORM}.zip`, depending on - the requested `kind`. + This internal rule merges a zip files together """, attrs = { - "base": attr.label( - doc = "Base zip file to which zips from `zips` will be merged with", - allow_single_file = [".zip"], - ), - "zips": attr.label_keyed_string_dict( - doc = "mapping from zip files to install prefixes", - allow_files = [".zip"], - ), - "zip_name": attr.string(doc = "Prefix to use for the output file name"), - "kind": attr.string(doc = "Which zip kind to consider", values = ["generic", "arch"]), + "srcs": attr.label_list(doc = "Zip file to include, either as straight up `.zip` files or `_ZipInfo` data"), + "out": attr.string(doc = "output file name"), "prefix": attr.string(doc = "Prefix posix path to add to the zip contents in the archive"), "_zipmerge": attr.label(default = "//misc/bazel/internal/zipmerge", executable = True, cfg = "exec"), - } | _PLAT_DETECTION_ATTRS, + }, ) +def _get_zip_filename(name_prefix, kind): + if kind == "arch": + return name_prefix + "-" + _detect_platform() + ".zip" # using + because there's a select + else: + return "%s-generic.zip" % name_prefix + def codeql_pack( *, name, @@ -252,47 +300,57 @@ def codeql_pack( zip_filename = zip_filename or name zips = zips or {} pkg_filegroup( - name = internal("base"), + name = internal("all"), srcs = srcs, visibility = ["//visibility:private"], **kwargs ) + if zips: + _zip_info( + name = internal("zip-info"), + srcs = zips, + visibility = ["//visibility:private"], + ) for kind in ("generic", "arch"): _extract_pkg_filegroup( name = internal(kind), - src = internal("base"), + src = internal("all"), kind = kind, visibility = ["//visibility:private"], ) if zips: pkg_zip( - name = internal(kind + "-zip-base"), + name = internal(kind, "zip-base"), srcs = [internal(kind)], visibility = ["//visibility:private"], compression_level = compression_level, ) - _zipmerge( - name = internal(kind + "-zip"), - base = internal(kind + "-zip-base"), - zips = zips, - zip_name = zip_filename, + _zip_info_filter( + name = internal(kind, "zip-info"), kind = kind, + srcs = [internal("zip-info")], + visibility = ["//visibility:private"], + ) + _zipmerge( + name = internal(kind, "zip"), + srcs = [internal(kind, "zip-base"), internal(kind, "zip-info")], + out = _get_zip_filename(name, kind), prefix = name, visibility = visibility, ) else: pkg_zip( - name = internal(kind + "-zip"), + name = internal(kind, "zip"), srcs = [internal(kind)], - visibility = ["//visibility:private"], + visibility = visibility, package_dir = name, - package_file_name = name + "-" + (_detect_platform() if kind == "arch" else "generic") + ".zip", + package_file_name = _get_zip_filename(name, kind), compression_level = compression_level, ) if zips: _imported_zips_manifest( name = internal("zip-manifest"), - zips = zips, + srcs = [internal("generic-zip-info"), internal("arch-zip-info")], visibility = ["//visibility:private"], ) From 332d17829866db896ee99d1857b281cd0f5e411b Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Tue, 28 May 2024 17:37:34 +0200 Subject: [PATCH 36/40] Zipmerge: allow test to be run from internal repo --- misc/bazel/internal/zipmerge/zipmerge_test.cpp | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/misc/bazel/internal/zipmerge/zipmerge_test.cpp b/misc/bazel/internal/zipmerge/zipmerge_test.cpp index f58660d4e1db..ee30b764a6d2 100644 --- a/misc/bazel/internal/zipmerge/zipmerge_test.cpp +++ b/misc/bazel/internal/zipmerge/zipmerge_test.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -12,6 +13,7 @@ using bazel::tools::cpp::runfiles::Runfiles; using namespace std::string_literals; +namespace fs = std::filesystem; namespace codeql_testing { @@ -72,7 +74,15 @@ std::string get_file(const char* name) { EXPECT_TRUE(ret) << error; return ret; }(); - return runfiles->Rlocation("_main/misc/bazel/internal/zipmerge/test-files/"s + name); + // this works from both `codeql` and the internal repository + for (auto prefix : {"_main", "codeql~"}) { + auto ret = runfiles->Rlocation(prefix + "/misc/bazel/internal/zipmerge/test-files/"s + name); + if (fs::exists(ret)) { + return ret; + } + } + EXPECT_TRUE(false) << "test file " << name << " not found"; + return ""; } void expect_same_file(const char* actual, const char* expected) { From fbe1b56f2d65542f398898c69192d4a6f084ee9d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 29 May 2024 08:55:06 +0200 Subject: [PATCH 37/40] Zipmerge: link test statically --- misc/bazel/internal/zipmerge/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/misc/bazel/internal/zipmerge/BUILD.bazel b/misc/bazel/internal/zipmerge/BUILD.bazel index d46bf74cc1e2..c5a5a004e1bd 100644 --- a/misc/bazel/internal/zipmerge/BUILD.bazel +++ b/misc/bazel/internal/zipmerge/BUILD.bazel @@ -20,6 +20,7 @@ cc_binary( cc_test( name = "test", size = "small", + linkstatic = True, # required to build the test in the internal repo srcs = ["zipmerge_test.cpp"], data = glob(["test-files/*"]), deps = [ From 5672ddf8f3c30b376eb58988e6b983daec748933 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 29 May 2024 09:53:31 +0200 Subject: [PATCH 38/40] Fix bazel formatting --- misc/bazel/internal/zipmerge/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/bazel/internal/zipmerge/BUILD.bazel b/misc/bazel/internal/zipmerge/BUILD.bazel index c5a5a004e1bd..cae83d529211 100644 --- a/misc/bazel/internal/zipmerge/BUILD.bazel +++ b/misc/bazel/internal/zipmerge/BUILD.bazel @@ -20,9 +20,9 @@ cc_binary( cc_test( name = "test", size = "small", - linkstatic = True, # required to build the test in the internal repo srcs = ["zipmerge_test.cpp"], data = glob(["test-files/*"]), + linkstatic = True, # required to build the test in the internal repo deps = [ ":lib", "@bazel_tools//tools/cpp/runfiles", From e8061ecd3881e2d7c8f0b637e7465c2d1a2b9b33 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 29 May 2024 11:59:18 +0200 Subject: [PATCH 39/40] Bazel: fix `_zipmerge` rule --- misc/bazel/pkg.bzl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 7ecf4b3af785..709ba303acf7 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -231,12 +231,12 @@ def _zipmerge_impl(ctx): ] zips.append(zip) else: - zips = zip_target.files.to_list() - for zip in zips: + zip_files = zip_target.files.to_list() + for zip in zip_files: if zip.extension != "zip": fail("%s file found while expecting a .zip file " % zip.short_path) args.append("--prefix=%s" % ctx.attr.prefix) - args += [z.path for z in zips] + args += [z.path for z in zip_files] transitive_zips.append(zip_target.files) ctx.actions.run( outputs = [output], From 336ec089ccd266b89be97c233ca2b472b9913596 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 29 May 2024 12:02:02 +0200 Subject: [PATCH 40/40] Bazel: use `extend(...)` instead of `+= list(...)` --- misc/bazel/pkg.bzl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index 709ba303acf7..fdfdb6be746b 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -190,7 +190,7 @@ def _imported_zips_manifest_impl(ctx): for zip_info in ctx.attr.srcs: zip_info = zip_info[_ZipInfo] manifest += ["%s:%s" % (p, z.short_path) for z, p in zip_info.zips_to_prefixes.items()] - files += list(zip_info.zips_to_prefixes) + files.extend(zip_info.zips_to_prefixes) output = ctx.actions.declare_file(ctx.label.name + ".params") ctx.actions.write(