diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl index a5e8ef3eac2a3..a12234813c326 100644 --- a/misc/bazel/pkg.bzl +++ b/misc/bazel/pkg.bzl @@ -5,11 +5,11 @@ Wrappers and helpers around `rules_pkg` to build codeql packs. load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files", _strip_prefix = "strip_prefix") load("@rules_pkg//pkg:pkg.bzl", "pkg_zip") +load("@rules_pkg//pkg:providers.bzl", "PackageFilegroupInfo", "PackageFilesInfo") load("@rules_python//python:defs.bzl", "py_binary") -load("//:defs.bzl", "codeql_platform") def _make_internal(name): - def internal(suffix): + def internal(suffix = "internal"): return "%s-%s" % (name, suffix) return internal @@ -20,151 +20,166 @@ def _get_subrule(label, suffix): path, _, pkg = label.rpartition("/") return "%s/%s:%s-%s" % (path, pkg, pkg, suffix) +CodeqlFilesInfo = provider( + doc = """Wrapper around `rules_pkg` `PackageFilesInfo` splitting into generic and arch-specific files.""", + fields = { + "generic": "list of `(PackageFilesInfo, Label)` tuples for generic files.", + "arch": "list of `(PackageFilesInfo, Label)` tuples for arch-specific files.", + }, +) + +def _codeql_pkg_filegroup_impl(ctx): + if ctx.target_platform_has_constraint(ctx.attr._windows[platform_common.ConstraintValueInfo]): + plat = "windows64" + elif ctx.target_platform_has_constraint(ctx.attr._macos[platform_common.ConstraintValueInfo]): + plat = "osx64" + else: + plat = "linux64" + generic_prefix = ctx.attr.prefix + if generic_prefix: + generic_prefix += "/" + arch_prefix = generic_prefix + plat + "/" + + def transform_pfi_lbl(pfi_lbl, prefix): + pfi, lbl = pfi_lbl + return (PackageFilesInfo( + attributes = pfi.attributes, + dest_src_map = {prefix + d: s for d, s in pfi.dest_src_map.items()}, + ), lbl) + + generic = [] + arch = [] + for dest, prefix, srcs in ( + (generic, generic_prefix, ctx.attr.srcs), + (arch, arch_prefix, ctx.attr.arch_srcs), + ): + for src in srcs: + if PackageFilesInfo in src: + dest.append(transform_pfi_lbl((src[PackageFilesInfo], src.label), prefix)) + elif PackageFilegroupInfo in src: + pfgi = src[PackageFilegroupInfo] + if pfgi.pkg_dirs or pfgi.pkg_symlinks: + fail("while assembling %s found %s which contains `pkg_dirs` or `pkg_symlinks` targets" % + (ctx.label, src.label)) + dest += [transform_pfi_lbl(item, prefix) for item in pfgi.pkg_files] + else: + cfi = src[CodeqlFilesInfo] + generic += [transform_pfi_lbl(item, generic_prefix) for item in cfi.generic] + + # arch-specific prefix was already applied + arch += [transform_pfi_lbl(item, generic_prefix) for item in cfi.arch] + + srcs = ctx.attr.srcs + ctx.attr.arch_srcs + return [ + CodeqlFilesInfo( + generic = generic, + arch = arch, + ), + DefaultInfo( + files = depset(transitive = [src[DefaultInfo].files for src in srcs]), + ), + ] + +codeql_pkg_filegroup = rule( + implementation = _codeql_pkg_filegroup_impl, + doc = """CodeQL specific packaging mapping. No `pkg_mkdirs` or `pkg_symlink` rules are supported, either directly + or transitively. Only `pkg_files` and `pkg_filegroup` thereof are allowed.""", + attrs = { + "srcs": attr.label_list( + doc = "List of arch-agnostic `pkg_files`, `pkg_filegroup` or `codeql_pkg_filegroup` targets", + providers = [DefaultInfo], + default = [], + ), + "arch_srcs": attr.label_list( + doc = "List of arch-specific `pkg_files` or `pkg_filegroup` targets", + providers = [DefaultInfo], + default = [], + ), + "prefix": attr.string(doc = "Prefix to add to the files"), + "_windows": attr.label(default = "@platforms//os:windows"), + "_macos": attr.label(default = "@platforms//os:macos"), + }, +) + def codeql_pkg_files( *, name, - arch_specific = False, srcs = None, exes = None, - renames = None, prefix = None, visibility = None, **kwargs): """ Wrapper around `pkg_files`. Added functionality: - * `exes` get their file attributes set to be executable. This is important only for POSIX files, there's no need - to mark windows executables as such - * `arch_specific` auto-adds the codeql platform specific directory (linux64, osx64 or windows64), and will be - consumed by a downstream `codeql_pack` to create the arch specific zip. - This should be consumed by `codeql_pkg_filegroup` and `codeql_pack` only. + * `exes` will get their file attributes set to be executable. This is important only for POSIX files, there's no + need to mark windows executables as such + If `exes` and `srcs` are both used, the resulting rule is a `pkg_filegroup` one. """ internal = _make_internal(name) - main_rule = internal("generic") - empty_rule = internal("arch") - if arch_specific: - main_rule, empty_rule = empty_rule, main_rule - prefix = (prefix + "/" if prefix else "") + codeql_platform - pkg_files( - name = empty_rule, - srcs = [], - visibility = visibility, - ) - if not srcs and not exes: - fail("either srcs or exes should be specified for %s" % name) + if "attributes" in kwargs: + fail("codeql_pkg_files does not support `attributes`. Use `exes` to mark executable files.") + internal_srcs = [] if srcs and exes: - if renames: - src_renames = {k: v for k, v in renames.items() if k in srcs} - exe_renames = {k: v for k, v in renames.items() if k in exes} - else: - src_renames = None - exe_renames = None pkg_files( name = internal("srcs"), srcs = srcs, - renames = src_renames, - prefix = prefix, visibility = ["//visibility:private"], **kwargs ) pkg_files( name = internal("exes"), srcs = exes, - renames = exe_renames, - prefix = prefix, visibility = ["//visibility:private"], - attributes = pkg_attributes(mode = "0755"), + attributes = pkg_attributes(mode = "755"), **kwargs ) pkg_filegroup( - name = main_rule, + name = name, srcs = [internal("srcs"), internal("exes")], + prefix = prefix, visibility = visibility, ) else: pkg_files( - name = main_rule, + name = name, srcs = srcs or exes, - attributes = pkg_attributes(mode = "0755") if exes else None, - prefix = prefix, visibility = visibility, + prefix = prefix, + attributes = pkg_attributes(mode = "755") if exes else None, **kwargs ) - native.filegroup( - name = name, - srcs = [main_rule], - visibility = visibility, - ) -def codeql_pkg_wrap(*, name, srcs, arch_specific = False, prefix = None, visibility = None, **kwargs): - """ - Wrap a native `rules_pkg` rule, providing the `arch_specific` functionality and making it consumable by - `codeql_pkg_filegroup` and `codeql_pack`. - """ - internal = _make_internal(name) - main_rule = internal("generic") - empty_rule = internal("arch") - if arch_specific: - main_rule, empty_rule = empty_rule, main_rule - prefix = (prefix + "/" if prefix else "") + codeql_platform - pkg_filegroup( - name = main_rule, - srcs = srcs, - prefix = prefix, - visibility = visibility, - **kwargs - ) - pkg_files( - name = empty_rule, - srcs = [], - visibility = visibility, - ) - native.filegroup( - name = name, - srcs = [main_rule], - visibility = visibility, - ) +def _extract_pkg_filegroup_impl(ctx): + src = ctx.attr.src[CodeqlFilesInfo] + pfi_lbl_list = src.arch if ctx.attr.arch_specific else src.generic + files = [] + for pfi, lbl in pfi_lbl_list: + files.append(depset(pfi.dest_src_map.values())) + return [ + PackageFilegroupInfo(pkg_files = pfi_lbl_list, pkg_dirs = [], pkg_symlinks = []), + DefaultInfo(files = depset(transitive = files)), + ] -def codeql_pkg_filegroup( +_extrac_pkg_filegroup = rule( + implementation = _extract_pkg_filegroup_impl, + attrs = { + "src": attr.label(providers = [CodeqlFilesInfo, DefaultInfo]), + "arch_specific": attr.bool(), + }, +) + +def codeql_pack( *, name, - srcs, - srcs_select = None, + srcs = None, + arch_srcs = None, + zip_prefix = None, + zip_filename = "extractor", visibility = None, + install_dest = "extractor-pack", **kwargs): """ - Combine `codeql_pkg_files` and other `codeql_pkg_filegroup` rules, similar to what `pkg_filegroup` does. - `srcs` is not selectable, but `srcs_select` accepts the same dictionary that a select would accept. - """ - internal = _make_internal(name) - - def transform(srcs, suffix): - return [_get_subrule(src, suffix) for src in srcs] - - pkg_filegroup( - name = internal("generic"), - srcs = transform(srcs, "generic") + (select( - {k: transform(v, "generic") for k, v in srcs_select.items()}, - ) if srcs_select else []), - visibility = visibility, - **kwargs - ) - pkg_filegroup( - name = internal("arch"), - srcs = transform(srcs, "arch") + (select( - {k: transform(v, "arch") for k, v in srcs_select.items()}, - ) if srcs_select else []), - visibility = visibility, - **kwargs - ) - native.filegroup( - name = name, - srcs = [internal("generic"), internal("arch")], - visibility = visibility, - ) - -def codeql_pack(*, name, srcs, zip_prefix = None, zip_filename = "extractor", visibility = visibility, install_dest = "extractor-pack", **kwargs): - """ - Define a codeql pack. This accepts the same arguments as `codeql_pkg_filegroup`, and additionally: + Define a codeql pack. This accepts `pkg_files`, `pkg_filegroup` or their `codeql_*` counterparts as `srcs`. * defines a `-generic-zip` target creating a `-generic.zip` archive with the generic bits, prefixed with `zip_prefix` (`name` by default) * defines a `-arch-zip` target creating a `-.zip` archive with the @@ -179,27 +194,34 @@ def codeql_pack(*, name, srcs, zip_prefix = None, zip_filename = "extractor", vi codeql_pkg_filegroup( name = name, srcs = srcs, + arch_srcs = arch_srcs, visibility = visibility, **kwargs ) - codeql_pkg_filegroup( - name = internal("zip-contents"), - srcs = [name], - prefix = zip_prefix, + _extrac_pkg_filegroup( + name = internal("generic"), + src = name, + arch_specific = False, visibility = ["//visibility:private"], ) - pkg_zip( - name = internal("generic-zip"), - srcs = [internal("zip-contents-generic")], - package_file_name = zip_filename + "-generic.zip", - visibility = visibility, - ) - pkg_zip( - name = internal("arch-zip"), - srcs = [internal("zip-contents-arch")], - package_file_name = zip_filename + "-" + codeql_platform + ".zip", - visibility = visibility, + _extrac_pkg_filegroup( + name = internal("arch"), + src = name, + arch_specific = True, + visibility = ["//visibility:private"], ) + for kind in ("generic", "arch"): + pkg_filegroup( + name = internal(kind + "-zip-contents"), + srcs = [internal(kind)], + visibility = ["//visibility:private"], + ) + pkg_zip( + name = internal(kind + "-zip"), + srcs = [internal(kind + "-zip-contents")], + package_file_name = "%s-%s.zip" % (zip_filename, kind), + visibility = visibility, + ) pkg_install( name = internal("script"), srcs = [internal("generic"), internal("arch")], diff --git a/swift/BUILD.bazel b/swift/BUILD.bazel index 7735da4fa6995..31aa754a1a090 100644 --- a/swift/BUILD.bazel +++ b/swift/BUILD.bazel @@ -4,7 +4,6 @@ load( "codeql_pkg_filegroup", "codeql_pkg_files", "codeql_pkg_runfiles", - "codeql_pkg_wrap", ) filegroup( @@ -25,69 +24,60 @@ filegroup( visibility = ["//swift:__subpackages__"], ) -codeql_pkg_files( - name = "dbscheme_files", - srcs = [ - "ql/lib/swift.dbscheme.stats", - "//swift/extractor/trap:generated_dbscheme", - ], -) - -codeql_pkg_files( - name = "manifest", - srcs = ["codeql-extractor.yml"], -) - -codeql_pkg_filegroup( - name = "extractor", - srcs = ["//swift/extractor:pkg"], - prefix = "tools", -) - codeql_pkg_files( name = "autobuilder-incompatible-os", - arch_specific = True, exes = ["//swift/tools/diagnostics:autobuilder-incompatible-os"], - prefix = "tools", ) codeql_pkg_runfiles( name = "autobuilder", - arch_specific = True, exes = ["//swift/swift-autobuilder"], - prefix = "tools", ) -codeql_pkg_wrap( - name = "resource-dir", - srcs = ["//swift/third_party/swift-llvm-support:swift-resource-dir"], - arch_specific = True, - prefix = "resource-dir", -) - -codeql_pack( - name = "swift", +codeql_pkg_filegroup( + name = "tools", srcs = [ - ":dbscheme_files", - ":manifest", - "//swift/downgrades", "//swift/tools", ], - srcs_select = { + arch_srcs = select({ "@platforms//os:macos": [ - ":extractor", - ":resource-dir", ":autobuilder", + "//swift/extractor:pkg", ], "@platforms//os:linux": [ - ":extractor", - ":resource-dir", ":autobuilder-incompatible-os", + "//swift/extractor:pkg", ], "@platforms//os:windows": [ ":autobuilder-incompatible-os", ], - }, + }), + prefix = "tools", +) + +codeql_pkg_filegroup( + name = "resource-dir", + arch_srcs = ["//swift/third_party/swift-llvm-support:swift-resource-dir"], + prefix = "resource-dir", +) + +codeql_pkg_files( + name = "root-files", + srcs = [ + "codeql-extractor.yml", + "ql/lib/swift.dbscheme.stats", + "//swift/extractor/trap:generated_dbscheme", + ], +) + +codeql_pack( + name = "swift", + srcs = [ + ":resource-dir", + ":root-files", + ":tools", + "//swift/downgrades", + ], visibility = ["//visibility:public"], ) diff --git a/swift/extractor/BUILD.bazel b/swift/extractor/BUILD.bazel index 02b1a91e3f9a4..8290aec41216d 100644 --- a/swift/extractor/BUILD.bazel +++ b/swift/extractor/BUILD.bazel @@ -31,7 +31,6 @@ sh_binary( codeql_pkg_runfiles( name = "pkg", - arch_specific = True, excludes = ["extractor.sh"], # script gets copied as "extractor", no need for the original .sh file exes = [":extractor"], visibility = ["//swift:__pkg__"], diff --git a/swift/tools/BUILD.bazel b/swift/tools/BUILD.bazel index 5c08835fe3523..ae04a587a9440 100644 --- a/swift/tools/BUILD.bazel +++ b/swift/tools/BUILD.bazel @@ -27,6 +27,5 @@ codeql_pkg_files( ":identify-environment", ":qltest", ], - prefix = "tools", visibility = ["//swift:__pkg__"], )