diff --git a/misc/bazel/internal/BUILD.bazel b/misc/bazel/internal/BUILD.bazel index e69de29bb2d1..d9663e7f0c01 100644 --- a/misc/bazel/internal/BUILD.bazel +++ b/misc/bazel/internal/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["install.py"]) diff --git a/misc/bazel/internal/install.py b/misc/bazel/internal/install.py new file mode 100644 index 000000000000..f57cae04596c --- /dev/null +++ b/misc/bazel/internal/install.py @@ -0,0 +1,25 @@ +import argparse +import pathlib +import shutil +import subprocess +from python.runfiles import runfiles + +runfiles = runfiles.Create() +if not runfiles: + raise Exception("Installer should be run with `bazel run`") + +parser = argparse.ArgumentParser() +parser.add_argument("--destdir", type=pathlib.Path, required=True) +parser.add_argument("--script", required=True) +parser.add_argument("--build-file", required=True) +opts = parser.parse_args() + +script = runfiles.Rlocation(opts.script) +build_file = runfiles.Rlocation(opts.build_file) +destdir = pathlib.Path(build_file).parent / opts.destdir + +if destdir.exists(): + shutil.rmtree(destdir) + +destdir.mkdir(parents=True) +subprocess.run([script, "--destdir", destdir], check=True) diff --git a/misc/bazel/pkg.bzl b/misc/bazel/pkg.bzl new file mode 100644 index 000000000000..a5e8ef3eac2a --- /dev/null +++ b/misc/bazel/pkg.bzl @@ -0,0 +1,264 @@ +""" +Wrappers and helpers around `rules_pkg` to build codeql packs. +""" + +load("@rules_pkg//pkg:install.bzl", "pkg_install") +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files", _strip_prefix = "strip_prefix") +load("@rules_pkg//pkg:pkg.bzl", "pkg_zip") +load("@rules_python//python:defs.bzl", "py_binary") +load("//:defs.bzl", "codeql_platform") + +def _make_internal(name): + def internal(suffix): + return "%s-%s" % (name, suffix) + + return internal + +def _get_subrule(label, suffix): + if ":" in label or "/" not in label: + return "%s-%s" % (label, suffix) + path, _, pkg = label.rpartition("/") + return "%s/%s:%s-%s" % (path, pkg, pkg, suffix) + +def codeql_pkg_files( + *, + name, + arch_specific = False, + srcs = None, + exes = None, + renames = None, + prefix = None, + visibility = None, + **kwargs): + """ + Wrapper around `pkg_files`. Added functionality: + * `exes` get their file attributes set to be executable. This is important only for POSIX files, there's no need + to mark windows executables as such + * `arch_specific` auto-adds the codeql platform specific directory (linux64, osx64 or windows64), and will be + consumed by a downstream `codeql_pack` to create the arch specific zip. + This should be consumed by `codeql_pkg_filegroup` and `codeql_pack` only. + """ + internal = _make_internal(name) + main_rule = internal("generic") + empty_rule = internal("arch") + if arch_specific: + main_rule, empty_rule = empty_rule, main_rule + prefix = (prefix + "/" if prefix else "") + codeql_platform + pkg_files( + name = empty_rule, + srcs = [], + visibility = visibility, + ) + if not srcs and not exes: + fail("either srcs or exes should be specified for %s" % name) + if srcs and exes: + if renames: + src_renames = {k: v for k, v in renames.items() if k in srcs} + exe_renames = {k: v for k, v in renames.items() if k in exes} + else: + src_renames = None + exe_renames = None + pkg_files( + name = internal("srcs"), + srcs = srcs, + renames = src_renames, + prefix = prefix, + visibility = ["//visibility:private"], + **kwargs + ) + pkg_files( + name = internal("exes"), + srcs = exes, + renames = exe_renames, + prefix = prefix, + visibility = ["//visibility:private"], + attributes = pkg_attributes(mode = "0755"), + **kwargs + ) + pkg_filegroup( + name = main_rule, + srcs = [internal("srcs"), internal("exes")], + visibility = visibility, + ) + else: + pkg_files( + name = main_rule, + srcs = srcs or exes, + attributes = pkg_attributes(mode = "0755") if exes else None, + prefix = prefix, + visibility = visibility, + **kwargs + ) + native.filegroup( + name = name, + srcs = [main_rule], + visibility = visibility, + ) + +def codeql_pkg_wrap(*, name, srcs, arch_specific = False, prefix = None, visibility = None, **kwargs): + """ + Wrap a native `rules_pkg` rule, providing the `arch_specific` functionality and making it consumable by + `codeql_pkg_filegroup` and `codeql_pack`. + """ + internal = _make_internal(name) + main_rule = internal("generic") + empty_rule = internal("arch") + if arch_specific: + main_rule, empty_rule = empty_rule, main_rule + prefix = (prefix + "/" if prefix else "") + codeql_platform + pkg_filegroup( + name = main_rule, + srcs = srcs, + prefix = prefix, + visibility = visibility, + **kwargs + ) + pkg_files( + name = empty_rule, + srcs = [], + visibility = visibility, + ) + native.filegroup( + name = name, + srcs = [main_rule], + visibility = visibility, + ) + +def codeql_pkg_filegroup( + *, + name, + srcs, + srcs_select = None, + visibility = None, + **kwargs): + """ + Combine `codeql_pkg_files` and other `codeql_pkg_filegroup` rules, similar to what `pkg_filegroup` does. + `srcs` is not selectable, but `srcs_select` accepts the same dictionary that a select would accept. + """ + internal = _make_internal(name) + + def transform(srcs, suffix): + return [_get_subrule(src, suffix) for src in srcs] + + pkg_filegroup( + name = internal("generic"), + srcs = transform(srcs, "generic") + (select( + {k: transform(v, "generic") for k, v in srcs_select.items()}, + ) if srcs_select else []), + visibility = visibility, + **kwargs + ) + pkg_filegroup( + name = internal("arch"), + srcs = transform(srcs, "arch") + (select( + {k: transform(v, "arch") for k, v in srcs_select.items()}, + ) if srcs_select else []), + visibility = visibility, + **kwargs + ) + native.filegroup( + name = name, + srcs = [internal("generic"), internal("arch")], + visibility = visibility, + ) + +def codeql_pack(*, name, srcs, zip_prefix = None, zip_filename = "extractor", visibility = visibility, install_dest = "extractor-pack", **kwargs): + """ + Define a codeql pack. This accepts the same arguments as `codeql_pkg_filegroup`, and additionally: + * defines a `-generic-zip` target creating a `-generic.zip` archive with the generic bits, + prefixed with `zip_prefix` (`name` by default) + * defines a `-arch-zip` target creating a `-.zip` archive with the + arch-specific bits, prefixed with `zip_prefix` (`name` by default) + * defines a runnable `-installer` target that will install the pack in `install_dest`, relative to where the + rule is used. The install destination can be overridden appending `-- --destdir=...` to the `bazel run` + invocation. This installation does not use the `zip_prefix`. + """ + internal = _make_internal(name) + zip_prefix = zip_prefix or name + zip_filename = zip_filename or name + codeql_pkg_filegroup( + name = name, + srcs = srcs, + visibility = visibility, + **kwargs + ) + codeql_pkg_filegroup( + name = internal("zip-contents"), + srcs = [name], + prefix = zip_prefix, + visibility = ["//visibility:private"], + ) + pkg_zip( + name = internal("generic-zip"), + srcs = [internal("zip-contents-generic")], + package_file_name = zip_filename + "-generic.zip", + visibility = visibility, + ) + pkg_zip( + name = internal("arch-zip"), + srcs = [internal("zip-contents-arch")], + package_file_name = zip_filename + "-" + codeql_platform + ".zip", + visibility = visibility, + ) + pkg_install( + name = internal("script"), + srcs = [internal("generic"), internal("arch")], + visibility = ["//visibility:private"], + ) + native.filegroup( + # used to locate current source directory + name = internal("build-file"), + srcs = ["BUILD.bazel"], + visibility = ["//visibility:private"], + ) + py_binary( + name = internal("installer"), + srcs = ["//misc/bazel/internal:install.py"], + main = "//misc/bazel/internal:install.py", + data = [internal("build-file"), internal("script")], + deps = ["@rules_python//python/runfiles"], + args = [ + "--build-file=$(rlocationpath %s)" % internal("build-file"), + "--script=$(rlocationpath %s)" % internal("script"), + "--destdir", + install_dest, + ], + visibility = visibility, + ) + +strip_prefix = _strip_prefix + +def _runfiles_group_impl(ctx): + files = [] + for src in ctx.attr.srcs: + rf = src[DefaultInfo].default_runfiles + if rf != None: + files.append(rf.files) + return [ + DefaultInfo( + files = depset(transitive = files), + ), + ] + +_runfiles_group = rule( + implementation = _runfiles_group_impl, + attrs = { + "srcs": attr.label_list(), + }, +) + +def codeql_pkg_runfiles(*, name, exes, **kwargs): + """ + Create a `codeql_pkg_files` with all runfiles from files in `exes`, flattened together. + """ + internal = _make_internal(name) + _runfiles_group( + name = internal("runfiles"), + srcs = exes, + visibility = ["//visibility:private"], + ) + codeql_pkg_files( + name = name, + exes = [internal("runfiles")], + **kwargs + ) diff --git a/misc/bazel/pkg_runfiles.bzl b/misc/bazel/pkg_runfiles.bzl deleted file mode 100644 index 3d3bd8c028d5..000000000000 --- a/misc/bazel/pkg_runfiles.bzl +++ /dev/null @@ -1,33 +0,0 @@ -load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files") - -def _runfiles_group_impl(ctx): - files = [] - for src in ctx.attr.srcs: - rf = src[DefaultInfo].default_runfiles - if rf != None: - files.append(rf.files) - return [ - DefaultInfo( - files = depset(transitive = files), - ), - ] - -_runfiles_group = rule( - implementation = _runfiles_group_impl, - attrs = { - "srcs": attr.label_list(), - }, -) - -def pkg_runfiles(*, name, srcs, **kwargs): - internal_name = "_%s_runfiles" % name - _runfiles_group( - name = internal_name, - srcs = srcs, - ) - kwargs.setdefault("attributes", pkg_attributes(mode = "0755")) - pkg_files( - name = name, - srcs = [internal_name], - **kwargs - ) diff --git a/swift/BUILD.bazel b/swift/BUILD.bazel index 1ced5c9f1ca0..7735da4fa699 100644 --- a/swift/BUILD.bazel +++ b/swift/BUILD.bazel @@ -1,8 +1,11 @@ -load("@rules_pkg//pkg:install.bzl", "pkg_install") -load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files") -load("//:defs.bzl", "codeql_platform") -load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles") -load("//misc/bazel/cmake:cmake.bzl", "generate_cmake") +load( + "//misc/bazel:pkg.bzl", + "codeql_pack", + "codeql_pkg_filegroup", + "codeql_pkg_files", + "codeql_pkg_runfiles", + "codeql_pkg_wrap", +) filegroup( name = "schema", @@ -22,7 +25,7 @@ filegroup( visibility = ["//swift:__subpackages__"], ) -pkg_files( +codeql_pkg_files( name = "dbscheme_files", srcs = [ "ql/lib/swift.dbscheme.stats", @@ -30,100 +33,78 @@ pkg_files( ], ) -pkg_files( +codeql_pkg_files( name = "manifest", srcs = ["codeql-extractor.yml"], ) -pkg_filegroup( - name = "extractor-pack-generic", - srcs = [ - ":manifest", - "//swift/tools", - ] + select({ - "@platforms//os:windows": [], - "//conditions:default": [ - ":dbscheme_files", - "//swift/downgrades", - ], - }), - visibility = ["//visibility:public"], -) - -pkg_filegroup( +codeql_pkg_filegroup( name = "extractor", srcs = ["//swift/extractor:pkg"], - prefix = "tools/" + codeql_platform, + prefix = "tools", ) -pkg_runfiles( - name = "swift-autobuilder", - srcs = ["//swift/swift-autobuilder"], - prefix = "tools/" + codeql_platform, +codeql_pkg_files( + name = "autobuilder-incompatible-os", + arch_specific = True, + exes = ["//swift/tools/diagnostics:autobuilder-incompatible-os"], + prefix = "tools", ) -pkg_runfiles( - name = "diagnostics", - srcs = ["//swift/tools/diagnostics:autobuilder-incompatible-os"], - prefix = "tools/" + codeql_platform, +codeql_pkg_runfiles( + name = "autobuilder", + arch_specific = True, + exes = ["//swift/swift-autobuilder"], + prefix = "tools", ) -pkg_filegroup( - name = "resource-dir-arch", +codeql_pkg_wrap( + name = "resource-dir", srcs = ["//swift/third_party/swift-llvm-support:swift-resource-dir"], - prefix = "resource-dir/" + codeql_platform, - visibility = ["//visibility:public"], + arch_specific = True, + prefix = "resource-dir", ) -pkg_filegroup( - name = "extractor-pack-arch", - srcs = select({ - "@platforms//os:windows": [], - "//conditions:default": [ +codeql_pack( + name = "swift", + srcs = [ + ":dbscheme_files", + ":manifest", + "//swift/downgrades", + "//swift/tools", + ], + srcs_select = { + "@platforms//os:macos": [ ":extractor", - ":resource-dir-arch", + ":resource-dir", + ":autobuilder", ], - }) + select({ - "@platforms//os:macos": [ - ":swift-autobuilder", + "@platforms//os:linux": [ + ":extractor", + ":resource-dir", + ":autobuilder-incompatible-os", ], - "//conditions:default": [ - ":diagnostics", + "@platforms//os:windows": [ + ":autobuilder-incompatible-os", ], - }), - visibility = ["//visibility:public"], -) - -pkg_filegroup( - name = "extractor-pack", - srcs = [ - ":extractor-pack-arch", - ":extractor-pack-generic", - ], + }, visibility = ["//visibility:public"], ) -pkg_install( - name = "_create_extractor_pack", - srcs = ["//swift:extractor-pack"], +alias( + name = "create-extractor-pack", + actual = ":swift-installer", ) -py_binary( - name = "create-extractor-pack", - srcs = ["create_extractor_pack.py"], - main = "create_extractor_pack.py", - deps = [":_create_extractor_pack"], +# TODO: aliases for internal repo backward compatibility +alias( + name = "extractor-pack-generic", + actual = "swift-generic", + visibility = ["//visibility:public"], ) -# TODO this is unneeded here but still used in the internal repo. Remove once it's not -generate_cmake( - name = "cmake", - targets = [ - "//swift/extractor:extractor.real", - "//swift/logging/tests/assertion-diagnostics:assert-false", - ] + select({ - "@platforms//os:linux": ["//swift/tools/diagnostics:autobuilder-incompatible-os"], - "@platforms//os:macos": ["//swift/swift-autobuilder"], - }), +alias( + name = "extractor-pack-arch", + actual = "swift-arch", visibility = ["//visibility:public"], ) diff --git a/swift/downgrades/BUILD.bazel b/swift/downgrades/BUILD.bazel index 5f643aea185d..7f5f6fe4dd6e 100644 --- a/swift/downgrades/BUILD.bazel +++ b/swift/downgrades/BUILD.bazel @@ -1,6 +1,6 @@ -load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") +load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix") -pkg_files( +codeql_pkg_files( name = "downgrades", srcs = glob( ["**"], diff --git a/swift/extractor/BUILD.bazel b/swift/extractor/BUILD.bazel index 3acdbf014e34..02b1a91e3f9a 100644 --- a/swift/extractor/BUILD.bazel +++ b/swift/extractor/BUILD.bazel @@ -1,4 +1,4 @@ -load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles") +load("//misc/bazel:pkg.bzl", "codeql_pkg_runfiles") load("//swift:rules.bzl", "swift_cc_binary") swift_cc_binary( @@ -29,9 +29,10 @@ sh_binary( data = [":extractor.real"], ) -pkg_runfiles( +codeql_pkg_runfiles( name = "pkg", - srcs = [":extractor"], + arch_specific = True, excludes = ["extractor.sh"], # script gets copied as "extractor", no need for the original .sh file + exes = [":extractor"], visibility = ["//swift:__pkg__"], ) diff --git a/swift/tools/BUILD.bazel b/swift/tools/BUILD.bazel index e59561bf528d..5c08835fe352 100644 --- a/swift/tools/BUILD.bazel +++ b/swift/tools/BUILD.bazel @@ -1,4 +1,4 @@ -load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") +load("//misc/bazel:pkg.bzl", "codeql_pkg_files") sh_binary( name = "qltest", @@ -16,29 +16,17 @@ sh_binary( srcs = ["identify-environment.sh"], ) -pkg_files( - name = "scripts", +codeql_pkg_files( + name = "tools", srcs = [ "autobuild.cmd", + "tracing-config.lua", + ], + exes = [ ":autobuild", ":identify-environment", ":qltest", ], - attributes = pkg_attributes(mode = "0755"), - prefix = "tools", -) - -pkg_files( - name = "tracing-config", - srcs = ["tracing-config.lua"], prefix = "tools", -) - -pkg_filegroup( - name = "tools", - srcs = [ - ":scripts", - ":tracing-config", - ], visibility = ["//swift:__pkg__"], )