Skip to content

Commit

Permalink
Rewrite third-party helpers
Browse files Browse the repository at this point in the history
Summary:
This diff rewrites the third party helpers to use constraints instead
of labels. This allows the necessary system packages to be resolved as part of
the query graph, and more reliably extracted than from labels. It also adds
implicit typo enforcement - since we are working with constraints, buck will
fail if a distro is typo'd.

This diff contains the new macros/rules in defs.bzl, rewrites the macros in
third_party.bzl to use them internally, and migrates a limited number of BUCK
files that depended on third_party_library() behavior that didn't work with the
macro rewrite. The remaining targets are migrated and third_party.bzl is
deleted in a stacked diff.

This diff depends on D63295525, which adds a new field to one of the prelude
helpers. The prelude is packaged with the buck2 binary, meaning if buck2 is run
through dotslash, it may be incompatible. If there are any errors related to a
"package" attribute, a newer version of buck2 is needed.

Reviewed By: bigfootjon

Differential Revision: D63653719

fbshipit-source-id: 2eb759300522feefdd59ff985056b2dacb53f322
  • Loading branch information
ckwalsh authored and facebook-github-bot committed Oct 1, 2024
1 parent f20e257 commit 22d17a7
Show file tree
Hide file tree
Showing 6 changed files with 247 additions and 105 deletions.
33 changes: 23 additions & 10 deletions build/buck2/install_deps/install_deps.sh
Original file line number Diff line number Diff line change
Expand Up @@ -30,26 +30,39 @@ if [ -z "${BUCK2_COMMAND}" ]; then
fi
fi

__confirm() {
echo "Press \"y\" to continue"
read -r REPLY
expr "X$REPLY" : '^X[Yy]$' >/dev/null
}

PKG_FILE=$(mktemp /tmp/buck2-install-pkgs.XXXXXX)

eval "$BUCK2_COMMAND \\
cquery \"attrregexfilter(labels, 'third-party:$ID:', deps(//...))\" \\
--json --output-attribute=labels 2>/dev/null \\
| grep -o \"third-party:$ID:[^\\\"]*\" \\
if ! command -v jq >/dev/null; then
echo "Failed to find jq command, attempting to install with"
echo
echo "$INSTALL_COMMAND" jq
echo
if __confirm; then
eval "$INSTALL_COMMAND jq"
else
echo "Not confirmed, exiting";
exit 1
fi
fi

eval "$BUCK2_COMMAND cquery 'kind(system_packages, deps(//...))' \\
--output-attribute=packages --modifier $ID --json 2>/dev/null \\
| jq -r '.[].packages[]' \\
| sort \\
| uniq \\
| sed \"s/third-party:$ID://\" \\
> $PKG_FILE"

echo "About to install the project dependencies with the following command:"
echo
eval "cat $PKG_FILE | xargs echo $INSTALL_COMMAND"
echo
echo "Press \"y\" to continue"
read -r REPLY
echo

if expr "X$REPLY" : '^X[Yy]$' >/dev/null; then
if __confirm; then
eval "cat $PKG_FILE | xargs -r $INSTALL_COMMAND"
else
echo "Not installing dependencies"
Expand Down
30 changes: 21 additions & 9 deletions shim/third-party/bzip2/BUCK
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,31 @@
# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
# of this source tree.

load("@shim//third-party:third_party.bzl", "third_party_library")
load("@//third-party:defs.bzl", "pkgconfig_system_library")

oncall("open_source")

third_party_library(
prebuilt_cxx_library(
name = "pkgconfig_unsupported",
linker_flags = select({
"//os:linux-ubuntu": ["-lbz2"],
"DEFAULT": [],
}),
target_compatible_with = [
"//os:linux-ubuntu",
],
visibility = [],
)

pkgconfig_system_library(
name = "bz2",
pkgconfig_fallback = struct(
linker_flags = ["-lbz2"],
),
packages = {
"//os:linux-fedora": ["bzip2-devel"],
"//os:linux-ubuntu": ["libbz2-dev"],
"//os:macos-homebrew": ["bzip2"],
},
pkgconfig_name = "bzip2",
repo_package_names = {
"fedora": "bzip2-devel",
"homebrew": "bzip2",
"ubuntu": "libbz2-dev",
unsupported = {
"//os:linux-ubuntu": [":pkgconfig_unsupported"],
},
)
157 changes: 157 additions & 0 deletions shim/third-party/defs.bzl
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
# Copyright (c) Meta Platforms, Inc. and affiliates.
#
# This source code is licensed under both the MIT license found in the
# LICENSE-MIT file in the root directory of this source tree and the Apache
# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
# of this source tree.

load("@prelude//third-party:pkgconfig.bzl", "external_pkgconfig_library")

HOMEBREW_CONSTRAINT = "//os:macos-homebrew"

def system_library(
name: str,
packages = None,
visibility = ["PUBLIC"],
deps = [],
exported_deps = [],
**kwargs):
system_packages_target_name = "__{}_system_pkgs".format(name)
packages = packages or dict()
packages["DEFAULT"] = []
system_packages(
name = system_packages_target_name,
packages = select(packages),
)
deps = deps + [":" + system_packages_target_name]

brews = packages.get(HOMEBREW_CONSTRAINT)
if brews != None:
exported_deps = exported_deps + select({
HOMEBREW_CONSTRAINT: _system_homebrew_targets(name, brews),
"DEFAULT": [],
})

native.prebuilt_cxx_library(
name = name,
visibility = visibility,
deps = deps,
exported_deps = exported_deps,
**kwargs
)

def pkgconfig_system_library(
name: str,
pkgconfig_name = None,
packages = None,
visibility = ["PUBLIC"],
deps = [],
exported_deps = [],
unsupported = dict(),
**kwargs):
system_packages_target_name = "__{}_system_pkgs".format(name)
packages = packages or dict()
packages["DEFAULT"] = []
system_packages(
name = system_packages_target_name,
packages = select(packages),
)

deps = exported_deps + deps

if len(unsupported) == 0:
external_pkgconfig_library(
name = name,
package = pkgconfig_name,
visibility = visibility,
deps = deps + [":" + system_packages_target_name],
**kwargs
)
else:
exported_deps_select_map = {}
for constraint, constraint_exported_deps in unsupported.items():
if constraint == HOMEBREW_CONSTRAINT:
brews = packages.get(constraint, [])
constraint_exported_deps = constraint_exported_deps + _system_homebrew_targets(name, brews)

exported_deps_select_map[constraint] = constraint_exported_deps

pkgconfig_target_name = "__{}_pkgconfig".format(name)
external_pkgconfig_library(
name = pkgconfig_target_name,
package = pkgconfig_name,
visibility = [],
deps = deps,
**kwargs
)
exported_deps_select_map["DEFAULT"] = [":" + pkgconfig_target_name]

native.prebuilt_cxx_library(
name = name,
visibility = visibility,
deps = [":" + system_packages_target_name],
exported_deps = select(exported_deps_select_map),
)

def _system_homebrew_targets(
name: str,
brews):
deps = []
for brew in brews:
homebrew_target_name = "__{}_homebrew_{}".format(name, brew)
homebrew_library(
name = homebrew_target_name,
brew = brew,
)
deps.append(":" + homebrew_target_name)

return deps

def _system_packages_impl(ctx: AnalysisContext) -> list[Provider]:
return [DefaultInfo()]

system_packages = rule(
impl = lambda _ctx: [DefaultInfo()],
attrs = {
"deps": attrs.list(attrs.dep(), default = []),
"packages": attrs.list(attrs.string()),
},
)

def homebrew_library(
name: str,
brew: str,
homebrew_header_path = "include",
exported_preprocessor_flags = [],
linker_flags = [],
target_compatible_with = ["//os:macos-homebrew"],
**kwargs):
preproc_flags_rule_name = "__{}__{}__preproc_flags".format(name, brew)
native.genrule(
name = preproc_flags_rule_name,
type = "homebrew_library_preproc_flags",
out = "out",
cmd = "echo \"-I`brew --prefix {}`/{}\" > $OUT".format(brew, homebrew_header_path),
target_compatible_with = target_compatible_with,
)

linker_flags_rule_name = "__{}__{}__linker_flags".format(name, brew)
native.genrule(
name = linker_flags_rule_name,
type = "homebrew_library_linker_flags",
out = "out",
cmd = "echo \"-L`brew --prefix {}`/lib\" > $OUT".format(brew),
target_compatible_with = target_compatible_with,
)

native.prebuilt_cxx_library(
name = name,
exported_preprocessor_flags = exported_preprocessor_flags + [
"@$(location :{})/preproc_flags.txt".format(preproc_flags_rule_name),
],
linker_flags = linker_flags + [
"@$(location :{})/linker_flags.txt".format(linker_flags_rule_name),
],
target_compatible_with = target_compatible_with,
**kwargs
)
30 changes: 21 additions & 9 deletions shim/third-party/libdwarf/BUCK
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,31 @@
# License, Version 2.0 found in the LICENSE-APACHE file in the root directory
# of this source tree.

load("@shim//third-party:third_party.bzl", "third_party_library")
load("@//third-party:defs.bzl", "pkgconfig_system_library")

oncall("open_source")

third_party_library(
prebuilt_cxx_library(
name = "pkgconfig_unsupported",
exported_preprocessor_flags = select({
"//os:linux-ubuntu": ["-I/usr/include/libdwarf"],
"DEFAULT": [],
}),
target_compatible_with = [
"//os:linux-ubuntu",
],
visibility = [],
)

pkgconfig_system_library(
name = "dwarf",
pkgconfig_fallback = struct(
preprocessor_flags = ["-I/usr/include/libdwarf"],
),
packages = {
"//os:linux-fedora": ["libdwarf-devel"],
"//os:linux-ubuntu": ["libdwarf-dev"],
"//os:macos-homebrew": ["libdwarf"],
},
pkgconfig_name = "libdwarf",
repo_package_names = {
"fedora": "libdwarf-devel",
"homebrew": "libdwarf",
"ubuntu": "libdwarf-dev",
unsupported = {
"//os:linux-ubuntu": [":pkgconfig_unsupported"],
},
)
1 change: 0 additions & 1 deletion shim/third-party/python/BUCK
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ oncall("open_source")

third_party_library(
name = "python",
homebrew_header_path = "Frameworks/Python.framework/Headers",
pkgconfig_name = "python3",
repo_package_names = {
"fedora": "python3-devel",
Expand Down
Loading

0 comments on commit 22d17a7

Please sign in to comment.