Rewrite translate_target() [5/7]

Summary:
This rewrites the translate_target() helper to be more scalable in the longer
term, and reduces the repo specific logic it contains. Instead, it enables
config settings to be specified in .buckconfig that achieve the same results.

Reviewed By: bigfootjon

Differential Revision: D63295975

fbshipit-source-id: 45dc91f59d0b26be41d9ced0d58bcdba4baee743

shim/build_defs/lib/oss.bzl

@@ -5,56 +5,232 @@
 # License, Version 2.0 found in the LICENSE-APACHE file in the root directory
 # of this source tree.
 
-def translate_target(target: str) -> str:
-    def remove_version(target: str) -> str:
-        # When upgrading libraries we either suffix them as `-old` or with a version, e.g. `-1-08`
-        # Strip those so we grab the right one in open source.
-        if target.endswith(":md-5"):  # md-5 is the one exception
-            return target
-        xs = target.split("-")
-        for i in reversed(range(len(xs))):
-            s = xs[i]
-            if s == "old" or s.isdigit():
-                xs.pop(i)
-            else:
-                break
-        return "-".join(xs)
-
-    if target == "//common/rust/shed/fbinit:fbinit":
-        return "fbsource//third-party/rust:fbinit"
-    elif target == "//common/rust/shed/sorted_vector_map:sorted_vector_map":
-        return "fbsource//third-party/rust:sorted_vector_map"
-    elif target == "//watchman/rust/watchman_client:watchman_client":
-        return "fbsource//third-party/rust:watchman_client"
-    elif target.startswith("fbsource//third-party/rust:"):
-        return remove_version(target)
-    elif target.startswith(":"):
+def _filter_empty_strings(string_list):
+    return filter(lambda d: d != "", string_list)
+
+def _parse_prefix_mappings(raw_rules):
+    rules = []
+    for raw_rule in rules:
+        (match, replace) = raw_rule.split("->", 1)
+
+        (cell, root_dir) = match.split("//")
+        match = struct(cell = cell, root_dir = root_dir)
+
+        (cell, root_dir) = replace.split("//")
+        replace = struct(cell = cell, root_dir = root_dir)
+
+        rules.append(struct(match = match, replace = replace))
+
+    return rules
+
+def _strip_third_party_rust_version(target: str) -> str:
+    # When upgrading libraries we either suffix them as `-old` or with a version, e.g. `-1-08`
+    # Strip those so we grab the right one in open source.
+    if target.endswith(":md-5"):  # md-5 is the one exception
         return target
-    elif target.startswith("//buck2/"):
-        return "root//" + target.removeprefix("//buck2/")
-    elif target.startswith("fbcode//common/ocaml/interop/"):
-        return "root//" + target.removeprefix("fbcode//common/ocaml/interop/")
-    elif target.startswith("fbcode//third-party-buck/platform010/build/supercaml"):
-        return "shim//third-party/ocaml" + target.removeprefix("fbcode//third-party-buck/platform010/build/supercaml")
-    elif target.startswith("fbcode//third-party-buck/platform010/build"):
-        return "shim//third-party" + target.removeprefix("fbcode//third-party-buck/platform010/build")
-    elif target.startswith("fbsource//third-party"):
-        return "shim//third-party" + target.removeprefix("fbsource//third-party")
-    elif target.startswith("third-party//"):
-        return "shim//third-party/" + target.removeprefix("third-party//")
-    elif target.startswith("//folly"):
-        oss_depends_on_folly = read_config("oss_depends_on", "folly", False)
-        if oss_depends_on_folly:
-            return "root//folly/" + target.removeprefix("//")
-        return "root//" + target.removeprefix("//")
-    elif target.startswith("root//folly"):
+    xs = target.split("-")
+    for i in reversed(range(len(xs))):
+        s = xs[i]
+        if s == "old" or s.isdigit():
+            xs.pop(i)
+        else:
+            break
+    return "-".join(xs)
+
+# Cell the BUCK file being processed belongs to
+ACTIVE_CELL = native.get_cell_name()
+
+# The root cell of this project, generally "root" and does not need to be set.
+# Targets that explicitly reference this cell will not be rewritten, and
+# targets that do not end up referencing a cell will be replaced with targets
+# that reference this cell
+ROOT_CELL = read_config("oss", "root_cell", "root")
+
+# The cell this file and the rest of the shim directory belong to, generally
+# "shim" and does not need to be set.
+SHIM_CELL = read_config("oss", "shim_cell", "shim")
+
+# The internal cell this project originally belonged to.
+#
+# When applying rewrites, the cell of the target is often considered. Targets
+# that do not explicitly specify a cell (eg: "//foo:bar") will be considered
+# to belong to INTERNAL_CELL.
+INTERNAL_CELL = read_config("oss", "internal_cell", "fbcode")
+
+# There can be situations where a target specifies a cell explicitly and the
+# path is part of the local checkout, rather than potentially needing to be
+# shimmed. In this case, we want to rewrite the target to use the root cell.
+#
+# If a target's cell is unspecified or matches the internal cell, and the path
+# starts with an entry in this list, The cell replaced with the ROOT_CELL.
+#
+# Entries are separated by spaces, and evaluated in order. Once a match is
+# found, the rewrite is complete and the following entries will not be
+# evaluated.
+#
+# Examples:
+#     internal_cell//oss_project/foo:bar -> root//oss_project/foo:bar
+PROJECT_DIRS = _filter_empty_strings(read_config("oss", "project_dirs", "").split(" "))
+
+# There are some situations where prefix of the internal directory structure is
+# removed from the public filepaths, such as rewriting "internal/foo/bar/baz"
+# to "oss/baz". When this happens, the BUCK files are not converted to reflect
+# the public directory structure, and targets need to be rewritten to account
+# for the discrepancy.
+#
+# Entries behave similarly to PROJECT_DIRS, except that the root directory will
+# also be removed from the path in the rewritten target. This setting is
+# applied after PROJECT_DIRS.
+#
+# Entries are separated by spaces and evaluated in order. Once a match is
+# found, the rewrite is complete and the following entries will not be
+# evaluated.
+#
+# Examples:
+#     //oss_project/foo:bar -> root//foo:bar
+#     internal_cell//oss_project/foo:bar -> root//foo:bar
+STRIPPED_ROOT_DIRS = _filter_empty_strings(read_config("oss", "stripped_root_dirs", "").split(" "))
+
+# Internally, most code shares the same cell in a monorepo, but public projects
+# only contain a subset, importing dependencies via git submodules or other
+# mechanisms. When this happens, the dependency may end up in a different
+# filepath, or may have it's own buck2 configuration and should be treated as
+# an on disk external cell.
+#
+# If the target's cell is a match (or if unspecified, INTERNAL_CELL is a
+# match),unspecified) matches, and the target's path is within the root
+# directory, both the cell and root directory prefix are replaced with the new
+# values.
+#
+# Entries are in the form of "MATCH->REPLACEMENT". Both MATCH and replacement
+# shall be in the format of "CELL//DIR_PREFIX".
+#
+# Entries are separated by spaces and evaluated in order. Once a match is
+# found, the rewrite is complete and the following entries will not be
+# evaluated.
+#
+# Examples:
+#     internal//foo->foo//foo; internal//foo/bar:baz -> foo//foo/bar:baz
+PREFIX_MAPPINGS = _parse_prefix_mappings(
+    _filter_empty_strings(read_config("oss", "prefix_mappings", "").split(" ")),
+)
+
+# Hardcoded rewrite rules that apply to many projects and only produce targets
+# within the shim cell. They are applied after the rules from .buckconfig, and
+# will not be applied if any other rules match.
+IMPLICIT_REWRITE_RULES = {
+    "fbcode": struct(
+        exact = {
+            "common/rust/shed/fbinit:fbinit": "third-party/rust:fbinit",
+            "common/rust/shed/sorted_vector_map:sorted_vector_map": "third-party/rust:sorted_vector_map",
+            "watchman/rust/watchman_client:watchman_client": "third-party/rust:watchman_client",
+        },
+        dirs = [
+            ("third-party-buck/platform010/build/supercaml", "third-party/ocaml"),
+            ("third-party-buck/platform010/build", "third-party"),
+        ],
+    ),
+    "fbsource": struct(
+        dirs = [
+            ("third-party", "third-party"),
+        ],
+        dynamic = [
+            ("third-party/rust", _strip_third_party_rust_version),
+        ],
+    ),
+    "third-party": struct(
+        dirs = [
+            ("", "third-party"),
+        ],
+        dynamic = [
+            ("rust", lambda path: "third-party/" + _strip_third_party_rust_version(path)),
+        ],
+    ),
+}
+
+DEFAULT_REWRITE_CTX = struct(
+    cells = struct(
+        active = ACTIVE_CELL,
+        root = ROOT_CELL,
+        shim = SHIM_CELL,
+        internal = INTERNAL_CELL,
+    ),
+    project_dirs = PROJECT_DIRS,
+    stripped_root_dirs = STRIPPED_ROOT_DIRS,
+    prefix_mappings = PREFIX_MAPPINGS,
+    implicit_rewrite_rules = IMPLICIT_REWRITE_RULES,
+)
+
+"""
+Rewrite an internal target string to one that is compatible with this OSS
+project.
+
+Some example use cases for this:
+- Map dependency targets to shim targets in this dir
+- Handle mismatching buck roots between internal and oss
+  (eg: internal/oss-project/... is exposed externally as oss-project/...)
+- Handle submodules that result in filepaths that do not match internal
+  (eg: internal/my_library/... and oss-project/my_library/my_library/...)
+"""
+
+def translate_target(
+        target: str,
+        ctx = DEFAULT_REWRITE_CTX) -> str:
+    if "//" not in target:
+        # This is a local target, aka ":foo". Don't touch
         return target
-    elif target.startswith("//fizz"):
-        return "root//" + target.removeprefix("//")
-    elif target.startswith("shim//"):
+
+    (cell, path) = target.split("//", 1)
+
+    if cell == ctx.cells.root:
+        # This cell is explicitly root. Don't touch
         return target
-    elif target.startswith("prelude//"):
+
+    resolved_cell = ctx.cells.active if cell == "" else cell
+    internal_cell = ctx.cells.internal if resolved_cell == ctx.cells.root else resolved_cell
+
+    if internal_cell == ctx.cells.internal:
+        for d in ctx.project_dirs:
+            if _path_rooted_in_dir(path, d):
+                return ctx.cells.root + "//" + path
+
+        for d in ctx.stripped_root_dirs:
+            if _path_rooted_in_dir(path, d):
+                return ctx.cells.root + "//" + _strip_root_dir_from_path(path, d)
+
+    for rule in ctx.prefix_mappings:
+        if internal_cell == rule.match.cell and _path_rooted_in_dir(path, rule.match.root_dir):
+            return rule.replace.cell + "//" + _swap_root_dir_for_path(path, rule.match.root_dir, rule.replace.root_dir)
+
+    rules = ctx.implicit_rewrite_rules.get(internal_cell)
+
+    if rules == None:
+        # No implicit rewrite rules
         return target
-    else:
-        fail("Dependency is unaccounted for `{}`.\n".format(target) +
-             "Did you forget 'oss-disable'?")
+
+    exact = getattr(rules, "exact", {}).get(path)
+    if exact != None:
+        return ctx.cells.shim + "//" + exact
+
+    for (match_root_dir, replace_root_dir) in getattr(rules, "dirs", []):
+        if _path_rooted_in_dir(path, match_root_dir):
+            return ctx.cells.shim + "//" + _swap_root_dir_for_path(path, match_root_dir, replace_root_dir)
+
+    for (match_root_dir, fn) in getattr(rules, "dynamic", []):
+        if _path_rooted_in_dir(path, match_root_dir):
+            return ctx.cells.shim + "//" + fn(path)
+
+    return target
+
+def _path_rooted_in_dir(path: str, d: str) -> bool:
+    return d == "" or path == d or path.startswith(d + "/") or path.startswith(d + ":")
+
+def _strip_root_dir_from_path(path: str, d: str) -> str:
+    return path.removeprefix(d).removeprefix("/")
+
+def _swap_root_dir_for_path(path: str, root_dir: str, new_root_dir) -> str:
+    suffix = _strip_root_dir_from_path(path, root_dir)
+    if not suffix.startswith(":"):
+        suffix = "/" + suffix
+    replace_path = new_root_dir.removesuffix("/") + suffix
+    return replace_path.removeprefix("/")

shim/build_defs/lib/test/oss.bzl

@@ -7,9 +7,67 @@
 
 load("@shim//build_defs/lib:oss.bzl", "translate_target")
 
-def _assert_eq(x, y):
-    if x != y:
-        fail("Expected {} == {}".format(x, y))
+TEST_CTX = struct(
+    cells = struct(
+        active = "root",
+        root = "root",
+        shim = "shim",
+        internal = "internal",
+    ),
+    project_dirs = ["project"],
+    stripped_root_dirs = ["root_dir"],
+    prefix_mappings = [
+        struct(
+            match = struct(cell = "internal", root_dir = "dep"),
+            replace = struct(cell = "dep", root_dir = "dep_rename"),
+        ),
+    ],
+    implicit_rewrite_rules = {
+        "internal": struct(
+            exact = {
+                "exact:exact": "foo/shimmed:exact",
+            },
+            dirs = [
+                ("third-party", "third_party"),
+            ],
+            dynamic = [
+                ("dynamic", lambda path: path.upper()),
+            ],
+        ),
+    },
+)
+
+def _test_target(target: str, expected: str):
+    actual = translate_target(target, TEST_CTX)
+
+    if actual != expected:
+        fail("Expected {} == {}".format(actual, expected))
 
 def test_translate_target():
-    _assert_eq(translate_target("fbsource//third-party/rust:derive_more-1"), "fbsource//third-party/rust:derive_more")
+    _test_target("//:foo", "//:foo")
+    _test_target("root//:foo", "root//:foo")
+    _test_target("other//:foo", "other//:foo")
+
+    _test_target("//project/foo:bar", "root//project/foo:bar")
+    _test_target("internal//project/foo:bar", "root//project/foo:bar")
+    _test_target("internal//project2/foo:bar", "internal//project2/foo:bar")
+
+    _test_target("//root_dir/foo:bar", "root//foo:bar")
+    _test_target("//root_dir/with/subdir/foo:bar", "root//with/subdir/foo:bar")
+    _test_target("internal//root_dir/foo:bar", "root//foo:bar")
+
+    _test_target("//dep:foo", "dep//dep_rename:foo")
+    _test_target("//dep/with/subdir:foo", "dep//dep_rename/with/subdir:foo")
+    _test_target("internal//dep:foo", "dep//dep_rename:foo")
+    _test_target("other//dep:foo", "other//dep:foo")
+
+    _test_target("//exact:exact", "shim//foo/shimmed:exact")
+    _test_target("internal//exact:exact", "shim//foo/shimmed:exact")
+    _test_target("other//exact:exact", "other//exact:exact")
+
+    _test_target("//third-party/lib/foo:bar", "shim//third_party/lib/foo:bar")
+    _test_target("internal//third-party/lib/foo:bar", "shim//third_party/lib/foo:bar")
+
+    _test_target("//dynamic:foo", "shim//DYNAMIC:FOO")
+    _test_target("internal//dynamic:foo", "shim//DYNAMIC:FOO")
+    _test_target("other//dynamic:foo", "other//dynamic:foo")