Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rust: add some performance diagnostics #18116

Merged
merged 13 commits into from
Dec 4, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion MODULE.bazel
Original file line number Diff line number Diff line change
@@ -68,7 +68,7 @@ use_repo(py_deps, "vendor__anyhow-1.0.44", "vendor__cc-1.0.70", "vendor__clap-2.
# deps for ruby+rust
# keep in sync by running `misc/bazel/3rdparty/update_cargo_deps.sh`
tree_sitter_extractors_deps = use_extension("//misc/bazel/3rdparty:tree_sitter_extractors_extension.bzl", "r")
use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.93", "vendor__argfile-0.2.1", "vendor__chrono-0.4.38", "vendor__clap-4.5.20", "vendor__dunce-1.0.5", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.34", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.10.5", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.89", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.232", "vendor__ra_ap_cfg-0.0.232", "vendor__ra_ap_hir-0.0.232", "vendor__ra_ap_hir_def-0.0.232", "vendor__ra_ap_hir_expand-0.0.232", "vendor__ra_ap_ide_db-0.0.232", "vendor__ra_ap_intern-0.0.232", "vendor__ra_ap_load-cargo-0.0.232", "vendor__ra_ap_parser-0.0.232", "vendor__ra_ap_paths-0.0.232", "vendor__ra_ap_project_model-0.0.232", "vendor__ra_ap_span-0.0.232", "vendor__ra_ap_syntax-0.0.232", "vendor__ra_ap_vfs-0.0.232", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.214", "vendor__serde_json-1.0.132", "vendor__serde_with-3.11.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.87", "vendor__tracing-0.1.40", "vendor__tracing-subscriber-0.3.18", "vendor__tree-sitter-0.24.4", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1")
use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.93", "vendor__argfile-0.2.1", "vendor__chrono-0.4.38", "vendor__clap-4.5.20", "vendor__dunce-1.0.5", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.34", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.10.5", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.89", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.232", "vendor__ra_ap_cfg-0.0.232", "vendor__ra_ap_hir-0.0.232", "vendor__ra_ap_hir_def-0.0.232", "vendor__ra_ap_hir_expand-0.0.232", "vendor__ra_ap_ide_db-0.0.232", "vendor__ra_ap_intern-0.0.232", "vendor__ra_ap_load-cargo-0.0.232", "vendor__ra_ap_parser-0.0.232", "vendor__ra_ap_paths-0.0.232", "vendor__ra_ap_project_model-0.0.232", "vendor__ra_ap_span-0.0.232", "vendor__ra_ap_syntax-0.0.232", "vendor__ra_ap_vfs-0.0.232", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.214", "vendor__serde_json-1.0.133", "vendor__serde_with-3.11.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.87", "vendor__tracing-0.1.40", "vendor__tracing-subscriber-0.3.18", "vendor__tree-sitter-0.24.4", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1")

dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet")
dotnet.toolchain(dotnet_version = "9.0.100")

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 9 additions & 7 deletions misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion misc/codegen/generators/dbschemegen.py
Original file line number Diff line number Diff line change
@@ -110,7 +110,8 @@ def cls_to_dbscheme(cls: schema.Class, lookup: typing.Dict[str, schema.Class], a

def get_declarations(data: schema.Schema):
add_or_none_except = data.root_class.name if data.null else None
declarations = [d for cls in data.classes.values() for d in cls_to_dbscheme(cls, data.classes, add_or_none_except)]
declarations = [d for cls in data.classes.values() if not cls.imported for d in cls_to_dbscheme(cls,
data.classes, add_or_none_except)]
if data.null:
property_classes = {
prop.type for cls in data.classes.values() for prop in cls.properties
35 changes: 26 additions & 9 deletions misc/codegen/generators/qlgen.py
Original file line number Diff line number Diff line change
@@ -105,8 +105,17 @@ def _get_doc(cls: schema.Class, prop: schema.Property, plural=None):
return f"{prop_name} of this {class_name}"


def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dict[str, schema.Class],
def _type_is_hideable(t: str, lookup: typing.Dict[str, schema.ClassBase]) -> bool:
if t in lookup:
match lookup[t]:
case schema.Class() as cls:
return "ql_hideable" in cls.pragmas
return False


def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dict[str, schema.ClassBase],
prev_child: str = "") -> ql.Property:

args = dict(
type=prop.type if not prop.is_predicate else "predicate",
qltest_skip="qltest_skip" in prop.pragmas,
@@ -116,7 +125,8 @@ def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dic
is_unordered=prop.is_unordered,
description=prop.description,
synth=bool(cls.synth) or prop.synth,
type_is_hideable="ql_hideable" in lookup[prop.type].pragmas if prop.type in lookup else False,
type_is_hideable=_type_is_hideable(prop.type, lookup),
type_is_codegen_class=prop.type in lookup and not lookup[prop.type].imported,
internal="ql_internal" in prop.pragmas,
)
ql_name = prop.pragmas.get("ql_name", prop.name)
@@ -155,7 +165,7 @@ def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dic
return ql.Property(**args)


def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.Class]) -> ql.Class:
def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.ClassBase]) -> ql.Class:
if "ql_name" in cls.pragmas:
raise Error("ql_name is not supported yet for classes, only for properties")
prev_child = ""
@@ -392,14 +402,15 @@ def generate(opts, renderer):

data = schemaloader.load_file(input)

classes = {name: get_ql_class(cls, data.classes) for name, cls in data.classes.items()}
classes = {name: get_ql_class(cls, data.classes) for name, cls in data.classes.items() if not cls.imported}
if not classes:
raise NoClasses
root = next(iter(classes.values()))
if root.has_children:
raise RootElementHasChildren(root)

imports = {}
pre_imports = {n: cls.module for n, cls in data.classes.items() if cls.imported}
imports = dict(pre_imports)
imports_impl = {}
classes_used_by = {}
cfg_classes = []
@@ -411,7 +422,7 @@ def generate(opts, renderer):
force=opts.force) as renderer:

db_classes = [cls for name, cls in classes.items() if not data.classes[name].synth]
renderer.render(ql.DbClasses(db_classes), out / "Raw.qll")
renderer.render(ql.DbClasses(classes=db_classes, imports=sorted(set(pre_imports.values()))), out / "Raw.qll")

classes_by_dir_and_name = sorted(classes.values(), key=lambda cls: (cls.dir, cls.name))
for c in classes_by_dir_and_name:
@@ -440,6 +451,8 @@ def generate(opts, renderer):
renderer.render(cfg_classes_val, cfg_qll)

for c in data.classes.values():
if c.imported:
continue
path = _get_path(c)
path_impl = _get_path_impl(c)
stub_file = stub_out / path_impl
@@ -458,20 +471,23 @@ def generate(opts, renderer):
renderer.render(class_public, class_public_file)

# for example path/to/elements -> path/to/elements.qll
renderer.render(ql.ImportList([i for name, i in imports.items() if not classes[name].internal]),
renderer.render(ql.ImportList([i for name, i in imports.items() if name not in classes or not classes[name].internal]),
include_file)

elements_module = get_import(include_file, opts.root_dir)

renderer.render(
ql.GetParentImplementation(
classes=list(classes.values()),
imports=[elements_module] + [i for name, i in imports.items() if classes[name].internal],
imports=[elements_module] + [i for name,
i in imports.items() if name in classes and classes[name].internal],
),
out / 'ParentChild.qll')

if test_out:
for c in data.classes.values():
if c.imported:
continue
if should_skip_qltest(c, data.classes):
continue
test_with_name = c.pragmas.get("qltest_test_with")
@@ -501,7 +517,8 @@ def generate(opts, renderer):
constructor_imports = []
synth_constructor_imports = []
stubs = {}
for cls in sorted(data.classes.values(), key=lambda cls: (cls.group, cls.name)):
for cls in sorted((cls for cls in data.classes.values() if not cls.imported),
key=lambda cls: (cls.group, cls.name)):
synth_type = get_ql_synth_class(cls)
if synth_type.is_final:
final_synth_types.append(synth_type)
16 changes: 10 additions & 6 deletions misc/codegen/generators/rustgen.py
Original file line number Diff line number Diff line change
@@ -49,7 +49,7 @@ def _get_field(cls: schema.Class, p: schema.Property) -> rust.Field:


def _get_properties(
cls: schema.Class, lookup: dict[str, schema.Class],
cls: schema.Class, lookup: dict[str, schema.ClassBase],
) -> typing.Iterable[tuple[schema.Class, schema.Property]]:
for b in cls.bases:
yield from _get_properties(lookup[b], lookup)
@@ -58,20 +58,22 @@ def _get_properties(


def _get_ancestors(
cls: schema.Class, lookup: dict[str, schema.Class]
cls: schema.Class, lookup: dict[str, schema.ClassBase]
) -> typing.Iterable[schema.Class]:
for b in cls.bases:
base = lookup[b]
yield base
yield from _get_ancestors(base, lookup)
if not base.imported:
base = typing.cast(schema.Class, base)
yield base
yield from _get_ancestors(base, lookup)


class Processor:
def __init__(self, data: schema.Schema):
self._classmap = data.classes

def _get_class(self, name: str) -> rust.Class:
cls = self._classmap[name]
cls = typing.cast(schema.Class, self._classmap[name])
properties = [
(c, p)
for c, p in _get_properties(cls, self._classmap)
@@ -101,8 +103,10 @@ def _get_class(self, name: str) -> rust.Class:
def get_classes(self):
ret = {"": []}
for k, cls in self._classmap.items():
if not cls.synth:
if not cls.imported and not cls.synth:
ret.setdefault(cls.group, []).append(self._get_class(cls.name))
elif cls.imported:
ret[""].append(rust.Class(name=cls.name))
return ret


2 changes: 2 additions & 0 deletions misc/codegen/generators/rusttestgen.py
Original file line number Diff line number Diff line change
@@ -56,6 +56,8 @@ def generate(opts, renderer):
registry=opts.ql_test_output / ".generated_tests.list",
force=opts.force) as renderer:
for cls in schema.classes.values():
if cls.imported:
continue
if (qlgen.should_skip_qltest(cls, schema.classes) or
"rust_skip_doc_test" in cls.pragmas):
continue
6 changes: 2 additions & 4 deletions misc/codegen/lib/ql.py
Original file line number Diff line number Diff line change
@@ -44,6 +44,7 @@ class Property:
doc_plural: Optional[str] = None
synth: bool = False
type_is_hideable: bool = False
type_is_codegen_class: bool = False
internal: bool = False
cfg: bool = False

@@ -66,10 +67,6 @@ def indefinite_getter(self):
article = "An" if self.singular[0] in "AEIO" else "A"
return f"get{article}{self.singular}"

@property
def type_is_class(self):
return bool(self.type) and self.type[0].isupper()

@property
def is_repeated(self):
return bool(self.plural)
@@ -191,6 +188,7 @@ class DbClasses:
template: ClassVar = 'ql_db'

classes: List[Class] = field(default_factory=list)
imports: List[str] = field(default_factory=list)


@dataclass
Loading