diff --git a/Cargo.lock b/Cargo.lock index 2b31f24bac4b..65a77b316c9d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -381,6 +381,7 @@ version = "0.1.0" dependencies = [ "anyhow", "argfile", + "chrono", "clap", "codeql-extractor", "dunce", @@ -405,6 +406,7 @@ dependencies = [ "ra_ap_vfs", "rust-extractor-macros", "serde", + "serde_json", "serde_with", "stderrlog", "triomphe", @@ -2041,9 +2043,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.132" +version = "1.0.133" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03" +checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" dependencies = [ "itoa", "memchr", diff --git a/MODULE.bazel b/MODULE.bazel index ca09ada47001..08a4aaa78af9 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -68,7 +68,7 @@ use_repo(py_deps, "vendor__anyhow-1.0.44", "vendor__cc-1.0.70", "vendor__clap-2. # deps for ruby+rust # keep in sync by running `misc/bazel/3rdparty/update_cargo_deps.sh` tree_sitter_extractors_deps = use_extension("//misc/bazel/3rdparty:tree_sitter_extractors_extension.bzl", "r") -use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.93", "vendor__argfile-0.2.1", "vendor__chrono-0.4.38", "vendor__clap-4.5.20", "vendor__dunce-1.0.5", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.34", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.10.5", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.89", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.232", "vendor__ra_ap_cfg-0.0.232", "vendor__ra_ap_hir-0.0.232", "vendor__ra_ap_hir_def-0.0.232", "vendor__ra_ap_hir_expand-0.0.232", "vendor__ra_ap_ide_db-0.0.232", "vendor__ra_ap_intern-0.0.232", "vendor__ra_ap_load-cargo-0.0.232", "vendor__ra_ap_parser-0.0.232", "vendor__ra_ap_paths-0.0.232", "vendor__ra_ap_project_model-0.0.232", "vendor__ra_ap_span-0.0.232", "vendor__ra_ap_syntax-0.0.232", "vendor__ra_ap_vfs-0.0.232", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.214", "vendor__serde_json-1.0.132", "vendor__serde_with-3.11.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.87", "vendor__tracing-0.1.40", "vendor__tracing-subscriber-0.3.18", "vendor__tree-sitter-0.24.4", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1") +use_repo(tree_sitter_extractors_deps, "vendor__anyhow-1.0.93", "vendor__argfile-0.2.1", "vendor__chrono-0.4.38", "vendor__clap-4.5.20", "vendor__dunce-1.0.5", "vendor__encoding-0.2.33", "vendor__figment-0.10.19", "vendor__flate2-1.0.34", "vendor__glob-0.3.1", "vendor__globset-0.4.15", "vendor__itertools-0.10.5", "vendor__itertools-0.13.0", "vendor__lazy_static-1.5.0", "vendor__log-0.4.22", "vendor__num-traits-0.2.19", "vendor__num_cpus-1.16.0", "vendor__proc-macro2-1.0.89", "vendor__quote-1.0.37", "vendor__ra_ap_base_db-0.0.232", "vendor__ra_ap_cfg-0.0.232", "vendor__ra_ap_hir-0.0.232", "vendor__ra_ap_hir_def-0.0.232", "vendor__ra_ap_hir_expand-0.0.232", "vendor__ra_ap_ide_db-0.0.232", "vendor__ra_ap_intern-0.0.232", "vendor__ra_ap_load-cargo-0.0.232", "vendor__ra_ap_parser-0.0.232", "vendor__ra_ap_paths-0.0.232", "vendor__ra_ap_project_model-0.0.232", "vendor__ra_ap_span-0.0.232", "vendor__ra_ap_syntax-0.0.232", "vendor__ra_ap_vfs-0.0.232", "vendor__rand-0.8.5", "vendor__rayon-1.10.0", "vendor__regex-1.11.1", "vendor__serde-1.0.214", "vendor__serde_json-1.0.133", "vendor__serde_with-3.11.0", "vendor__stderrlog-0.6.0", "vendor__syn-2.0.87", "vendor__tracing-0.1.40", "vendor__tracing-subscriber-0.3.18", "vendor__tree-sitter-0.24.4", "vendor__tree-sitter-embedded-template-0.23.2", "vendor__tree-sitter-json-0.24.8", "vendor__tree-sitter-ql-0.23.1", "vendor__tree-sitter-ruby-0.23.1", "vendor__triomphe-0.1.14", "vendor__ungrammar-1.16.1") dotnet = use_extension("@rules_dotnet//dotnet:extensions.bzl", "dotnet") dotnet.toolchain(dotnet_version = "9.0.100") diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel index 8eb128bd79d6..0e886c727a04 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.bazel @@ -249,7 +249,7 @@ alias( alias( name = "serde_json", - actual = "@vendor__serde_json-1.0.132//:serde_json", + actual = "@vendor__serde_json-1.0.133//:serde_json", tags = ["manual"], ) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.cargo_metadata-0.18.1.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.cargo_metadata-0.18.1.bazel index e31c61cfac73..9ad331aa5f74 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.cargo_metadata-0.18.1.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.cargo_metadata-0.18.1.bazel @@ -86,7 +86,7 @@ rust_library( "@vendor__cargo-platform-0.1.8//:cargo_platform", "@vendor__semver-1.0.23//:semver", "@vendor__serde-1.0.214//:serde", - "@vendor__serde_json-1.0.132//:serde_json", + "@vendor__serde_json-1.0.133//:serde_json", "@vendor__thiserror-1.0.69//:thiserror", ], ) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_proc_macro_api-0.0.232.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_proc_macro_api-0.0.232.bazel index d1f5d480d719..fa3b71f570cd 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_proc_macro_api-0.0.232.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_proc_macro_api-0.0.232.bazel @@ -96,7 +96,7 @@ rust_library( "@vendor__ra_ap_tt-0.0.232//:ra_ap_tt", "@vendor__rustc-hash-1.1.0//:rustc_hash", "@vendor__serde-1.0.214//:serde", - "@vendor__serde_json-1.0.132//:serde_json", + "@vendor__serde_json-1.0.133//:serde_json", "@vendor__tracing-0.1.40//:tracing", ], ) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_project_model-0.0.232.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_project_model-0.0.232.bazel index 6f6f3bb303a4..5608e6dd6b0d 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_project_model-0.0.232.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.ra_ap_project_model-0.0.232.bazel @@ -102,7 +102,7 @@ rust_library( "@vendor__rustc-hash-1.1.0//:rustc_hash", "@vendor__semver-1.0.23//:semver", "@vendor__serde-1.0.214//:serde", - "@vendor__serde_json-1.0.132//:serde_json", + "@vendor__serde_json-1.0.133//:serde_json", "@vendor__tracing-0.1.40//:tracing", "@vendor__triomphe-0.1.14//:triomphe", ], diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_json-1.0.132.bazel b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_json-1.0.133.bazel similarity index 97% rename from misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_json-1.0.132.bazel rename to misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_json-1.0.133.bazel index ecbae32eeafa..710772e4f37b 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_json-1.0.132.bazel +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/BUILD.serde_json-1.0.133.bazel @@ -83,13 +83,13 @@ rust_library( "@rules_rust//rust/platform:x86_64-unknown-none": [], "//conditions:default": ["@platforms//:incompatible"], }), - version = "1.0.132", + version = "1.0.133", deps = [ "@vendor__itoa-1.0.11//:itoa", "@vendor__memchr-2.7.4//:memchr", "@vendor__ryu-1.0.18//:ryu", "@vendor__serde-1.0.214//:serde", - "@vendor__serde_json-1.0.132//:build_script_build", + "@vendor__serde_json-1.0.133//:build_script_build", ], ) @@ -143,7 +143,7 @@ cargo_build_script( "noclippy", "norustfmt", ], - version = "1.0.132", + version = "1.0.133", visibility = ["//visibility:private"], ) diff --git a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl index e8ea45d091fe..dfdfdd8560df 100644 --- a/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl +++ b/misc/bazel/3rdparty/tree_sitter_extractors_deps/defs.bzl @@ -320,6 +320,7 @@ _NORMAL_DEPENDENCIES = { _COMMON_CONDITION: { "anyhow": Label("@vendor__anyhow-1.0.93//:anyhow"), "argfile": Label("@vendor__argfile-0.2.1//:argfile"), + "chrono": Label("@vendor__chrono-0.4.38//:chrono"), "clap": Label("@vendor__clap-4.5.20//:clap"), "dunce": Label("@vendor__dunce-1.0.5//:dunce"), "figment": Label("@vendor__figment-0.10.19//:figment"), @@ -342,6 +343,7 @@ _NORMAL_DEPENDENCIES = { "ra_ap_syntax": Label("@vendor__ra_ap_syntax-0.0.232//:ra_ap_syntax"), "ra_ap_vfs": Label("@vendor__ra_ap_vfs-0.0.232//:ra_ap_vfs"), "serde": Label("@vendor__serde-1.0.214//:serde"), + "serde_json": Label("@vendor__serde_json-1.0.133//:serde_json"), "serde_with": Label("@vendor__serde_with-3.11.0//:serde_with"), "stderrlog": Label("@vendor__stderrlog-0.6.0//:stderrlog"), "triomphe": Label("@vendor__triomphe-0.1.14//:triomphe"), @@ -364,7 +366,7 @@ _NORMAL_DEPENDENCIES = { "rayon": Label("@vendor__rayon-1.10.0//:rayon"), "regex": Label("@vendor__regex-1.11.1//:regex"), "serde": Label("@vendor__serde-1.0.214//:serde"), - "serde_json": Label("@vendor__serde_json-1.0.132//:serde_json"), + "serde_json": Label("@vendor__serde_json-1.0.133//:serde_json"), "tracing": Label("@vendor__tracing-0.1.40//:tracing"), "tracing-subscriber": Label("@vendor__tracing-subscriber-0.3.18//:tracing_subscriber"), "tree-sitter": Label("@vendor__tree-sitter-0.24.4//:tree_sitter"), @@ -2519,12 +2521,12 @@ def crate_repositories(): maybe( http_archive, - name = "vendor__serde_json-1.0.132", - sha256 = "d726bfaff4b320266d395898905d0eba0345aae23b54aee3a737e260fd46db03", + name = "vendor__serde_json-1.0.133", + sha256 = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377", type = "tar.gz", - urls = ["https://static.crates.io/crates/serde_json/1.0.132/download"], - strip_prefix = "serde_json-1.0.132", - build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.serde_json-1.0.132.bazel"), + urls = ["https://static.crates.io/crates/serde_json/1.0.133/download"], + strip_prefix = "serde_json-1.0.133", + build_file = Label("//misc/bazel/3rdparty/tree_sitter_extractors_deps:BUILD.serde_json-1.0.133.bazel"), ) maybe( @@ -3353,7 +3355,7 @@ def crate_repositories(): struct(repo = "vendor__rayon-1.10.0", is_dev_dep = False), struct(repo = "vendor__regex-1.11.1", is_dev_dep = False), struct(repo = "vendor__serde-1.0.214", is_dev_dep = False), - struct(repo = "vendor__serde_json-1.0.132", is_dev_dep = False), + struct(repo = "vendor__serde_json-1.0.133", is_dev_dep = False), struct(repo = "vendor__serde_with-3.11.0", is_dev_dep = False), struct(repo = "vendor__stderrlog-0.6.0", is_dev_dep = False), struct(repo = "vendor__syn-2.0.87", is_dev_dep = False), diff --git a/misc/codegen/generators/dbschemegen.py b/misc/codegen/generators/dbschemegen.py index 2c3cd5598d55..e2cc4220dc71 100755 --- a/misc/codegen/generators/dbschemegen.py +++ b/misc/codegen/generators/dbschemegen.py @@ -110,7 +110,8 @@ def cls_to_dbscheme(cls: schema.Class, lookup: typing.Dict[str, schema.Class], a def get_declarations(data: schema.Schema): add_or_none_except = data.root_class.name if data.null else None - declarations = [d for cls in data.classes.values() for d in cls_to_dbscheme(cls, data.classes, add_or_none_except)] + declarations = [d for cls in data.classes.values() if not cls.imported for d in cls_to_dbscheme(cls, + data.classes, add_or_none_except)] if data.null: property_classes = { prop.type for cls in data.classes.values() for prop in cls.properties diff --git a/misc/codegen/generators/qlgen.py b/misc/codegen/generators/qlgen.py index 96174109ce3d..5cdc16a8af5d 100755 --- a/misc/codegen/generators/qlgen.py +++ b/misc/codegen/generators/qlgen.py @@ -105,8 +105,17 @@ def _get_doc(cls: schema.Class, prop: schema.Property, plural=None): return f"{prop_name} of this {class_name}" -def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dict[str, schema.Class], +def _type_is_hideable(t: str, lookup: typing.Dict[str, schema.ClassBase]) -> bool: + if t in lookup: + match lookup[t]: + case schema.Class() as cls: + return "ql_hideable" in cls.pragmas + return False + + +def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dict[str, schema.ClassBase], prev_child: str = "") -> ql.Property: + args = dict( type=prop.type if not prop.is_predicate else "predicate", qltest_skip="qltest_skip" in prop.pragmas, @@ -116,7 +125,8 @@ def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dic is_unordered=prop.is_unordered, description=prop.description, synth=bool(cls.synth) or prop.synth, - type_is_hideable="ql_hideable" in lookup[prop.type].pragmas if prop.type in lookup else False, + type_is_hideable=_type_is_hideable(prop.type, lookup), + type_is_codegen_class=prop.type in lookup and not lookup[prop.type].imported, internal="ql_internal" in prop.pragmas, ) ql_name = prop.pragmas.get("ql_name", prop.name) @@ -155,7 +165,7 @@ def get_ql_property(cls: schema.Class, prop: schema.Property, lookup: typing.Dic return ql.Property(**args) -def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.Class]) -> ql.Class: +def get_ql_class(cls: schema.Class, lookup: typing.Dict[str, schema.ClassBase]) -> ql.Class: if "ql_name" in cls.pragmas: raise Error("ql_name is not supported yet for classes, only for properties") prev_child = "" @@ -392,14 +402,15 @@ def generate(opts, renderer): data = schemaloader.load_file(input) - classes = {name: get_ql_class(cls, data.classes) for name, cls in data.classes.items()} + classes = {name: get_ql_class(cls, data.classes) for name, cls in data.classes.items() if not cls.imported} if not classes: raise NoClasses root = next(iter(classes.values())) if root.has_children: raise RootElementHasChildren(root) - imports = {} + pre_imports = {n: cls.module for n, cls in data.classes.items() if cls.imported} + imports = dict(pre_imports) imports_impl = {} classes_used_by = {} cfg_classes = [] @@ -411,7 +422,7 @@ def generate(opts, renderer): force=opts.force) as renderer: db_classes = [cls for name, cls in classes.items() if not data.classes[name].synth] - renderer.render(ql.DbClasses(db_classes), out / "Raw.qll") + renderer.render(ql.DbClasses(classes=db_classes, imports=sorted(set(pre_imports.values()))), out / "Raw.qll") classes_by_dir_and_name = sorted(classes.values(), key=lambda cls: (cls.dir, cls.name)) for c in classes_by_dir_and_name: @@ -440,6 +451,8 @@ def generate(opts, renderer): renderer.render(cfg_classes_val, cfg_qll) for c in data.classes.values(): + if c.imported: + continue path = _get_path(c) path_impl = _get_path_impl(c) stub_file = stub_out / path_impl @@ -458,7 +471,7 @@ def generate(opts, renderer): renderer.render(class_public, class_public_file) # for example path/to/elements -> path/to/elements.qll - renderer.render(ql.ImportList([i for name, i in imports.items() if not classes[name].internal]), + renderer.render(ql.ImportList([i for name, i in imports.items() if name not in classes or not classes[name].internal]), include_file) elements_module = get_import(include_file, opts.root_dir) @@ -466,12 +479,15 @@ def generate(opts, renderer): renderer.render( ql.GetParentImplementation( classes=list(classes.values()), - imports=[elements_module] + [i for name, i in imports.items() if classes[name].internal], + imports=[elements_module] + [i for name, + i in imports.items() if name in classes and classes[name].internal], ), out / 'ParentChild.qll') if test_out: for c in data.classes.values(): + if c.imported: + continue if should_skip_qltest(c, data.classes): continue test_with_name = c.pragmas.get("qltest_test_with") @@ -501,7 +517,8 @@ def generate(opts, renderer): constructor_imports = [] synth_constructor_imports = [] stubs = {} - for cls in sorted(data.classes.values(), key=lambda cls: (cls.group, cls.name)): + for cls in sorted((cls for cls in data.classes.values() if not cls.imported), + key=lambda cls: (cls.group, cls.name)): synth_type = get_ql_synth_class(cls) if synth_type.is_final: final_synth_types.append(synth_type) diff --git a/misc/codegen/generators/rustgen.py b/misc/codegen/generators/rustgen.py index f4c977c0fdbe..b47e5cc4bd9d 100644 --- a/misc/codegen/generators/rustgen.py +++ b/misc/codegen/generators/rustgen.py @@ -49,7 +49,7 @@ def _get_field(cls: schema.Class, p: schema.Property) -> rust.Field: def _get_properties( - cls: schema.Class, lookup: dict[str, schema.Class], + cls: schema.Class, lookup: dict[str, schema.ClassBase], ) -> typing.Iterable[tuple[schema.Class, schema.Property]]: for b in cls.bases: yield from _get_properties(lookup[b], lookup) @@ -58,12 +58,14 @@ def _get_properties( def _get_ancestors( - cls: schema.Class, lookup: dict[str, schema.Class] + cls: schema.Class, lookup: dict[str, schema.ClassBase] ) -> typing.Iterable[schema.Class]: for b in cls.bases: base = lookup[b] - yield base - yield from _get_ancestors(base, lookup) + if not base.imported: + base = typing.cast(schema.Class, base) + yield base + yield from _get_ancestors(base, lookup) class Processor: @@ -71,7 +73,7 @@ def __init__(self, data: schema.Schema): self._classmap = data.classes def _get_class(self, name: str) -> rust.Class: - cls = self._classmap[name] + cls = typing.cast(schema.Class, self._classmap[name]) properties = [ (c, p) for c, p in _get_properties(cls, self._classmap) @@ -101,8 +103,10 @@ def _get_class(self, name: str) -> rust.Class: def get_classes(self): ret = {"": []} for k, cls in self._classmap.items(): - if not cls.synth: + if not cls.imported and not cls.synth: ret.setdefault(cls.group, []).append(self._get_class(cls.name)) + elif cls.imported: + ret[""].append(rust.Class(name=cls.name)) return ret diff --git a/misc/codegen/generators/rusttestgen.py b/misc/codegen/generators/rusttestgen.py index d360db27a658..e7a23fedacdc 100644 --- a/misc/codegen/generators/rusttestgen.py +++ b/misc/codegen/generators/rusttestgen.py @@ -56,6 +56,8 @@ def generate(opts, renderer): registry=opts.ql_test_output / ".generated_tests.list", force=opts.force) as renderer: for cls in schema.classes.values(): + if cls.imported: + continue if (qlgen.should_skip_qltest(cls, schema.classes) or "rust_skip_doc_test" in cls.pragmas): continue diff --git a/misc/codegen/lib/ql.py b/misc/codegen/lib/ql.py index 1182c7f5dc16..b9362a556ef1 100644 --- a/misc/codegen/lib/ql.py +++ b/misc/codegen/lib/ql.py @@ -44,6 +44,7 @@ class Property: doc_plural: Optional[str] = None synth: bool = False type_is_hideable: bool = False + type_is_codegen_class: bool = False internal: bool = False cfg: bool = False @@ -66,10 +67,6 @@ def indefinite_getter(self): article = "An" if self.singular[0] in "AEIO" else "A" return f"get{article}{self.singular}" - @property - def type_is_class(self): - return bool(self.type) and self.type[0].isupper() - @property def is_repeated(self): return bool(self.plural) @@ -191,6 +188,7 @@ class DbClasses: template: ClassVar = 'ql_db' classes: List[Class] = field(default_factory=list) + imports: List[str] = field(default_factory=list) @dataclass diff --git a/misc/codegen/lib/schema.py b/misc/codegen/lib/schema.py index 23f1aea2ba42..5178e61d3844 100644 --- a/misc/codegen/lib/schema.py +++ b/misc/codegen/lib/schema.py @@ -3,7 +3,7 @@ import typing from collections.abc import Iterable from dataclasses import dataclass, field -from typing import List, Set, Union, Dict, Optional +from typing import List, Set, Union, Dict, Optional, FrozenSet from enum import Enum, auto import functools @@ -87,8 +87,22 @@ class SynthInfo: @dataclass -class Class: +class ClassBase: + imported: typing.ClassVar[bool] name: str + + +@dataclass +class ImportedClass(ClassBase): + imported: typing.ClassVar[bool] = True + + module: str + + +@dataclass +class Class(ClassBase): + imported: typing.ClassVar[bool] = False + bases: List[str] = field(default_factory=list) derived: Set[str] = field(default_factory=set) properties: List[Property] = field(default_factory=list) @@ -133,7 +147,7 @@ def group(self) -> str: @dataclass class Schema: - classes: Dict[str, Class] = field(default_factory=dict) + classes: Dict[str, ClassBase] = field(default_factory=dict) includes: List[str] = field(default_factory=list) null: Optional[str] = None @@ -155,7 +169,7 @@ def iter_properties(self, cls: str) -> Iterable[Property]: predicate_marker = object() -TypeRef = Union[type, str] +TypeRef = type | str | ImportedClass def get_type_name(arg: TypeRef) -> str: @@ -164,6 +178,8 @@ def get_type_name(arg: TypeRef) -> str: return arg.__name__ case str(): return arg + case ImportedClass(): + return arg.name case _: raise Error(f"Not a schema type or string ({arg})") @@ -172,9 +188,9 @@ def _make_property(arg: object) -> Property: match arg: case _ if arg is predicate_marker: return PredicateProperty() - case str() | type(): + case (str() | type() | ImportedClass()) as arg: return SingleProperty(type=get_type_name(arg)) - case Property(): + case Property() as arg: return arg case _: raise Error(f"Illegal property specifier {arg}") diff --git a/misc/codegen/lib/schemadefs.py b/misc/codegen/lib/schemadefs.py index 8651240c1a3d..c81b2f2e215a 100644 --- a/misc/codegen/lib/schemadefs.py +++ b/misc/codegen/lib/schemadefs.py @@ -8,8 +8,6 @@ import inspect as _inspect from dataclasses import dataclass as _dataclass -from misc.codegen.lib.schema import Property - _set = set @@ -69,6 +67,9 @@ def include(source: str): _inspect.currentframe().f_back.f_locals.setdefault("includes", []).append(source) +imported = _schema.ImportedClass + + @_dataclass class _Namespace: """ simple namespacing mechanism """ @@ -264,7 +265,7 @@ class _PropertyModifierList(_schema.PropertyModifier): def __or__(self, other: _schema.PropertyModifier): return _PropertyModifierList(self._mods + (other,)) - def modify(self, prop: Property): + def modify(self, prop: _schema.Property): for m in self._mods: m.modify(prop) diff --git a/misc/codegen/loaders/schemaloader.py b/misc/codegen/loaders/schemaloader.py index dd1edee1de09..3b5f20cbbede 100644 --- a/misc/codegen/loaders/schemaloader.py +++ b/misc/codegen/loaders/schemaloader.py @@ -132,6 +132,7 @@ def _check_test_with(classes: typing.Dict[str, schema.Class]): def load(m: types.ModuleType) -> schema.Schema: includes = set() classes = {} + imported_classes = {} known = {"int", "string", "boolean"} known.update(n for n in m.__dict__ if not n.startswith("__")) import misc.codegen.lib.schemadefs as defs @@ -146,6 +147,9 @@ def load(m: types.ModuleType) -> schema.Schema: continue if isinstance(data, types.ModuleType): continue + if isinstance(data, schema.ImportedClass): + imported_classes[name] = data + continue cls = _get_class(data) if classes and not cls.bases: raise schema.Error( @@ -162,7 +166,7 @@ def load(m: types.ModuleType) -> schema.Schema: _fill_hideable_information(classes) _check_test_with(classes) - return schema.Schema(includes=includes, classes=_toposort_classes_by_group(classes), null=null) + return schema.Schema(includes=includes, classes=imported_classes | _toposort_classes_by_group(classes), null=null) def load_file(path: pathlib.Path) -> schema.Schema: diff --git a/misc/codegen/templates/ql_class.mustache b/misc/codegen/templates/ql_class.mustache index da6524d6d375..d39238ff9ba2 100644 --- a/misc/codegen/templates/ql_class.mustache +++ b/misc/codegen/templates/ql_class.mustache @@ -113,7 +113,7 @@ module Generated { */ {{type}} {{getter}}({{#is_indexed}}int index{{/is_indexed}}) { {{^synth}} - {{^is_predicate}}result = {{/is_predicate}}{{#type_is_class}}Synth::convert{{type}}FromRaw({{/type_is_class}}Synth::convert{{name}}ToRaw(this){{^root}}.(Raw::{{name}}){{/root}}.{{getter}}({{#is_indexed}}index{{/is_indexed}}){{#type_is_class}}){{/type_is_class}} + {{^is_predicate}}result = {{/is_predicate}}{{#type_is_codegen_class}}Synth::convert{{type}}FromRaw({{/type_is_codegen_class}}Synth::convert{{name}}ToRaw(this){{^root}}.(Raw::{{name}}){{/root}}.{{getter}}({{#is_indexed}}index{{/is_indexed}}){{#type_is_codegen_class}}){{/type_is_codegen_class}} {{/synth}} {{#synth}} none() diff --git a/misc/codegen/templates/ql_db.mustache b/misc/codegen/templates/ql_db.mustache index 8326bb3adb7d..e63e0aae9034 100644 --- a/misc/codegen/templates/ql_db.mustache +++ b/misc/codegen/templates/ql_db.mustache @@ -3,6 +3,10 @@ * This module holds thin fully generated class definitions around DB entities. */ module Raw { + {{#imports}} + private import {{.}} + {{/imports}} + {{#classes}} /** * INTERNAL: Do not use. diff --git a/misc/codegen/test/test_ql.py b/misc/codegen/test/test_ql.py index eef840ddad6d..e326e65a9e4f 100644 --- a/misc/codegen/test/test_ql.py +++ b/misc/codegen/test/test_ql.py @@ -12,21 +12,6 @@ def test_property_has_first_table_param_marked(): assert [p.param for p in prop.tableparams] == tableparams -@pytest.mark.parametrize("type,expected", [ - ("Foo", True), - ("Bar", True), - ("foo", False), - ("bar", False), - (None, False), -]) -def test_property_is_a_class(type, expected): - tableparams = ["a", "result", "b"] - expected_tableparams = ["a", "result" if expected else "result", "b"] - prop = ql.Property("Prop", type, tableparams=tableparams) - assert prop.type_is_class is expected - assert [p.param for p in prop.tableparams] == expected_tableparams - - indefinite_getters = [ ("Argument", "getAnArgument"), ("Element", "getAnElement"), diff --git a/misc/codegen/test/test_qlgen.py b/misc/codegen/test/test_qlgen.py index 684d3d6a1a10..431f25d5aaeb 100644 --- a/misc/codegen/test/test_qlgen.py +++ b/misc/codegen/test/test_qlgen.py @@ -448,7 +448,8 @@ def test_single_class_property(generate_classes, is_child, prev_child): ql.Property(singular="Foo", type="Bar", tablename="my_objects", tableparams=[ "this", "result"], - prev_child=prev_child, doc="foo of this my object"), + prev_child=prev_child, doc="foo of this my object", + type_is_codegen_class=True), ], )), "Bar.qll": (a_ql_class_public(name="Bar"), a_ql_stub(name="Bar"), a_ql_class(name="Bar", final=True, imports=[stub_import_prefix + "Bar"])), @@ -1006,6 +1007,7 @@ def test_hideable_property(generate_classes): final=True, properties=[ ql.Property(singular="X", type="MyObject", tablename="others", type_is_hideable=True, + type_is_codegen_class=True, tableparams=["this", "result"], doc="x of this other"), ])), } diff --git a/rust/extractor/Cargo.toml b/rust/extractor/Cargo.toml index 971a6f5e74ef..97d348f41190 100644 --- a/rust/extractor/Cargo.toml +++ b/rust/extractor/Cargo.toml @@ -33,4 +33,6 @@ codeql-extractor = { path = "../../shared/tree-sitter-extractor" } rust-extractor-macros = { path = "macros" } itertools = "0.13.0" glob = "0.3.1" +chrono = { version = "0.4.38", features = ["serde"] } +serde_json = "1.0.133" dunce = "1.0.5" diff --git a/rust/extractor/src/config.rs b/rust/extractor/src/config.rs index 70c390b99491..0e92e82a58cb 100644 --- a/rust/extractor/src/config.rs +++ b/rust/extractor/src/config.rs @@ -45,6 +45,7 @@ pub struct Config { pub scratch_dir: PathBuf, pub trap_dir: PathBuf, pub source_archive_dir: PathBuf, + pub diagnostic_dir: PathBuf, pub cargo_target_dir: Option, pub cargo_target: Option, pub cargo_features: Vec, diff --git a/rust/extractor/src/diagnostics.rs b/rust/extractor/src/diagnostics.rs new file mode 100644 index 000000000000..92743a923d4a --- /dev/null +++ b/rust/extractor/src/diagnostics.rs @@ -0,0 +1,255 @@ +use crate::config::Config; +use anyhow::Context; +use chrono::{DateTime, Utc}; +use log::{debug, info}; +use ra_ap_project_model::ProjectManifest; +use serde::ser::SerializeMap; +use serde::Serialize; +use std::collections::HashMap; +use std::fmt::Display; +use std::fs::File; +use std::path::{Path, PathBuf}; +use std::time::Instant; + +#[derive(Default, Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +#[allow(dead_code)] +enum Severity { + #[default] + Note, + Warning, + Error, +} + +#[derive(Default, Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "camelCase")] +struct Visibility { + status_page: bool, + cli_summary_table: bool, + telemetry: bool, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +#[allow(dead_code)] +enum Message { + TextMessage(String), + MarkdownMessage(String), +} + +impl Default for Message { + fn default() -> Self { + Message::TextMessage("".to_string()) + } +} + +#[derive(Default, Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct Source { + id: String, + name: String, + extractor_name: String, +} + +#[derive(Default, Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct Location { + file: PathBuf, + start_line: u32, + start_column: u32, + end_line: u32, + end_column: u32, +} + +#[derive(Default, Debug, Clone, Serialize)] +pub struct Diagnostics { + source: Source, + visibility: Visibility, + severity: Severity, + #[serde(flatten)] + message: Message, + timestamp: DateTime, + #[serde(skip_serializing_if = "Option::is_none")] + location: Option, + attributes: T, +} + +#[derive(Default, Debug, Clone, Copy, Serialize, PartialEq, Eq, Hash)] +#[serde(rename_all = "camelCase")] +pub enum ExtractionStepKind { + #[default] + LoadManifest, + LoadSource, + Parse, + Extract, +} + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ExtractionStep { + pub action: ExtractionStepKind, + pub file: PathBuf, + pub ms: u128, +} + +impl ExtractionStep { + fn new(start: Instant, action: ExtractionStepKind, file: PathBuf) -> Self { + let ret = ExtractionStep { + action, + file, + ms: start.elapsed().as_millis(), + }; + debug!("{ret:?}"); + ret + } + + pub fn load_manifest(start: Instant, target: &ProjectManifest) -> Self { + Self::new( + start, + ExtractionStepKind::LoadManifest, + PathBuf::from(target.manifest_path()), + ) + } + + pub fn parse(start: Instant, target: &Path) -> Self { + Self::new(start, ExtractionStepKind::Parse, PathBuf::from(target)) + } + + pub fn extract(start: Instant, target: &Path) -> Self { + Self::new(start, ExtractionStepKind::Extract, PathBuf::from(target)) + } + + pub fn load_source(start: Instant, target: &Path) -> Self { + Self::new(start, ExtractionStepKind::LoadSource, PathBuf::from(target)) + } +} + +#[derive(Debug, Default, Clone)] +struct HumanReadableDuration(u128); + +impl Serialize for HumanReadableDuration { + fn serialize(&self, serializer: S) -> Result { + let mut map = serializer.serialize_map(Some(2))?; + map.serialize_entry("ms", &self.0)?; + map.serialize_entry("pretty", &self.pretty())?; + map.end() + } +} + +impl HumanReadableDuration { + pub fn add(&mut self, other: u128) { + self.0 += other; + } + + pub fn pretty(&self) -> String { + let milliseconds = self.0 % 1000; + let mut seconds = self.0 / 1000; + if seconds < 60 { + return format!("{seconds}.{milliseconds:03}s"); + } + let mut minutes = seconds / 60; + seconds %= 60; + if minutes < 60 { + return format!("{minutes}min{seconds:02}.{milliseconds:03}s"); + } + let hours = minutes / 60; + minutes %= 60; + format!("{hours}h{minutes:02}min{seconds:02}.{milliseconds:03}s") + } +} + +impl From for HumanReadableDuration { + fn from(val: u128) -> Self { + HumanReadableDuration(val) + } +} + +impl Display for HumanReadableDuration { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str(&self.pretty()) + } +} + +#[derive(Debug, Default, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct DurationsSummary { + #[serde(flatten)] + durations: HashMap, + total: HumanReadableDuration, +} + +#[derive(Debug, Default, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +struct ExtractionSummary { + number_of_manifests: usize, + number_of_files: usize, + durations: DurationsSummary, +} + +type ExtractionDiagnostics = Diagnostics; + +fn summary(start: Instant, steps: &[ExtractionStep]) -> ExtractionSummary { + let mut number_of_manifests = 0; + let mut number_of_files = 0; + let mut durations = HashMap::new(); + for step in steps { + match &step.action { + ExtractionStepKind::LoadManifest => { + number_of_manifests += 1; + } + ExtractionStepKind::Parse => { + number_of_files += 1; + } + _ => {} + } + durations + .entry(step.action) + .or_insert(HumanReadableDuration(0)) + .add(step.ms); + } + let total = start.elapsed().as_millis().into(); + for (key, value) in &durations { + info!("total duration ({key:?}): {value}"); + } + info!("total duration: {total}"); + ExtractionSummary { + number_of_manifests, + number_of_files, + durations: DurationsSummary { durations, total }, + } +} + +pub fn emit_extraction_diagnostics( + start: Instant, + config: &Config, + steps: &[ExtractionStep], +) -> anyhow::Result<()> { + let summary = summary(start, steps); + let diagnostics = ExtractionDiagnostics { + source: Source { + id: "rust/extractor/telemetry".to_owned(), + name: "telemetry".to_string(), + extractor_name: "rust".to_string(), + }, + visibility: Visibility { + telemetry: true, + ..Default::default() + }, + timestamp: Utc::now(), + attributes: summary, + ..Default::default() + }; + + std::fs::create_dir_all(&config.diagnostic_dir).with_context(|| { + format!( + "creating diagnostics directory {}", + config.diagnostic_dir.display() + ) + })?; + let target = config.diagnostic_dir.join("extraction.jsonc"); + let mut output = File::create(&target) + .with_context(|| format!("creating diagnostics file {}", target.display()))?; + serde_json::to_writer_pretty(&mut output, &diagnostics) + .with_context(|| format!("writing to diagnostics file {}", target.display()))?; + Ok(()) +} diff --git a/rust/extractor/src/generated/.generated.list b/rust/extractor/src/generated/.generated.list index 5ae8000953f3..d50633c020b6 100644 --- a/rust/extractor/src/generated/.generated.list +++ b/rust/extractor/src/generated/.generated.list @@ -1,2 +1,2 @@ mod.rs 4bcb9def847469aae9d8649461546b7c21ec97cf6e63d3cf394e339915ce65d7 4bcb9def847469aae9d8649461546b7c21ec97cf6e63d3cf394e339915ce65d7 -top.rs aad41fc41df08b35aeca0700426e38538c23aef9030b42aeaac964dced524575 aad41fc41df08b35aeca0700426e38538c23aef9030b42aeaac964dced524575 +top.rs c77ccf1e73a5b139f768e80580a261f7f3ab76823a1c9095b06f704a4912e8f1 c77ccf1e73a5b139f768e80580a261f7f3ab76823a1c9095b06f704a4912e8f1 diff --git a/rust/extractor/src/generated/top.rs b/rust/extractor/src/generated/top.rs index 48586680e7dd..d74cf3c97498 100644 --- a/rust/extractor/src/generated/top.rs +++ b/rust/extractor/src/generated/top.rs @@ -4,6 +4,15 @@ use crate::trap; +#[derive(Debug)] +pub struct File { + _unused: () +} + +impl trap::TrapClass for File { + fn class_name() -> &'static str { "File" } +} + #[derive(Debug)] pub struct Element { _unused: () @@ -13,6 +22,37 @@ impl trap::TrapClass for Element { fn class_name() -> &'static str { "Element" } } +#[derive(Debug)] +pub struct ExtractorStep { + pub id: trap::TrapId, + pub action: String, + pub file: trap::Label, + pub duration_ms: usize, +} + +impl trap::TrapEntry for ExtractorStep { + fn extract_id(&mut self) -> trap::TrapId { + std::mem::replace(&mut self.id, trap::TrapId::Star) + } + + fn emit(self, id: trap::Label, out: &mut trap::Writer) { + out.add_tuple("extractor_steps", vec![id.into(), self.action.into(), self.file.into(), self.duration_ms.into()]); + } +} + +impl trap::TrapClass for ExtractorStep { + fn class_name() -> &'static str { "ExtractorStep" } +} + +impl From> for trap::Label { + fn from(value: trap::Label) -> Self { + // SAFETY: this is safe because in the dbscheme ExtractorStep is a subclass of Element + unsafe { + Self::from_untyped(value.as_untyped()) + } + } +} + #[derive(Debug)] pub struct Locatable { _unused: () diff --git a/rust/extractor/src/main.rs b/rust/extractor/src/main.rs index ecbdc965f8e7..86aeb09f6a4d 100644 --- a/rust/extractor/src/main.rs +++ b/rust/extractor/src/main.rs @@ -1,13 +1,16 @@ +use crate::diagnostics::{emit_extraction_diagnostics, ExtractionStep}; use crate::rust_analyzer::path_to_file_id; +use crate::trap::TrapId; use anyhow::Context; use archive::Archiver; -use log::info; +use log::{info, warn}; use ra_ap_hir::Semantics; use ra_ap_ide_db::line_index::{LineCol, LineIndex}; use ra_ap_ide_db::RootDatabase; -use ra_ap_project_model::ProjectManifest; +use ra_ap_project_model::{CargoConfig, ProjectManifest}; use ra_ap_vfs::Vfs; use rust_analyzer::{ParseResult, RustAnalyzer}; +use std::time::Instant; use std::{ collections::HashMap, path::{Path, PathBuf}, @@ -15,6 +18,7 @@ use std::{ mod archive; mod config; +mod diagnostics; pub mod generated; mod qltest; mod rust_analyzer; @@ -24,18 +28,31 @@ pub mod trap; struct Extractor<'a> { archiver: &'a Archiver, traps: &'a trap::TrapFileProvider, + steps: Vec, } -impl Extractor<'_> { - fn extract(&self, rust_analyzer: &rust_analyzer::RustAnalyzer, file: &std::path::Path) { +impl<'a> Extractor<'a> { + pub fn new(archiver: &'a Archiver, traps: &'a trap::TrapFileProvider) -> Self { + Self { + archiver, + traps, + steps: Vec::new(), + } + } + + fn extract(&mut self, rust_analyzer: &rust_analyzer::RustAnalyzer, file: &std::path::Path) { self.archiver.archive(file); + let before_parse = Instant::now(); let ParseResult { ast, text, errors, semantics_info, } = rust_analyzer.parse(file); + self.steps.push(ExtractionStep::parse(before_parse, file)); + + let before_extract = Instant::now(); let line_index = LineIndex::new(text.as_ref()); let display_path = file.to_string_lossy(); let mut trap = self.traps.create("source", file); @@ -73,22 +90,79 @@ impl Extractor<'_> { err.to_string() ) }); + self.steps + .push(ExtractionStep::extract(before_extract, file)); } pub fn extract_with_semantics( - &self, + &mut self, file: &Path, semantics: &Semantics<'_, RootDatabase>, vfs: &Vfs, ) { self.extract(&RustAnalyzer::new(vfs, semantics), file); } - pub fn extract_without_semantics(&self, file: &Path, reason: &str) { + + pub fn extract_without_semantics(&mut self, file: &Path, reason: &str) { self.extract(&RustAnalyzer::WithoutSemantics { reason }, file); } + + pub fn load_manifest( + &mut self, + project: &ProjectManifest, + config: &CargoConfig, + ) -> Option<(RootDatabase, Vfs)> { + let before = Instant::now(); + let ret = RustAnalyzer::load_workspace(project, config); + self.steps + .push(ExtractionStep::load_manifest(before, project)); + ret + } + + pub fn load_source( + &mut self, + file: &Path, + semantics: &Semantics<'_, RootDatabase>, + vfs: &Vfs, + ) -> Result<(), String> { + let before = Instant::now(); + let Some(id) = path_to_file_id(file, vfs) else { + return Err("not included in files loaded from manifest".to_string()); + }; + if semantics.file_to_module_def(id).is_none() { + return Err("not included as a module".to_string()); + } + self.steps.push(ExtractionStep::load_source(before, file)); + Ok(()) + } + + pub fn emit_extraction_diagnostics( + self, + start: Instant, + cfg: &config::Config, + ) -> anyhow::Result<()> { + emit_extraction_diagnostics(start, cfg, &self.steps)?; + let mut trap = self.traps.create("diagnostics", "extraction"); + for step in self.steps { + let file = trap.emit_file(&step.file); + let duration_ms = usize::try_from(step.ms).unwrap_or_else(|_e| { + warn!("extraction step duration overflowed ({step:?})"); + i32::MAX as usize + }); + trap.emit(generated::ExtractorStep { + id: TrapId::Star, + action: format!("{:?}", step.action), + file, + duration_ms, + }); + } + trap.commit()?; + Ok(()) + } } fn main() -> anyhow::Result<()> { + let start = Instant::now(); let mut cfg = config::Config::extract().context("failed to load configuration")?; stderrlog::new() .module(module_path!()) @@ -103,10 +177,7 @@ fn main() -> anyhow::Result<()> { let archiver = archive::Archiver { root: cfg.source_archive_dir.clone(), }; - let extractor = Extractor { - archiver: &archiver, - traps: &traps, - }; + let mut extractor = Extractor::new(&archiver, &traps); let files: Vec = cfg .inputs .iter() @@ -132,21 +203,13 @@ fn main() -> anyhow::Result<()> { } let cargo_config = cfg.to_cargo_config(); for (manifest, files) in map.values().filter(|(_, files)| !files.is_empty()) { - if let Some((ref db, ref vfs)) = RustAnalyzer::load_workspace(manifest, &cargo_config) { + if let Some((ref db, ref vfs)) = extractor.load_manifest(manifest, &cargo_config) { let semantics = Semantics::new(db); for file in files { - let Some(id) = path_to_file_id(file, vfs) else { - extractor.extract_without_semantics( - file, - "not included in files loaded from manifest", - ); - continue; + match extractor.load_source(file, &semantics, vfs) { + Ok(()) => extractor.extract_with_semantics(file, &semantics, vfs), + Err(reason) => extractor.extract_without_semantics(file, &reason), }; - if semantics.file_to_module_def(id).is_none() { - extractor.extract_without_semantics(file, "not included as a module"); - continue; - } - extractor.extract_with_semantics(file, &semantics, vfs); } } else { for file in files { @@ -155,5 +218,5 @@ fn main() -> anyhow::Result<()> { } } - Ok(()) + extractor.emit_extraction_diagnostics(start, &cfg) } diff --git a/rust/extractor/src/translate/base.rs b/rust/extractor/src/translate/base.rs index 003c86919b67..396816c6a0c8 100644 --- a/rust/extractor/src/translate/base.rs +++ b/rust/extractor/src/translate/base.rs @@ -4,7 +4,6 @@ use crate::generated::{self}; use crate::rust_analyzer::FileSemanticInformation; use crate::trap::{DiagnosticSeverity, TrapFile, TrapId}; use crate::trap::{Label, TrapClass}; -use codeql_extractor::trap::{self}; use itertools::Either; use log::Level; use ra_ap_base_db::salsa::InternKey; @@ -65,7 +64,7 @@ macro_rules! emit_detached { pub struct Translator<'a> { pub trap: TrapFile, path: &'a str, - label: trap::Label, + label: Label, line_index: LineIndex, file_id: Option, pub semantics: Option<&'a Semantics<'a, RootDatabase>>, @@ -75,7 +74,7 @@ impl<'a> Translator<'a> { pub fn new( trap: TrapFile, path: &'a str, - label: trap::Label, + label: Label, line_index: LineIndex, semantic_info: Option<&FileSemanticInformation<'a>>, ) -> Translator<'a> { diff --git a/rust/extractor/src/trap.rs b/rust/extractor/src/trap.rs index 58f56fb2b1e5..6c8e9b2c3c84 100644 --- a/rust/extractor/src/trap.rs +++ b/rust/extractor/src/trap.rs @@ -1,5 +1,5 @@ -use crate::config; use crate::config::Compression; +use crate::{config, generated}; use codeql_extractor::{extractor, file_paths, trap}; use log::debug; use ra_ap_ide_db::line_index::LineCol; @@ -138,7 +138,7 @@ pub enum DiagnosticSeverity { impl TrapFile { pub fn emit_location_label( &mut self, - file_label: UntypedLabel, + file_label: Label, start: LineCol, end: LineCol, ) -> UntypedLabel { @@ -149,7 +149,7 @@ impl TrapFile { extractor::location_label( &mut self.writer, trap::Location { - file_label, + file_label: file_label.as_untyped(), start_line, start_column, end_line, @@ -159,7 +159,7 @@ impl TrapFile { } pub fn emit_location( &mut self, - file_label: UntypedLabel, + file_label: Label, entity_label: Label, start: LineCol, end: LineCol, @@ -192,8 +192,10 @@ impl TrapFile { ], ); } - pub fn emit_file(&mut self, absolute_path: &Path) -> trap::Label { - extractor::populate_file(&mut self.writer, absolute_path) + pub fn emit_file(&mut self, absolute_path: &Path) -> Label { + let untyped = extractor::populate_file(&mut self.writer, absolute_path); + // SAFETY: populate_file emits `@file` typed labels + unsafe { Label::from_untyped(untyped) } } pub fn label(&mut self, id: TrapId) -> Label { @@ -243,8 +245,8 @@ impl TrapFileProvider { }) } - pub fn create(&self, category: &str, key: &Path) -> TrapFile { - let path = file_paths::path_for(&self.trap_dir.join(category), key, "trap"); + pub fn create(&self, category: &str, key: impl AsRef) -> TrapFile { + let path = file_paths::path_for(&self.trap_dir.join(category), key.as_ref(), "trap"); debug!("creating trap file {}", path.display()); let mut writer = trap::Writer::new(); extractor::populate_empty_location(&mut writer); diff --git a/rust/ql/.generated.list b/rust/ql/.generated.list index 575320194090..03441896ce03 100644 --- a/rust/ql/.generated.list +++ b/rust/ql/.generated.list @@ -226,6 +226,8 @@ lib/codeql/rust/elements/internal/ExternCrateImpl.qll ade4df9d3f87daf6534b8e79ff lib/codeql/rust/elements/internal/ExternItemImpl.qll 577c8ac387c47746e3b45f943374c7ab641e8ad119e8591c31f219a5f08d3a29 bba88b974d1c03c78e0caf3d8f4118426d2aa8bd6ffd6f59a3da8ff1524a173f lib/codeql/rust/elements/internal/ExternItemListConstructor.qll 9e4f6a036707c848c0553119272fd2b11c1740dd9910a626a9a0cf68a55b249b efde86b18bd419154fb5b6d28790a14ea989b317d84b5c1ddbdfb29c6924fd86 lib/codeql/rust/elements/internal/ExternItemListImpl.qll e89d0cf938f6e137ba1ce7907a923b1ab2be7be2fdd642c3b7a722f11b9199f8 85906d3ce89e5abc301cc96ea5104d53e90af3f5f22f8d54ec437687096e39d8 +lib/codeql/rust/elements/internal/ExtractorStep.qll 1c65668007ea71d05333e44132eccc01dc2a2b4908fb37d0a73995119d3ed5f0 8cbe1eeb35bc2bc95c1b7765070d1ff58aae03fd28dc94896b091858eea40efe +lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll 00c527a3139ad399ea1efd0ebe4656372d70f6c4e79136bc497a6cb84becae8e 93817f3dddeaf2c0964ab31c2df451dcee0aeba7cb6520803d8ce42cefcb3703 lib/codeql/rust/elements/internal/FieldExprConstructor.qll b3be2c4ccaf2c8a1283f3d5349d7f4f49f87b35e310ef33491023c5ab6f3abc5 645d0d4073b032f6b7284fc36a10a6ec85596fb95c68f30c09504f2c5a6f789f lib/codeql/rust/elements/internal/FieldListImpl.qll 02a09d1d146030c68cead4614f4eef75854f19e55ed1eda60b34c4858a8d4a88 9b9f5e77546434c771d2f785119577ec46569a18473daa4169fb84a097369493 lib/codeql/rust/elements/internal/FnPtrTypeReprConstructor.qll 61d8808ea027a6e04d5304c880974332a0195451f6b4474f84b3695ec907d865 0916c63a02b01a839fe23ec8b189d37dc1b8bc4e1ba753cbf6d6f5067a46965a @@ -456,6 +458,7 @@ lib/codeql/rust/elements/internal/generated/ExternBlock.qll c292d804a1f8d2cf6a44 lib/codeql/rust/elements/internal/generated/ExternCrate.qll 35fea4e810a896c1656adb4682c4c3bc20283768073e26ae064189ce310433c8 fc504dff79ba758d89b10cd5049539fbc766ee9862ff495066cea26abf0b5e0b lib/codeql/rust/elements/internal/generated/ExternItem.qll 749b064ad60f32197d5b85e25929afe18e56e12f567b73e21e43e2fdf4c447e3 e2c2d423876675cf2dae399ca442aef7b2860319da9bfadeff29f2c6946f8de7 lib/codeql/rust/elements/internal/generated/ExternItemList.qll 6bc97fdae6c411cab5c501129c1d6c2321c1011cccb119515d75d07dc55c253b 6b5aa808025c0a4270cac540c07ba6faede1b3c70b8db5fd89ec5d46df9041b2 +lib/codeql/rust/elements/internal/generated/ExtractorStep.qll b83ce7f18009bdd36374260652c2a8a5cd5a9b5404a1c147bbec49ad251e43f3 e6e55595300126f9c5a6fd7bde5321b2a0026b491326114d16fcc2395a1fc483 lib/codeql/rust/elements/internal/generated/FieldExpr.qll 3e506b5cb93793ec30f56bb637a600db869fcba6181b068516a671d55c362739 7bbf953696d763ad6b210f378f487ba85b875fa115b22c0c0508599a63633502 lib/codeql/rust/elements/internal/generated/FieldList.qll 43c13c6e3c9ba75a7a4cb870fc4f18752001584d48b9df0734055a6ebb789331 7c51b0b13eb02f1286d3365e53a976ba2655c4dbd8e735bc11c8b205c829e1ee lib/codeql/rust/elements/internal/generated/FnPtrTypeRepr.qll d490ab9f2e3654d9abde18a06e534abd99ca62f518ca08670b696a97e9d5c592 01500319820f66cb4bbda6fe7c26270f76ea934efff4bb3cbf88e9b1e07e8be2 @@ -519,7 +522,7 @@ lib/codeql/rust/elements/internal/generated/ParamList.qll c808c9d84dd7800573832b lib/codeql/rust/elements/internal/generated/ParenExpr.qll bc0731505bfe88516205ec360582a4222d2681d11342c93e15258590ddee82f2 d4bd6e0c80cf1d63746c88d4bcb3a01d4c75732e5da09e3ebd9437ced227fb60 lib/codeql/rust/elements/internal/generated/ParenPat.qll 4f168ef5d5bb87a903251cc31b2e44a759b099ec69c90af31783fbb15778c940 0e34f94a45a13396fd57d94c245dc64d1adde2ab0e22b56946f7e94c04e297fc lib/codeql/rust/elements/internal/generated/ParenTypeRepr.qll 40ab5c592e7699c621787793743e33988de71ff42ca27599f5ab3ddb70e3f7d8 12c0a6eed2202ee3e892f61da3b3ce77ac3190854cdf3097e8d2be98aa3cb91d -lib/codeql/rust/elements/internal/generated/ParentChild.qll c3f6352ef56aabb6bc784deafc35807b32df6f41c3bca3437221166776885c2d 3f04580798ab351e5835957379ced2b45fec628b76578c0f33f571cf03abfc83 +lib/codeql/rust/elements/internal/generated/ParentChild.qll ed0af2cad8ec4d612e3c03d99548444298ae975516f2c3e909026fe5d31ab467 57801012cbe803516a093bbc0c5dcf839cb43df97a87fd2028ad2be400882c50 lib/codeql/rust/elements/internal/generated/Pat.qll 3605ac062be2f294ee73336e9669027b8b655f4ad55660e1eab35266275154ee 7f9400db2884d336dd1d21df2a8093759c2a110be9bf6482ce8e80ae0fd74ed4 lib/codeql/rust/elements/internal/generated/Path.qll 4c1c8e840ed57880e574142b081b11d7a7428a009f10e3aa8f4645e211f6b2e0 989668cf0f1bdee7557e2f97c01e41d2a56848227fed41477833f5fc1e1d35f6 lib/codeql/rust/elements/internal/generated/PathExpr.qll 4ff4b2ab77bce1dbfddb315e7d1ff13d6fcd6bb7c30c105402f8082d05f1d337 fbc4f4e05da75ab543af33cfb620cfe09239e2574b8312f2c5bedca8a65ea6e8 @@ -532,7 +535,7 @@ lib/codeql/rust/elements/internal/generated/PtrTypeRepr.qll 51d1e9e683fc79dddbff lib/codeql/rust/elements/internal/generated/PureSynthConstructors.qll e5b8e69519012bbaae29dcb82d53f7f7ecce368c0358ec27ef6180b228a0057f e5b8e69519012bbaae29dcb82d53f7f7ecce368c0358ec27ef6180b228a0057f lib/codeql/rust/elements/internal/generated/RangeExpr.qll 23cca03bf43535f33b22a38894f70d669787be4e4f5b8fe5c8f7b964d30e9027 18624cef6c6b679eeace2a98737e472432e0ead354cca02192b4d45330f047c9 lib/codeql/rust/elements/internal/generated/RangePat.qll 80826a6a6868a803aa2372e31c52a03e1811a3f1f2abdb469f91ca0bfdd9ecb6 34ee1e208c1690cba505dff2c588837c0cd91e185e2a87d1fe673191962276a9 -lib/codeql/rust/elements/internal/generated/Raw.qll f52ff91f985848ca0e251efee1e246ae80fdca13f530df301f7090a5b18bcf13 136a84549b183d222fb6063d34d4b714b7dd42f6eb3f756894285bf405c24a22 +lib/codeql/rust/elements/internal/generated/Raw.qll bcc8cfc3843e297546ba563bbd261f48863967588dbbf445bbb0894bf4539812 b8324ce13f8a5fffdcf2c24ca6f685ac11d959eb3139bdc6f89430c83dd6ddfd lib/codeql/rust/elements/internal/generated/RecordExpr.qll eb6cb662e463f9260efae1a6ce874fa781172063b916ef1963f861e9942d308d 1a21cbccc8f3799ff13281e822818ebfb21d81591720a427cac3625512cb9d40 lib/codeql/rust/elements/internal/generated/RecordExprField.qll 7e9f8663d3b74ebbc9603b10c9912f082febba6bd73d344b100bbd3edf837802 fbe6b578e7fd5d5a6f21bbb8c388957ab7210a6a249ec71510a50fb35b319ea1 lib/codeql/rust/elements/internal/generated/RecordExprFieldList.qll 179a97211fe7aa6265085d4d54115cdbc0e1cd7c9b2135591e8f36d6432f13d3 dd44bbbc1e83a1ed3a587afb729d7debf7aeb7b63245de181726af13090e50c0 @@ -558,8 +561,8 @@ lib/codeql/rust/elements/internal/generated/Static.qll ea22838e0b7d9796dfaf5deda lib/codeql/rust/elements/internal/generated/Stmt.qll 8473ff532dd5cc9d7decaddcd174b94d610f6ca0aec8e473cc051dad9f3db917 6ef7d2b5237c2dbdcacbf7d8b39109d4dc100229f2b28b5c9e3e4fbf673ba72b lib/codeql/rust/elements/internal/generated/StmtList.qll a667193e32341e17400867c6e359878c4e645ef9f5f4d97676afc0283a33a026 a320ed678ee359302e2fc1b70a9476705cd616fcfa44a499d32f0c7715627f73 lib/codeql/rust/elements/internal/generated/Struct.qll 4d57f0db12dc7ad3e31e750a24172ef1505406b4dab16386af0674bd18bf8f4b 1a73c83df926b996f629316f74c61ea775be04532ab61b56af904223354f033e -lib/codeql/rust/elements/internal/generated/Synth.qll 0bb53e7d628fcd74a28fb7b00c51dc7f212a775894a4dc485a68089a7c02d766 6c031f7c95974521f17d6fedd429d35a1aea142b7cef5123f34504a79c21670b -lib/codeql/rust/elements/internal/generated/SynthConstructors.qll 8f7f08d1599e2dcbe553a7e4428a8787cef76b1d12b88f1b5925a4c59d617fae 8f7f08d1599e2dcbe553a7e4428a8787cef76b1d12b88f1b5925a4c59d617fae +lib/codeql/rust/elements/internal/generated/Synth.qll 839aa6b03b461ed9c79192d905969c63c99e7a12074e3487e74d2163d20ee499 44b8caba3186aaeb6d6b5759610627f8bc791e62001acfd1a66324086c17536f +lib/codeql/rust/elements/internal/generated/SynthConstructors.qll 3ceb5f6ee40b94955ce5f47feb454cc9129941aad3cdbe6e337bbe41e76a8a23 3ceb5f6ee40b94955ce5f47feb454cc9129941aad3cdbe6e337bbe41e76a8a23 lib/codeql/rust/elements/internal/generated/Token.qll 77a91a25ca5669703cf3a4353b591cef4d72caa6b0b9db07bb9e005d69c848d1 2fdffc4882ed3a6ca9ac6d1fb5f1ac5a471ca703e2ffdc642885fa558d6e373b lib/codeql/rust/elements/internal/generated/TokenTree.qll 8577c2b097c1be2f0f7daa5acfcf146f78674a424d99563e08a84dd3e6d91b46 d2f30764e84dbfc0a6a5d3d8a5f935cd432413688cb32da9c94e420fbc10665c lib/codeql/rust/elements/internal/generated/Trait.qll 8fa41b50fa0f68333534f2b66bb4ec8e103ff09ac8fa5c2cc64bc04beafec205 ce1c9aa6d0e2f05d28aab8e1165c3b9fb8e24681ade0cf6a9df2e8617abeae7e @@ -593,7 +596,7 @@ lib/codeql/rust/elements/internal/generated/WhileExpr.qll 7edf1f23fbf953a2baabcd lib/codeql/rust/elements/internal/generated/WildcardPat.qll d74b70b57a0a66bfae017a329352a5b27a6b9e73dd5521d627f680e810c6c59e 4b913b548ba27ff3c82fcd32cf996ff329cb57d176d3bebd0fcef394486ea499 lib/codeql/rust/elements/internal/generated/YeetExpr.qll cac328200872a35337b4bcb15c851afb4743f82c080f9738d295571eb01d7392 94af734eea08129b587fed849b643e7572800e8330c0b57d727d41abda47930b lib/codeql/rust/elements/internal/generated/YieldExpr.qll 37e5f0c1e373a22bbc53d8b7f2c0e1f476e5be5080b8437c5e964f4e83fad79a 4a9a68643401637bf48e5c2b2f74a6bf0ddcb4ff76f6bffb61d436b685621e85 -lib/codeql/rust/elements.qll 35959f2de54b6ee534fc592e1928c0829aa6e881e281a5acf724c10e4c685070 35959f2de54b6ee534fc592e1928c0829aa6e881e281a5acf724c10e4c685070 +lib/codeql/rust/elements.qll a055d1f5bf70c9f8f6d5e34087146e813f3452b225b07b64f087d6e61f52a3b3 a055d1f5bf70c9f8f6d5e34087146e813f3452b225b07b64f087d6e61f52a3b3 test/extractor-tests/generated/Abi/Abi.ql 7f6e7dc4af86eca3ebdc79b10373988cd0871bd78b51997d3cffd969105e5fdd 2f936b6ca005c6157c755121584410c03e4a3949c23bee302fbe05ee10ce118f test/extractor-tests/generated/Abi/Abi_getAbiString.ql a496762fcec5a0887b87023bbf93e9b650f02e20113e25c44d6e4281ae8f5335 14109c7ce11ba25e3cd6e7f1b3fcb4cb00622f2a4eac91bfe43145c5f366bc52 test/extractor-tests/generated/ArgList/ArgList.ql e412927756e72165d0e7c5c9bd3fca89d08197bbf760db8fb7683c64bb2229bc 043dba8506946fbb87753e22c387987d7eded6ddb963aa067f9e60ef9024d684 diff --git a/rust/ql/.gitattributes b/rust/ql/.gitattributes index 34d2a4ab6815..a45b64fc8b74 100644 --- a/rust/ql/.gitattributes +++ b/rust/ql/.gitattributes @@ -228,6 +228,8 @@ /lib/codeql/rust/elements/internal/ExternItemImpl.qll linguist-generated /lib/codeql/rust/elements/internal/ExternItemListConstructor.qll linguist-generated /lib/codeql/rust/elements/internal/ExternItemListImpl.qll linguist-generated +/lib/codeql/rust/elements/internal/ExtractorStep.qll linguist-generated +/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll linguist-generated /lib/codeql/rust/elements/internal/FieldExprConstructor.qll linguist-generated /lib/codeql/rust/elements/internal/FieldListImpl.qll linguist-generated /lib/codeql/rust/elements/internal/FnPtrTypeReprConstructor.qll linguist-generated @@ -458,6 +460,7 @@ /lib/codeql/rust/elements/internal/generated/ExternCrate.qll linguist-generated /lib/codeql/rust/elements/internal/generated/ExternItem.qll linguist-generated /lib/codeql/rust/elements/internal/generated/ExternItemList.qll linguist-generated +/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll linguist-generated /lib/codeql/rust/elements/internal/generated/FieldExpr.qll linguist-generated /lib/codeql/rust/elements/internal/generated/FieldList.qll linguist-generated /lib/codeql/rust/elements/internal/generated/FnPtrTypeRepr.qll linguist-generated diff --git a/rust/ql/integration-tests/conftest.py b/rust/ql/integration-tests/conftest.py index fbe443a49276..a1fbcf4e18d2 100644 --- a/rust/ql/integration-tests/conftest.py +++ b/rust/ql/integration-tests/conftest.py @@ -9,13 +9,26 @@ def cargo(cwd): assert (cwd / "Cargo.toml").exists() (cwd / "rust-project.json").unlink(missing_ok=True) +@pytest.fixture(scope="session") +def rust_sysroot_src() -> str: + rust_sysroot = pathlib.Path(commands.run("rustc --print sysroot", _capture=True)) + ret = rust_sysroot.joinpath("lib", "rustlib", "src", "rust", "library") + assert ret.exists() + return str(ret) @pytest.fixture -def rust_project(cwd): +def rust_project(cwd, rust_sysroot_src): project_file = cwd / "rust-project.json" assert project_file.exists() - rust_sysroot = pathlib.Path(commands.run("rustc --print sysroot", _capture=True)) project = json.loads(project_file.read_text()) - project["sysroot_src"] = str(rust_sysroot.joinpath("lib", "rustlib", "src", "rust", "library")) + project["sysroot_src"] = rust_sysroot_src project_file.write_text(json.dumps(project, indent=4)) (cwd / "Cargo.toml").unlink(missing_ok=True) + +@pytest.fixture +def rust_check_diagnostics(check_diagnostics): + check_diagnostics.redact += [ + "attributes.durations.*.ms", + "attributes.durations.*.pretty", + ] + return check_diagnostics diff --git a/rust/ql/integration-tests/hello-project/diagnostics.expected b/rust/ql/integration-tests/hello-project/diagnostics.expected new file mode 100644 index 000000000000..f938b5b8ab5b --- /dev/null +++ b/rust/ql/integration-tests/hello-project/diagnostics.expected @@ -0,0 +1,39 @@ +{ + "attributes": { + "durations": { + "extract": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadManifest": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadSource": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "parse": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "total": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + } + }, + "numberOfFiles": 5, + "numberOfManifests": 1 + }, + "severity": "note", + "source": { + "extractorName": "rust", + "id": "rust/extractor/telemetry", + "name": "telemetry" + }, + "visibility": { + "cliSummaryTable": false, + "statusPage": false, + "telemetry": true + } +} diff --git a/rust/ql/integration-tests/hello-project/steps.cargo.expected b/rust/ql/integration-tests/hello-project/steps.cargo.expected new file mode 100644 index 000000000000..61987aa564bb --- /dev/null +++ b/rust/ql/integration-tests/hello-project/steps.cargo.expected @@ -0,0 +1,15 @@ +| Cargo.toml:0:0:0:0 | LoadManifest(Cargo.toml) | +| src/directory_module/mod.rs:0:0:0:0 | Extract(src/directory_module/mod.rs) | +| src/directory_module/mod.rs:0:0:0:0 | LoadSource(src/directory_module/mod.rs) | +| src/directory_module/mod.rs:0:0:0:0 | Parse(src/directory_module/mod.rs) | +| src/directory_module/nested_module.rs:0:0:0:0 | Extract(src/directory_module/nested_module.rs) | +| src/directory_module/nested_module.rs:0:0:0:0 | LoadSource(src/directory_module/nested_module.rs) | +| src/directory_module/nested_module.rs:0:0:0:0 | Parse(src/directory_module/nested_module.rs) | +| src/directory_module/not_loaded.rs:0:0:0:0 | Extract(src/directory_module/not_loaded.rs) | +| src/directory_module/not_loaded.rs:0:0:0:0 | Parse(src/directory_module/not_loaded.rs) | +| src/file_module.rs:0:0:0:0 | Extract(src/file_module.rs) | +| src/file_module.rs:0:0:0:0 | LoadSource(src/file_module.rs) | +| src/file_module.rs:0:0:0:0 | Parse(src/file_module.rs) | +| src/main.rs:0:0:0:0 | Extract(src/main.rs) | +| src/main.rs:0:0:0:0 | LoadSource(src/main.rs) | +| src/main.rs:0:0:0:0 | Parse(src/main.rs) | diff --git a/rust/ql/integration-tests/hello-project/steps.ql b/rust/ql/integration-tests/hello-project/steps.ql new file mode 100644 index 000000000000..17358a1c1000 --- /dev/null +++ b/rust/ql/integration-tests/hello-project/steps.ql @@ -0,0 +1,4 @@ +import codeql.rust.elements.internal.ExtractorStep + +from ExtractorStep step +select step diff --git a/rust/ql/integration-tests/hello-project/steps.rust-project.expected b/rust/ql/integration-tests/hello-project/steps.rust-project.expected new file mode 100644 index 000000000000..d1d2e9ddee3f --- /dev/null +++ b/rust/ql/integration-tests/hello-project/steps.rust-project.expected @@ -0,0 +1,15 @@ +| rust-project.json:0:0:0:0 | LoadManifest(rust-project.json) | +| src/directory_module/mod.rs:0:0:0:0 | Extract(src/directory_module/mod.rs) | +| src/directory_module/mod.rs:0:0:0:0 | LoadSource(src/directory_module/mod.rs) | +| src/directory_module/mod.rs:0:0:0:0 | Parse(src/directory_module/mod.rs) | +| src/directory_module/nested_module.rs:0:0:0:0 | Extract(src/directory_module/nested_module.rs) | +| src/directory_module/nested_module.rs:0:0:0:0 | LoadSource(src/directory_module/nested_module.rs) | +| src/directory_module/nested_module.rs:0:0:0:0 | Parse(src/directory_module/nested_module.rs) | +| src/directory_module/not_loaded.rs:0:0:0:0 | Extract(src/directory_module/not_loaded.rs) | +| src/directory_module/not_loaded.rs:0:0:0:0 | Parse(src/directory_module/not_loaded.rs) | +| src/file_module.rs:0:0:0:0 | Extract(src/file_module.rs) | +| src/file_module.rs:0:0:0:0 | LoadSource(src/file_module.rs) | +| src/file_module.rs:0:0:0:0 | Parse(src/file_module.rs) | +| src/main.rs:0:0:0:0 | Extract(src/main.rs) | +| src/main.rs:0:0:0:0 | LoadSource(src/main.rs) | +| src/main.rs:0:0:0:0 | Parse(src/main.rs) | diff --git a/rust/ql/integration-tests/hello-project/summary.expected b/rust/ql/integration-tests/hello-project/summary.expected index 09e03490eb13..44c14e790fe3 100644 --- a/rust/ql/integration-tests/hello-project/summary.expected +++ b/rust/ql/integration-tests/hello-project/summary.expected @@ -1,4 +1,4 @@ -| Elements extracted | 50 | +| Elements extracted | 65 | | Elements unextracted | 0 | | Extraction errors | 0 | | Extraction warnings | 1 | diff --git a/rust/ql/integration-tests/hello-project/test_project.py b/rust/ql/integration-tests/hello-project/test_project.py index 6a7e21ebedbc..c52002d0b221 100644 --- a/rust/ql/integration-tests/hello-project/test_project.py +++ b/rust/ql/integration-tests/hello-project/test_project.py @@ -1,5 +1,10 @@ -def test_cargo(codeql, rust, cargo, check_source_archive): +import pytest + + +@pytest.mark.ql_test("steps.ql", expected=".cargo.expected") +def test_cargo(codeql, rust, cargo, check_source_archive, rust_check_diagnostics): codeql.database.create() -def test_rust_project(codeql, rust, rust_project, check_source_archive): +@pytest.mark.ql_test("steps.ql", expected=".rust-project.expected") +def test_rust_project(codeql, rust, rust_project, check_source_archive, rust_check_diagnostics): codeql.database.create() diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected new file mode 100644 index 000000000000..7d44256db5a4 --- /dev/null +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.cargo.expected @@ -0,0 +1,39 @@ +{ + "attributes": { + "durations": { + "extract": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadManifest": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadSource": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "parse": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "total": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + } + }, + "numberOfFiles": 4, + "numberOfManifests": 2 + }, + "severity": "note", + "source": { + "extractorName": "rust", + "id": "rust/extractor/telemetry", + "name": "telemetry" + }, + "visibility": { + "cliSummaryTable": false, + "statusPage": false, + "telemetry": true + } +} diff --git a/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected new file mode 100644 index 000000000000..4c5dbc75d84b --- /dev/null +++ b/rust/ql/integration-tests/hello-workspace/diagnostics.rust-project.expected @@ -0,0 +1,39 @@ +{ + "attributes": { + "durations": { + "extract": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadManifest": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "loadSource": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "parse": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + }, + "total": { + "ms": "__REDACTED__", + "pretty": "__REDACTED__" + } + }, + "numberOfFiles": 4, + "numberOfManifests": 1 + }, + "severity": "note", + "source": { + "extractorName": "rust", + "id": "rust/extractor/telemetry", + "name": "telemetry" + }, + "visibility": { + "cliSummaryTable": false, + "statusPage": false, + "telemetry": true + } +} diff --git a/rust/ql/integration-tests/hello-workspace/steps.cargo.expected b/rust/ql/integration-tests/hello-workspace/steps.cargo.expected new file mode 100644 index 000000000000..138bd77b690d --- /dev/null +++ b/rust/ql/integration-tests/hello-workspace/steps.cargo.expected @@ -0,0 +1,14 @@ +| exe/Cargo.toml:0:0:0:0 | LoadManifest(exe/Cargo.toml) | +| exe/src/a_module.rs:0:0:0:0 | Extract(exe/src/a_module.rs) | +| exe/src/a_module.rs:0:0:0:0 | LoadSource(exe/src/a_module.rs) | +| exe/src/a_module.rs:0:0:0:0 | Parse(exe/src/a_module.rs) | +| exe/src/main.rs:0:0:0:0 | Extract(exe/src/main.rs) | +| exe/src/main.rs:0:0:0:0 | LoadSource(exe/src/main.rs) | +| exe/src/main.rs:0:0:0:0 | Parse(exe/src/main.rs) | +| lib/Cargo.toml:0:0:0:0 | LoadManifest(lib/Cargo.toml) | +| lib/src/a_module/mod.rs:0:0:0:0 | Extract(lib/src/a_module/mod.rs) | +| lib/src/a_module/mod.rs:0:0:0:0 | LoadSource(lib/src/a_module/mod.rs) | +| lib/src/a_module/mod.rs:0:0:0:0 | Parse(lib/src/a_module/mod.rs) | +| lib/src/lib.rs:0:0:0:0 | Extract(lib/src/lib.rs) | +| lib/src/lib.rs:0:0:0:0 | LoadSource(lib/src/lib.rs) | +| lib/src/lib.rs:0:0:0:0 | Parse(lib/src/lib.rs) | diff --git a/rust/ql/integration-tests/hello-workspace/steps.ql b/rust/ql/integration-tests/hello-workspace/steps.ql new file mode 100644 index 000000000000..17358a1c1000 --- /dev/null +++ b/rust/ql/integration-tests/hello-workspace/steps.ql @@ -0,0 +1,4 @@ +import codeql.rust.elements.internal.ExtractorStep + +from ExtractorStep step +select step diff --git a/rust/ql/integration-tests/hello-workspace/steps.rust-project.expected b/rust/ql/integration-tests/hello-workspace/steps.rust-project.expected new file mode 100644 index 000000000000..09620925ce96 --- /dev/null +++ b/rust/ql/integration-tests/hello-workspace/steps.rust-project.expected @@ -0,0 +1,13 @@ +| exe/src/a_module.rs:0:0:0:0 | Extract(exe/src/a_module.rs) | +| exe/src/a_module.rs:0:0:0:0 | LoadSource(exe/src/a_module.rs) | +| exe/src/a_module.rs:0:0:0:0 | Parse(exe/src/a_module.rs) | +| exe/src/main.rs:0:0:0:0 | Extract(exe/src/main.rs) | +| exe/src/main.rs:0:0:0:0 | LoadSource(exe/src/main.rs) | +| exe/src/main.rs:0:0:0:0 | Parse(exe/src/main.rs) | +| lib/src/a_module/mod.rs:0:0:0:0 | Extract(lib/src/a_module/mod.rs) | +| lib/src/a_module/mod.rs:0:0:0:0 | LoadSource(lib/src/a_module/mod.rs) | +| lib/src/a_module/mod.rs:0:0:0:0 | Parse(lib/src/a_module/mod.rs) | +| lib/src/lib.rs:0:0:0:0 | Extract(lib/src/lib.rs) | +| lib/src/lib.rs:0:0:0:0 | LoadSource(lib/src/lib.rs) | +| lib/src/lib.rs:0:0:0:0 | Parse(lib/src/lib.rs) | +| rust-project.json:0:0:0:0 | LoadManifest(rust-project.json) | diff --git a/rust/ql/integration-tests/hello-workspace/summary.expected b/rust/ql/integration-tests/hello-workspace/summary.cargo.expected similarity index 94% rename from rust/ql/integration-tests/hello-workspace/summary.expected rename to rust/ql/integration-tests/hello-workspace/summary.cargo.expected index 4e800e60bd0c..ec1abea6252b 100644 --- a/rust/ql/integration-tests/hello-workspace/summary.expected +++ b/rust/ql/integration-tests/hello-workspace/summary.cargo.expected @@ -1,4 +1,4 @@ -| Elements extracted | 72 | +| Elements extracted | 86 | | Elements unextracted | 0 | | Extraction errors | 0 | | Extraction warnings | 0 | diff --git a/rust/ql/integration-tests/hello-workspace/summary.rust-project.expected b/rust/ql/integration-tests/hello-workspace/summary.rust-project.expected new file mode 100644 index 000000000000..5ca38e5a90ce --- /dev/null +++ b/rust/ql/integration-tests/hello-workspace/summary.rust-project.expected @@ -0,0 +1,17 @@ +| Elements extracted | 85 | +| Elements unextracted | 0 | +| Extraction errors | 0 | +| Extraction warnings | 0 | +| Files extracted - total | 4 | +| Files extracted - with errors | 0 | +| Files extracted - without errors | 4 | +| Inconsistencies - AST | 0 | +| Inconsistencies - CFG | 0 | +| Inconsistencies - data flow | 0 | +| Lines of code extracted | 9 | +| Lines of user code extracted | 9 | +| Macro calls - resolved | 2 | +| Macro calls - total | 2 | +| Macro calls - unresolved | 0 | +| Taint sources - active | 0 | +| Taint sources - total | 0 | diff --git a/rust/ql/integration-tests/hello-workspace/test_workspace.py b/rust/ql/integration-tests/hello-workspace/test_workspace.py index 6a7e21ebedbc..fe8dbc69141c 100644 --- a/rust/ql/integration-tests/hello-workspace/test_workspace.py +++ b/rust/ql/integration-tests/hello-workspace/test_workspace.py @@ -1,5 +1,14 @@ -def test_cargo(codeql, rust, cargo, check_source_archive): +import pytest + + +@pytest.mark.ql_test("steps.ql", expected=".cargo.expected") +@pytest.mark.ql_test("summary.qlref", expected=".cargo.expected") +def test_cargo(codeql, rust, cargo, check_source_archive, rust_check_diagnostics): + rust_check_diagnostics.expected_suffix = ".cargo.expected" codeql.database.create() -def test_rust_project(codeql, rust, rust_project, check_source_archive): +@pytest.mark.ql_test("steps.ql", expected=".rust-project.expected") +@pytest.mark.ql_test("summary.qlref", expected=".rust-project.expected") +def test_rust_project(codeql, rust, rust_project, check_source_archive, rust_check_diagnostics): + rust_check_diagnostics.expected_suffix = ".rust-project.expected" codeql.database.create() diff --git a/rust/ql/lib/codeql/files/FileSystem.qll b/rust/ql/lib/codeql/files/FileSystem.qll index 7136dcc3b5a2..b13793b36001 100644 --- a/rust/ql/lib/codeql/files/FileSystem.qll +++ b/rust/ql/lib/codeql/files/FileSystem.qll @@ -6,6 +6,7 @@ private import codeql.rust.elements.SourceFile private import codeql.rust.elements.AstNode private import codeql.rust.elements.Comment private import codeql.rust.Diagnostics +private import codeql.rust.elements.internal.ExtractorStep private module Input implements InputSig { abstract class ContainerBase extends @container { @@ -36,7 +37,9 @@ class Folder = Impl::Folder; /** A file. */ class File extends Container, Impl::File { /** Holds if this file was extracted from ordinary source code. */ - predicate fromSource() { any() } + predicate fromSource() { + exists(ExtractorStep s | s.getAction() = "Extract" and s.getFile() = this) + } /** * Gets the number of lines containing code in this file. This value @@ -58,11 +61,20 @@ class File extends Container, Impl::File { } } +/** + * A source file that was extracted. + * + * TODO: rename `SourceFile` from the generated AST to give that name to this class. + */ +class ExtractedFile extends File { + ExtractedFile() { this.fromSource() } +} + /** * A successfully extracted file, that is, a file that was extracted and * contains no extraction errors or warnings. */ -class SuccessfullyExtractedFile extends File { +class SuccessfullyExtractedFile extends ExtractedFile { SuccessfullyExtractedFile() { not exists(Diagnostic d | d.getLocation().getFile() = this and diff --git a/rust/ql/lib/codeql/rust/elements.qll b/rust/ql/lib/codeql/rust/elements.qll index de9ecf34467c..ca3cccabb5fe 100644 --- a/rust/ql/lib/codeql/rust/elements.qll +++ b/rust/ql/lib/codeql/rust/elements.qll @@ -3,6 +3,7 @@ * This module exports all modules providing `Element` subclasses. */ +import codeql.files.FileSystem import codeql.rust.elements.Abi import codeql.rust.elements.Addressable import codeql.rust.elements.ArgList diff --git a/rust/ql/lib/codeql/rust/elements/internal/ExtractorStep.qll b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStep.qll new file mode 100644 index 000000000000..64a4931ef43c --- /dev/null +++ b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStep.qll @@ -0,0 +1,13 @@ +// generated by codegen, do not edit +/** + * This module provides the class `ExtractorStep`. + */ + +private import ExtractorStepImpl +import codeql.rust.elements.Element +import codeql.files.FileSystem + +/** + * INTERNAL: Do not use. + */ +final class ExtractorStep = Impl::ExtractorStep; diff --git a/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll new file mode 100644 index 000000000000..76050bf99f86 --- /dev/null +++ b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepConstructor.qll @@ -0,0 +1,14 @@ +// generated by codegen, remove this comment if you wish to edit this file +/** + * This module defines the hook used internally to tweak the characteristic predicate of + * `ExtractorStep` synthesized instances. + * INTERNAL: Do not use. + */ + +private import codeql.rust.elements.internal.generated.Raw + +/** + * The characteristic predicate of `ExtractorStep` synthesized instances. + * INTERNAL: Do not use. + */ +predicate constructExtractorStep(Raw::ExtractorStep id) { any() } diff --git a/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepImpl.qll b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepImpl.qll new file mode 100644 index 000000000000..95c677e845e9 --- /dev/null +++ b/rust/ql/lib/codeql/rust/elements/internal/ExtractorStepImpl.qll @@ -0,0 +1,32 @@ +/** + * This module provides a hand-modifiable wrapper around the generated class `ExtractorStep`. + * + * INTERNAL: Do not use. + */ + +private import codeql.rust.elements.internal.generated.ExtractorStep + +/** + * INTERNAL: This module contains the customizable definition of `ExtractorStep` and should not + * be referenced directly. + */ +module Impl { + class ExtractorStep extends Generated::ExtractorStep { + override string toString() { + result = this.getAction() + "(" + this.getFile().getAbsolutePath() + ")" + } + + /** + * Provides location information for this step. + */ + predicate hasLocationInfo( + string filepath, int startline, int startcolumn, int endline, int endcolumn + ) { + this.getFile().getAbsolutePath() = filepath and + startline = 0 and + startcolumn = 0 and + endline = 0 and + endcolumn = 0 + } + } +} diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll new file mode 100644 index 000000000000..c2d7838865be --- /dev/null +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/ExtractorStep.qll @@ -0,0 +1,45 @@ +// generated by codegen, do not edit +/** + * This module provides the generated definition of `ExtractorStep`. + * INTERNAL: Do not import directly. + */ + +private import codeql.rust.elements.internal.generated.Synth +private import codeql.rust.elements.internal.generated.Raw +import codeql.rust.elements.internal.ElementImpl::Impl as ElementImpl +import codeql.files.FileSystem + +/** + * INTERNAL: This module contains the fully generated definition of `ExtractorStep` and should not + * be referenced directly. + */ +module Generated { + /** + * INTERNAL: Do not reference the `Generated::ExtractorStep` class directly. + * Use the subclass `ExtractorStep`, where the following predicates are available. + */ + class ExtractorStep extends Synth::TExtractorStep, ElementImpl::Element { + override string getAPrimaryQlClass() { result = "ExtractorStep" } + + /** + * Gets the action of this extractor step. + */ + string getAction() { + result = Synth::convertExtractorStepToRaw(this).(Raw::ExtractorStep).getAction() + } + + /** + * Gets the file of this extractor step. + */ + File getFile() { + result = Synth::convertExtractorStepToRaw(this).(Raw::ExtractorStep).getFile() + } + + /** + * Gets the duration ms of this extractor step. + */ + int getDurationMs() { + result = Synth::convertExtractorStepToRaw(this).(Raw::ExtractorStep).getDurationMs() + } + } +} diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/ParentChild.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/ParentChild.qll index 84c4c1472788..c036d05cd03e 100644 --- a/rust/ql/lib/codeql/rust/elements/internal/generated/ParentChild.qll +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/ParentChild.qll @@ -5,12 +5,28 @@ import codeql.rust.elements import codeql.rust.elements.internal.ArrayExprInternal +import codeql.rust.elements.internal.ExtractorStep private module Impl { private Element getImmediateChildOfElement(Element e, int index, string partialPredicateCall) { none() } + private Element getImmediateChildOfExtractorStep( + ExtractorStep e, int index, string partialPredicateCall + ) { + exists(int b, int bElement, int n | + b = 0 and + bElement = b + 1 + max(int i | i = -1 or exists(getImmediateChildOfElement(e, i, _)) | i) and + n = bElement and + ( + none() + or + result = getImmediateChildOfElement(e, index - b, partialPredicateCall) + ) + ) + } + private Element getImmediateChildOfLocatable(Locatable e, int index, string partialPredicateCall) { exists(int b, int bElement, int n | b = 0 and @@ -3705,6 +3721,8 @@ private module Impl { // * none() simplifies generation, as we can append `or ...` without a special case for the first item none() or + result = getImmediateChildOfExtractorStep(e, index, partialAccessor) + or result = getImmediateChildOfFormat(e, index, partialAccessor) or result = getImmediateChildOfFormatArgument(e, index, partialAccessor) diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/Raw.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/Raw.qll index 03d9f253a60f..5d4309988a60 100644 --- a/rust/ql/lib/codeql/rust/elements/internal/generated/Raw.qll +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/Raw.qll @@ -3,6 +3,8 @@ * This module holds thin fully generated class definitions around DB entities. */ module Raw { + private import codeql.files.FileSystem + /** * INTERNAL: Do not use. */ @@ -10,6 +12,28 @@ module Raw { string toString() { none() } } + /** + * INTERNAL: Do not use. + */ + class ExtractorStep extends @extractor_step, Element { + override string toString() { result = "ExtractorStep" } + + /** + * Gets the action of this extractor step. + */ + string getAction() { extractor_steps(this, result, _, _) } + + /** + * Gets the file of this extractor step. + */ + File getFile() { extractor_steps(this, _, result, _) } + + /** + * Gets the duration ms of this extractor step. + */ + int getDurationMs() { extractor_steps(this, _, _, result) } + } + /** * INTERNAL: Do not use. */ diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/Synth.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/Synth.qll index c95b7039deb0..7e13da92fd76 100644 --- a/rust/ql/lib/codeql/rust/elements/internal/generated/Synth.qll +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/Synth.qll @@ -142,6 +142,10 @@ module Synth { * INTERNAL: Do not use. */ TExternItemList(Raw::ExternItemList id) { constructExternItemList(id) } or + /** + * INTERNAL: Do not use. + */ + TExtractorStep(Raw::ExtractorStep id) { constructExtractorStep(id) } or /** * INTERNAL: Do not use. */ @@ -953,6 +957,12 @@ module Synth { */ TExternItemList convertExternItemListFromRaw(Raw::Element e) { result = TExternItemList(e) } + /** + * INTERNAL: Do not use. + * Converts a raw element to a synthesized `TExtractorStep`, if possible. + */ + TExtractorStep convertExtractorStepFromRaw(Raw::Element e) { result = TExtractorStep(e) } + /** * INTERNAL: Do not use. * Converts a raw element to a synthesized `TFieldExpr`, if possible. @@ -1842,6 +1852,8 @@ module Synth { * Converts a raw DB element to a synthesized `TElement`, if possible. */ TElement convertElementFromRaw(Raw::Element e) { + result = convertExtractorStepFromRaw(e) + or result = convertLocatableFromRaw(e) or result = convertUnextractedFromRaw(e) @@ -2367,6 +2379,12 @@ module Synth { */ Raw::Element convertExternItemListToRaw(TExternItemList e) { e = TExternItemList(result) } + /** + * INTERNAL: Do not use. + * Converts a synthesized `TExtractorStep` to a raw DB element, if possible. + */ + Raw::Element convertExtractorStepToRaw(TExtractorStep e) { e = TExtractorStep(result) } + /** * INTERNAL: Do not use. * Converts a synthesized `TFieldExpr` to a raw DB element, if possible. @@ -3254,6 +3272,8 @@ module Synth { * Converts a synthesized `TElement` to a raw DB element, if possible. */ Raw::Element convertElementToRaw(TElement e) { + result = convertExtractorStepToRaw(e) + or result = convertLocatableToRaw(e) or result = convertUnextractedToRaw(e) diff --git a/rust/ql/lib/codeql/rust/elements/internal/generated/SynthConstructors.qll b/rust/ql/lib/codeql/rust/elements/internal/generated/SynthConstructors.qll index 0a9ccee4d562..3b2a58083b14 100644 --- a/rust/ql/lib/codeql/rust/elements/internal/generated/SynthConstructors.qll +++ b/rust/ql/lib/codeql/rust/elements/internal/generated/SynthConstructors.qll @@ -35,6 +35,7 @@ import codeql.rust.elements.internal.ExprStmtConstructor import codeql.rust.elements.internal.ExternBlockConstructor import codeql.rust.elements.internal.ExternCrateConstructor import codeql.rust.elements.internal.ExternItemListConstructor +import codeql.rust.elements.internal.ExtractorStepConstructor import codeql.rust.elements.internal.FieldExprConstructor import codeql.rust.elements.internal.FnPtrTypeReprConstructor import codeql.rust.elements.internal.ForExprConstructor diff --git a/rust/ql/lib/rust.dbscheme b/rust/ql/lib/rust.dbscheme index 32c5387cdf18..c0f9cf44822d 100644 --- a/rust/ql/lib/rust.dbscheme +++ b/rust/ql/lib/rust.dbscheme @@ -120,10 +120,18 @@ locatable_locations( // from schema @element = - @locatable + @extractor_step +| @locatable | @unextracted ; +extractor_steps( + unique int id: @extractor_step, + string action: string ref, + int file: @file ref, + int duration_ms: int ref +); + @locatable = @ast_node ; diff --git a/rust/ql/src/queries/diagnostics/ExtractedFiles.ql b/rust/ql/src/queries/diagnostics/ExtractedFiles.ql index ce5aa699e671..112fb2949dc2 100644 --- a/rust/ql/src/queries/diagnostics/ExtractedFiles.ql +++ b/rust/ql/src/queries/diagnostics/ExtractedFiles.ql @@ -8,6 +8,6 @@ import rust -from File f +from ExtractedFile f where exists(f.getRelativePath()) select f, "File successfully extracted." diff --git a/rust/ql/src/queries/diagnostics/ExtractionErrors.ql b/rust/ql/src/queries/diagnostics/ExtractionErrors.ql index 68be66f8ca9b..bd5ec3426fb6 100644 --- a/rust/ql/src/queries/diagnostics/ExtractionErrors.ql +++ b/rust/ql/src/queries/diagnostics/ExtractionErrors.ql @@ -11,7 +11,7 @@ import codeql.files.FileSystem /** Gets the SARIF severity to associate with an error. */ int getSeverity() { result = 2 } -from ExtractionError error, File f +from ExtractionError error, ExtractedFile f where f = error.getLocation().getFile() and exists(f.getRelativePath()) diff --git a/rust/ql/src/queries/summary/SummaryStats.ql b/rust/ql/src/queries/summary/SummaryStats.ql index 09ee83fc5e64..ffe7cbf1a8fa 100644 --- a/rust/ql/src/queries/summary/SummaryStats.ql +++ b/rust/ql/src/queries/summary/SummaryStats.ql @@ -21,10 +21,13 @@ where or key = "Extraction warnings" and value = count(ExtractionWarning w) or - key = "Files extracted - total" and value = count(File f | exists(f.getRelativePath())) + key = "Files extracted - total" and value = count(ExtractedFile f | exists(f.getRelativePath())) or key = "Files extracted - with errors" and - value = count(File f | exists(f.getRelativePath()) and not f instanceof SuccessfullyExtractedFile) + value = + count(ExtractedFile f | + exists(f.getRelativePath()) and not f instanceof SuccessfullyExtractedFile + ) or key = "Files extracted - without errors" and value = count(SuccessfullyExtractedFile f | exists(f.getRelativePath())) diff --git a/rust/ql/test/extractor-tests/File/File.expected b/rust/ql/test/extractor-tests/File/File.expected index 924ed370b35b..77fee3c3c428 100644 --- a/rust/ql/test/extractor-tests/File/File.expected +++ b/rust/ql/test/extractor-tests/File/File.expected @@ -1,3 +1,4 @@ -| a_file.rs:0:0:0:0 | a_file.rs | -| another_file.rs:0:0:0:0 | another_file.rs | -| lib.rs:0:0:0:0 | lib.rs | +| Cargo.toml:0:0:0:0 | Cargo.toml | fromSource: no | +| a_file.rs:0:0:0:0 | a_file.rs | fromSource: yes | +| another_file.rs:0:0:0:0 | another_file.rs | fromSource: yes | +| lib.rs:0:0:0:0 | lib.rs | fromSource: yes | diff --git a/rust/ql/test/extractor-tests/File/File.ql b/rust/ql/test/extractor-tests/File/File.ql index fcb2b274e781..316099193d15 100644 --- a/rust/ql/test/extractor-tests/File/File.ql +++ b/rust/ql/test/extractor-tests/File/File.ql @@ -1,5 +1,7 @@ import rust -from File f -where exists(f.getRelativePath()) -select f +from File f, string fromSource +where + exists(f.getRelativePath()) and + if f.fromSource() then fromSource = "fromSource: yes" else fromSource = "fromSource: no" +select f, fromSource diff --git a/rust/ql/test/query-tests/diagnostics/ExtractedFiles.expected b/rust/ql/test/query-tests/diagnostics/ExtractedFiles.expected index bc8dc8cccf15..c5ebb7072304 100644 --- a/rust/ql/test/query-tests/diagnostics/ExtractedFiles.expected +++ b/rust/ql/test/query-tests/diagnostics/ExtractedFiles.expected @@ -5,4 +5,3 @@ | main.rs:0:0:0:0 | main.rs | File successfully extracted. | | my_macro.rs:0:0:0:0 | my_macro.rs | File successfully extracted. | | my_struct.rs:0:0:0:0 | my_struct.rs | File successfully extracted. | -| options.yml:0:0:0:0 | options.yml | File successfully extracted. | diff --git a/rust/ql/test/query-tests/diagnostics/LinesOfUserCodeInFiles.expected b/rust/ql/test/query-tests/diagnostics/LinesOfUserCodeInFiles.expected index 0a58a05feed2..f53c931e56ee 100644 --- a/rust/ql/test/query-tests/diagnostics/LinesOfUserCodeInFiles.expected +++ b/rust/ql/test/query-tests/diagnostics/LinesOfUserCodeInFiles.expected @@ -5,4 +5,5 @@ | lib.rs:0:0:0:0 | lib.rs | 5 | | does_not_compile.rs:0:0:0:0 | does_not_compile.rs | 3 | | error.rs:0:0:0:0 | error.rs | 3 | +| Cargo.toml:0:0:0:0 | Cargo.toml | 0 | | options.yml:0:0:0:0 | options.yml | 0 | diff --git a/rust/ql/test/query-tests/diagnostics/SummaryStats.expected b/rust/ql/test/query-tests/diagnostics/SummaryStats.expected index 74e1e461c6fe..9372843039cb 100644 --- a/rust/ql/test/query-tests/diagnostics/SummaryStats.expected +++ b/rust/ql/test/query-tests/diagnostics/SummaryStats.expected @@ -1,10 +1,10 @@ -| Elements extracted | 382 | +| Elements extracted | 404 | | Elements unextracted | 0 | | Extraction errors | 0 | | Extraction warnings | 7 | -| Files extracted - total | 8 | +| Files extracted - total | 7 | | Files extracted - with errors | 3 | -| Files extracted - without errors | 5 | +| Files extracted - without errors | 4 | | Inconsistencies - AST | 0 | | Inconsistencies - CFG | 0 | | Inconsistencies - data flow | 0 | diff --git a/rust/schema/prelude.py b/rust/schema/prelude.py index ffd65959b5ab..5b050872609b 100644 --- a/rust/schema/prelude.py +++ b/rust/schema/prelude.py @@ -3,6 +3,7 @@ include("../shared/tree-sitter-extractor/src/generator/prefix.dbscheme") include("prefix.dbscheme") +File = imported("File", "codeql.files.FileSystem") @qltest.skip class Element: @@ -93,3 +94,11 @@ class Resolvable(AstNode): """ resolved_path: optional[string] | rust.detach | ql.internal resolved_crate_origin: optional[string] | rust.detach | ql.internal + + +@qltest.skip +@ql.internal +class ExtractorStep(Element): + action: string + file: File + duration_ms: int diff --git a/swift/ql/.generated.list b/swift/ql/.generated.list index da39f2f2fe77..601205793210 100644 --- a/swift/ql/.generated.list +++ b/swift/ql/.generated.list @@ -712,7 +712,7 @@ lib/codeql/swift/generated/OtherAvailabilitySpec.qll d9feaa2a71acff3184ca389045b lib/codeql/swift/generated/ParentChild.qll d1814f2bad4c2ba9242ce49fe6fb8564ac99fc1fd3a7d12aa55e5c6dd7bb529b 1a2075b731d07a5e3c6a69d001796c8de925069d839671a294c9cba6c3db724a lib/codeql/swift/generated/PlatformVersionAvailabilitySpec.qll dc17b49a90a18a8f7607adf2433bc8f0c194fa3e803aa3822f809d4d4fbd6793 be48ea9f8ae17354c8508aaed24337a9e57ce01f288fece3dcecd99776cabcec lib/codeql/swift/generated/PureSynthConstructors.qll bc31a6c4d142fa3fbdcae69d5ba6f1cec00eb9ad92b46c8d7b91ebfa7ef6c1f4 bc31a6c4d142fa3fbdcae69d5ba6f1cec00eb9ad92b46c8d7b91ebfa7ef6c1f4 -lib/codeql/swift/generated/Raw.qll 118b43fedd4265b5aa15c33ef01a2f5a5db6e5597f95bef1078a01c3ff8da983 075aec2c8b232f0361ebf63f07ae9b66163f3975e6023583fb0fa2e40b979a33 +lib/codeql/swift/generated/Raw.qll a47950495630c10c00afee734d779e5e3e7f8e3e65ae3f04b9a1254e3e73921e 075aec2c8b232f0361ebf63f07ae9b66163f3975e6023583fb0fa2e40b979a33 lib/codeql/swift/generated/Synth.qll 31e318c6e156848c85a2a2664695b48b5e93c57c9bb22fa29d027069907b3ab0 8655ffcf772f55284b93f1d7f8e1b3d497a9744d5f2e0c17bc322c1fdf8bdba8 lib/codeql/swift/generated/SynthConstructors.qll 3e53c7853096020219c01dae85681fe80b34938d198a0ff359a209dda41c5ed7 3e53c7853096020219c01dae85681fe80b34938d198a0ff359a209dda41c5ed7 lib/codeql/swift/generated/UnknownFile.qll 247ddf2ebb49ce5ed4bf7bf91a969ddff37de6c78d43d8affccaf7eb586e06f2 452b29f0465ef45e978ef8b647b75e5a2a1e53f2a568fc003bc8f52f73b3fa4d