Kotlin: build extractor with bazel

.bazelrc

@@ -14,4 +14,7 @@ build:linux --cxxopt=-std=c++20
 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
 
+common --registry=file:///%workspace%/misc/bazel/registry
+common --registry=https://bcr.bazel.build
+
 try-import %workspace%/local.bazelrc

.bazelrc.internal

@@ -0,0 +1,4 @@
+# this file should contain bazel settings required to build things from `semmle-code`
+
+common --registry=file:///%workspace%/ql/misc/bazel/registry
+common --registry=https://bcr.bazel.build

.gitattributes

@@ -50,31 +50,32 @@
 *.dll -text
 *.pdb -text
 
-java/ql/test/stubs/**/*.java linguist-generated=true
-java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+/java/ql/test/stubs/**/*.java linguist-generated=true
+/java/ql/test/experimental/stubs/**/*.java linguist-generated=true
+/java/kotlin-extractor/deps/*.jar filter=lfs diff=lfs merge=lfs -text
 
 # Force git not to modify line endings for go or html files under the go/ql directory
-go/ql/**/*.go -text
-go/ql/**/*.html -text
+/go/ql/**/*.go -text
+/go/ql/**/*.html -text
 # Force git not to modify line endings for go dbschemes
-go/*.dbscheme -text
+/go/*.dbscheme -text
 # Preserve unusual line ending from codeql-go merge
-go/extractor/opencsv/CSVReader.java -text
+/go/extractor/opencsv/CSVReader.java -text
 
 # For some languages, upgrade script testing references really old dbscheme
 # files from legacy upgrades that have CRLF line endings. Since upgrade
 # resolution relies on object hashes, we must suppress line ending conversion
 # for those testing dbscheme files.
-*/ql/lib/upgrades/initial/*.dbscheme -text
+/*/ql/lib/upgrades/initial/*.dbscheme -text
 
 # Generated test files - these are synced from the standard JavaScript libraries using
 # `javascript/ql/experimental/adaptivethreatmodeling/test/update_endpoint_test_files.py`.
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
-javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge
+/javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.js linguist-generated=true -merge
+/javascript/ql/experimental/adaptivethreatmodeling/test/endpoint_large_scale/autogenerated/**/*.ts linguist-generated=true -merge
 
 # Auto-generated modeling for Python
-python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
+/python/ql/lib/semmle/python/frameworks/data/internal/subclass-capture/*.yml linguist-generated=true
 
 # auto-generated bazel lock file
-ruby/extractor/cargo-bazel-lock.json linguist-generated=true
-ruby/extractor/cargo-bazel-lock.json -merge
+/ruby/extractor/cargo-bazel-lock.json linguist-generated=true
+/ruby/extractor/cargo-bazel-lock.json -merge

.github/workflows/kotlin-build.yml

@@ -0,0 +1,28 @@
+name: "Kotlin Build"
+
+on:
+  pull_request:
+    paths:
+      - "java/kotlin-extractor/**"
+      - "misc/bazel/**"
+      - "misc/codegen/**"
+      - "*.bazel*"
+      - .github/workflows/kotlin-build.yml
+    branches:
+      - main
+      - rc/*
+      - codeql-cli-*
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          bazel query //java/kotlin-extractor/...
+          # only build the default version as a sanity check that we can build from `codeql`
+          # the full official build will be checked by QLucie
+          bazel build //java/kotlin-extractor

.lfsconfig

@@ -0,0 +1,5 @@
+[lfs]
+# codeql is publicly forked by many users, and we don't want any LFS file polluting their working
+# copies. We therefore exclude everything by default.
+# For files required by bazel builds, use rules in `misc/bazel/lfs.bzl` to download them on demand.
+fetchinclude = /nothing

MODULE.bazel

@@ -21,6 +21,7 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0")
 bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl")
 bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
 bazel_dep(name = "fmt", version = "10.0.0")
+bazel_dep(name = "rules_kotlin", version = "1.9.4-codeql.1")
 
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 pip.parse(
@@ -50,6 +51,51 @@ node.toolchain(
 )
 use_repo(node, "nodejs", "nodejs_toolchains")
 
+kotlin_extractor_deps = use_extension("//java/kotlin-extractor:deps.bzl", "kotlin_extractor_deps")
+
+# following list can be kept in sync by running `bazel mod tidy` in `codeql`
+use_repo(
+    kotlin_extractor_deps,
+    "codeql_kotlin_defaults",
+    "codeql_kotlin_embeddable",
+    "kotlin-compiler-1.5.0",
+    "kotlin-compiler-1.5.10",
+    "kotlin-compiler-1.5.20",
+    "kotlin-compiler-1.5.30",
+    "kotlin-compiler-1.6.0",
+    "kotlin-compiler-1.6.20",
+    "kotlin-compiler-1.7.0",
+    "kotlin-compiler-1.7.20",
+    "kotlin-compiler-1.8.0",
+    "kotlin-compiler-1.9.0-Beta",
+    "kotlin-compiler-1.9.20-Beta",
+    "kotlin-compiler-2.0.0-RC1",
+    "kotlin-compiler-embeddable-1.5.0",
+    "kotlin-compiler-embeddable-1.5.10",
+    "kotlin-compiler-embeddable-1.5.20",
+    "kotlin-compiler-embeddable-1.5.30",
+    "kotlin-compiler-embeddable-1.6.0",
+    "kotlin-compiler-embeddable-1.6.20",
+    "kotlin-compiler-embeddable-1.7.0",
+    "kotlin-compiler-embeddable-1.7.20",
+    "kotlin-compiler-embeddable-1.8.0",
+    "kotlin-compiler-embeddable-1.9.0-Beta",
+    "kotlin-compiler-embeddable-1.9.20-Beta",
+    "kotlin-compiler-embeddable-2.0.0-RC1",
+    "kotlin-stdlib-1.5.0",
+    "kotlin-stdlib-1.5.10",
+    "kotlin-stdlib-1.5.20",
+    "kotlin-stdlib-1.5.30",
+    "kotlin-stdlib-1.6.0",
+    "kotlin-stdlib-1.6.20",
+    "kotlin-stdlib-1.7.0",
+    "kotlin-stdlib-1.7.20",
+    "kotlin-stdlib-1.8.0",
+    "kotlin-stdlib-1.9.0-Beta",
+    "kotlin-stdlib-1.9.20-Beta",
+    "kotlin-stdlib-2.0.0-RC1",
+)
+
 register_toolchains(
     "@nodejs_toolchains//:all",
 )

java/downgrades/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@rules_pkg//:mappings.bzl", "pkg_files", "strip_prefix")
+
+pkg_files(
+    name = "downgrades",
+    srcs = glob(
+        ["**"],
+        exclude = ["BUILD.bazel"],
+    ),
+    prefix = "downgrades",
+    strip_prefix = strip_prefix.from_pkg(),
+    visibility = ["//visibility:public"],
+)

java/kotlin-extractor/BUILD.bazel

@@ -0,0 +1,181 @@
+"""
+# Usage overview
+Building the extractor can be done via
+```
+bazel build //java/kotlin-extractor:codeql-extractor-kotlin-<variant>-<version>
+```
+where `<variant>` is either `standalone` or `embeddable`, and `<version>` is one of the supported versions.
+
+```
+bazel build //java/kotlin-extractor
+```
+will build a default variant:
+* standalone, unless `CODEQL_KOTLIN_SINGLE_VERSION_EMBEDDABLE` is set to true, in which case it will go for embeddable
+* the version will be taken as the last supported version less than the version of the currently installed `kotlinc`
+    * if `CODEQL_KOTLIN_SINGLE_VERSION` is set, that will be used instead
+    * if `kotlinc` is not installed, `1.9.20-Beta` will be used
+
+If `kotlinc` is updated, bazel won't be aware of it and will therefore keep the same default version. Possible workarounds for that:
+* `bazel clean`
+* `bazel fetch --force @codeql_kotlin_defaults\\:all`
+* `CODEQL_KOTLIN_SINGLE_VERSION= bazel build //java/kotlin-extractor`
+"""
+
+# This file is used in the `@codeql_kotlin_embeddable` external repo, which means we need to
+# reference explicitly @codeql
+load(
+    "@codeql//java/kotlin-extractor:versions.bzl",
+    "VERSIONS",
+    "get_compatilibity_sources",
+    "get_language_version",
+    "version_less",
+)
+load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library")
+load("@rules_kotlin//kotlin:core.bzl", "kt_javac_options", "kt_kotlinc_options")
+
+package(default_visibility = ["//java/kotlin-extractor:__subpackages__"])
+
+_for_embeddable = repo_name().endswith("codeql_kotlin_embeddable")
+
+_common_extractor_name_prefix = "codeql-extractor-kotlin"
+
+_extractor_name_prefix = "%s-%s" % (
+    _common_extractor_name_prefix,
+    "embeddable" if _for_embeddable else "standalone",
+)
+
+py_binary(
+    name = "generate_dbscheme",
+    srcs = ["generate_dbscheme.py"],
+)
+
+_resources = [
+    (
+        r,
+        r[len("src/main/resources/"):],
+    )
+    for r in glob(["src/main/resources/**"])
+]
+
+kt_javac_options(
+    name = "javac-options",
+    release = "8",
+)
+
+[
+    (
+        kt_kotlinc_options(
+            name = "kotlinc-options-%s" % v,
+            include_stdlibs = "none",
+            jvm_target = "1.8",
+            language_version = get_language_version(v),
+            warn = "error",
+            x_optin = [
+                "kotlin.RequiresOptIn",
+                "org.jetbrains.kotlin.ir.symbols.%s" %
+                ("IrSymbolInternals" if version_less(v, "2.0.0") else "UnsafeDuringIrConstructionAPI"),
+            ],
+            x_suppress_version_warnings = True,
+        ),
+        # * extractor.name is different for each version, so we need to put it in different output dirs
+        # * in order to put it in `resources`, we need to define `resource_strip_prefix` to strip this version
+        # * `resource_strip_prefix` is unique per jar, so we must also put other resources under the same version prefix
+        genrule(
+            name = "resources-%s" % v,
+            srcs = [src for src, _ in _resources],
+            outs = [
+                "%s/com/github/codeql/extractor.name" % v,
+            ] + [
+                "%s/%s" % (v, tgt)
+                for _, tgt in _resources
+            ],
+            cmd = "\n".join([
+                "echo %s-%s > $(RULEDIR)/%s/com/github/codeql/extractor.name" % (_extractor_name_prefix, v, v),
+            ] + [
+                "cp $(execpath %s) $(RULEDIR)/%s/%s" % (src, v, tgt)
+                for src, tgt in _resources
+            ]),
+        ),
+        kt_jvm_library(
+            name = "%s-%s" % (_extractor_name_prefix, v),
+            srcs =
+                ["@codeql//java/kotlin-extractor:generated-dbscheme"] +
+                glob(
+                    [
+                        "src/**/*.kt",
+                        "src/**/*.java",
+                    ],
+                    exclude = ["src/main/kotlin/utils/versions/**"],
+                ) + get_compatilibity_sources(v, "src/main/kotlin/utils/versions"),
+            javac_opts = ":javac-options",
+            kotlinc_opts = ":kotlinc-options-%s" % v,
+            module_name = "codeql-kotlin-extractor",
+            # resource_strip_prefix is very nit-picky: the following makes it work from
+            # `codeql`, `@codeql_kotlin_embeddable` and `semmle-code`
+            resource_strip_prefix = (
+                ("../%s/" % repo_name() if repo_name() else "") +
+                ("%s/" % package_name() if package_name() else "") +
+                v
+            ),
+            resources = [
+                ":resources-%s" % v,
+            ],
+            visibility = ["//visibility:public"],
+            deps = [
+                "@kotlin-compiler%s-%s" % (
+                    "-embeddable" if _for_embeddable else "",
+                    v,
+                ),
+                "@kotlin-stdlib-%s" % v,
+            ],
+        ),
+        # if in main repository, alias the embeddable versions from the modified @codeql_kotlin_embeddable repo
+        alias(
+            name = "%s-embeddable-%s" % (_common_extractor_name_prefix, v),
+            actual = "@codeql_kotlin_embeddable//:%s-embeddable-%s" % (_common_extractor_name_prefix, v),
+            visibility = ["//visibility:public"],
+        ) if not _for_embeddable else None,
+    )
+    for v in VERSIONS
+]
+
+(
+    genrule(
+        name = "generated-dbscheme",
+        srcs = ["@codeql//java:dbscheme"],
+        outs = ["KotlinExtractorDbScheme.kt"],
+        cmd = "$(execpath :generate_dbscheme) $< $@",
+        tools = [":generate_dbscheme"],
+        visibility = ["@codeql_kotlin_embeddable//:__pkg__"],
+    ),
+    [
+        alias(
+            name = n,
+            actual = "//java/kotlin-extractor/defaults:%s" % n,
+            visibility = ["//visibility:public"],
+        )
+        for n in (
+            "%s-standalone" % _common_extractor_name_prefix,
+            "%s-embeddable" % _common_extractor_name_prefix,
+            _common_extractor_name_prefix,
+        )
+    ],
+    alias(
+        name = "kotlin-extractor",
+        actual = _common_extractor_name_prefix,
+        visibility = ["//visibility:public"],
+    ),
+    filegroup(
+        name = "many",
+        srcs = ["%s-%s-%s" % (
+            _common_extractor_name_prefix,
+            variant,
+            version,
+        ) for variant in ("standalone", "embeddable") for version in VERSIONS],
+        visibility = ["//visibility:public"],
+    ),
+    sh_binary(
+        name = "print-default-version",
+        srcs = ["//java/kotlin-extractor/defaults:default-version-printer"],
+    ),
+) if not _for_embeddable else None

java/kotlin-extractor/current_kotlin_version.py

@@ -0,0 +1,13 @@
+import subprocess
+import re
+import shutil
+
+kotlinc = shutil.which('kotlinc')
+if kotlinc is None:
+    raise Exception("kotlinc not found")
+output = subprocess.run([kotlinc, "-version"], text=True, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE,
+                        check=True).stderr
+m = re.match(r'.* kotlinc-jvm ([0-9]+\.[0-9]+\.[0-9]+(-[a-zA-Z][a-zA-Z0-9]*)?) .*', output)
+if m is None:
+    raise Exception(f'Cannot detect version of kotlinc (got {output})')
+print(m[1])

java/kotlin-extractor/defaults/BUILD.bazel

@@ -0,0 +1,36 @@
+load("@codeql_kotlin_defaults//:defaults.bzl", "kotlin_extractor_defaults")
+
+package(default_visibility = ["//java/kotlin-extractor:__pkg__"])
+
+_common_extractor_name_prefix = "codeql-extractor-kotlin"
+
+alias(
+    name = "%s-standalone" % _common_extractor_name_prefix,
+    actual = "//java/kotlin-extractor:%s-standalone-%s" % (
+        _common_extractor_name_prefix,
+        kotlin_extractor_defaults.extractor_version,
+    ),
+)
+
+alias(
+    name = "%s-embeddable" % _common_extractor_name_prefix,
+    actual = "//java/kotlin-extractor:%s-embeddable-%s" % (
+        _common_extractor_name_prefix,
+        kotlin_extractor_defaults.extractor_version,
+    ),
+)
+
+alias(
+    name = _common_extractor_name_prefix,
+    actual = "//java/kotlin-extractor:%s-%s-%s" % (
+        _common_extractor_name_prefix,
+        kotlin_extractor_defaults.variant,
+        kotlin_extractor_defaults.extractor_version,
+    ),
+)
+
+genrule(
+    name = "default-version-printer",
+    outs = ["print-default-version.sh"],
+    cmd = "echo echo %s > $@" % kotlin_extractor_defaults.version,
+)