Go: add bazel build

.pre-commit-config.yaml

@@ -29,6 +29,13 @@ repos:
         entry: bazel run //:buildifier
         pass_filenames: false
 
+      - id: go-gen
+        name: Check checked in generated files in go
+        files: ^go/.*
+        language: system
+        entry: bazel run //go:gen
+        pass_filenames: false
+
       - id: codeql-format
         name: Fix QL file formatting
         files: \.qll?$

MODULE.bazel

@@ -13,14 +13,16 @@ local_path_override(
 
 # see https://registry.bazel.build/ for a list of available packages
 
-bazel_dep(name = "platforms", version = "0.0.8")
+bazel_dep(name = "platforms", version = "0.0.9")
+bazel_dep(name = "rules_go", version = "0.47.0")
 bazel_dep(name = "rules_pkg", version = "0.10.1")
 bazel_dep(name = "rules_nodejs", version = "6.0.3")
 bazel_dep(name = "rules_python", version = "0.31.0")
 bazel_dep(name = "bazel_skylib", version = "1.5.0")
 bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl")
 bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
 bazel_dep(name = "fmt", version = "10.0.0")
+bazel_dep(name = "gazelle", version = "0.36.0")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
@@ -52,6 +54,9 @@ node.toolchain(
 )
 use_repo(node, "nodejs", "nodejs_toolchains")
 
+go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
+go_sdk.download(version = "1.22.2")  # default
+
 register_toolchains(
     "@nodejs_toolchains//:all",
 )

go/BUILD.bazel

@@ -0,0 +1,119 @@
+load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
+load("@gazelle//:def.bzl", "gazelle")
+load("@rules_pkg//pkg:install.bzl", "pkg_install")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files")
+load("@rules_pkg//pkg:zip.bzl", "pkg_zip")
+load("//:defs.bzl", "codeql_platform")
+
+gazelle(
+    name = "gazelle",
+    extra_args = ["go/extractor"],
+)
+
+_gen_binaries = [
+    "@rules_go//go",
+    ":gazelle",
+    "//go/extractor/cli/go-gen-dbscheme",
+]
+
+py_binary(
+    name = "gen",
+    srcs = ["gen.py"],
+    args = ["$(rlocationpath %s)" % bin for bin in _gen_binaries],
+    data = _gen_binaries,
+    deps = ["@rules_python//python/runfiles"],
+)
+
+# this is an internal copy of the dbscheme to be used by extractor-pack
+# this allows the extractor-pack target to be independent and up-to-date with respect to
+# having run //go:gen to update the checked in files
+genrule(
+    name = "dbscheme",
+    outs = ["go.dbscheme"],
+    cmd = "$(execpath //go/extractor/cli/go-gen-dbscheme) $@",
+    tools = ["//go/extractor/cli/go-gen-dbscheme"],
+)
+
+pkg_files(
+    name = "resources",
+    srcs = [
+        "LICENSE",
+        "codeql-extractor.yml",
+        "ql/lib/go.dbscheme.stats",
+        ":dbscheme",
+    ],
+)
+
+pkg_filegroup(
+    name = "extractor-pack-generic",
+    srcs = [
+        ":resources",
+        "//go/codeql-tools",
+        "//go/downgrades",
+        "//go/extractor:tokenizer",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+pkg_files(
+    name = "extractor-pack-arch",
+    srcs = [
+        "//go/extractor/cli/go-autobuilder",
+        "//go/extractor/cli/go-bootstrap",
+        "//go/extractor/cli/go-build-runner",
+        "//go/extractor/cli/go-extractor",
+        "//go/extractor/cli/go-gen-dbscheme",
+        "//go/extractor/cli/go-tokenizer",
+    ],
+    attributes = pkg_attributes(mode = "0755"),
+    prefix = "tools/" + codeql_platform,
+    visibility = ["//visibility:public"],
+)
+
+pkg_filegroup(
+    name = "extractor-pack",
+    srcs = [
+        ":extractor-pack-arch",
+        ":extractor-pack-generic",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+pkg_install(
+    name = "_extractor-pack-installer",
+    srcs = [":extractor-pack"],
+)
+
+# rules_pkg installer is currently broken on Windows
+# see https://github.com/bazelbuild/rules_pkg/issues/387
+# for now, work around it using an archive
+pkg_zip(
+    name = "_extractor-pack-zip",
+    srcs = [":extractor-pack"],
+)
+
+alias(
+    name = "_create-extractor-pack-arg",
+    actual = select({
+        "@platforms//os:windows": ":_extractor-pack-zip",
+        "//conditions:default": ":_extractor-pack-installer",
+    }),
+)
+
+py_binary(
+    name = "create-extractor-pack",
+    srcs = ["create_extractor_pack.py"],
+    args = ["$(rlocationpath :_create-extractor-pack-arg)"],
+    data = [":_create-extractor-pack-arg"],
+    main = "create_extractor_pack.py",
+    deps = ["@rules_python//python/runfiles"],
+)
+
+native_binary(
+    name = "gen-dbscheme",
+    src = "//go/extractor/cli/go-gen-dbscheme",
+    out = "go-gen-dbscheme",
+    args = [
+        "$$BUILD_WORKSPACE_DIRECTORY/go/ql/lib/go.dbscheme",
+    ],
+)

go/codeql-tools/BUILD.bazel

@@ -0,0 +1,28 @@
+load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files")
+
+pkg_files(
+    name = "sh-files",
+    srcs = glob(["*.sh"]),
+    attributes = pkg_attributes(mode = "0755"),
+)
+
+pkg_files(
+    name = "non-sh-files",
+    srcs = glob(
+        ["*"],
+        exclude = [
+            "*.sh",
+            "BUILD.bazel",
+        ],
+    ),
+)
+
+pkg_filegroup(
+    name = "codeql-tools",
+    srcs = [
+        ":non-sh-files",
+        ":sh-files",
+    ],
+    prefix = "tools",
+    visibility = ["//go:__pkg__"],
+)

go/create_extractor_pack.py

@@ -0,0 +1,26 @@
+import os
+import pathlib
+import shutil
+import sys
+import subprocess
+import zipfile
+from python.runfiles import runfiles
+
+try:
+    workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY'])
+except KeyError:
+    print("this should be run with bazel run", file=sys.stderr)
+    sys.exit(1)
+
+dest_dir = workspace_dir / 'go' / 'build' / 'codeql-extractor-go'
+installer_or_zip = pathlib.Path(runfiles.Create().Rlocation(sys.argv[1]))
+
+shutil.rmtree(dest_dir, ignore_errors=True)
+
+if installer_or_zip.suffix == '.zip':
+    dest_dir.mkdir()
+    with zipfile.ZipFile(installer_or_zip) as pack:
+        pack.extractall(dest_dir)
+else:
+    os.environ['DESTDIR'] = str(dest_dir)
+    subprocess.check_call([installer_or_zip])

go/downgrades/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix")
+
+pkg_files(
+    name = "downgrades",
+    srcs = glob(
+        ["**"],
+        exclude = ["BUILD.bazel"],
+    ),
+    prefix = "downgrades",
+    strip_prefix = strip_prefix.from_pkg(),
+    visibility = ["//go:__pkg__"],
+)

go/extractor/.gitattributes

@@ -0,0 +1 @@
+/*/**/BUILD.bazel linguist-generated=true

go/extractor/BUILD.bazel

@@ -0,0 +1,57 @@
+load("@rules_go//go:def.bzl", "go_library")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
+
+# gazelle:prefix github.com/github/codeql-go/extractor
+# gazelle:map_kind go_binary codeql_go_binary //go:rules.bzl
+
+# the immediately following `extractor` target is kept up to date by `bazel run //go:gen`, do not edit directly
+go_library(
+    name = "extractor",
+    srcs = [
+        "extractor.go",
+        "gomodextractor.go",
+        "semaphore.go",
+    ],
+    importpath = "github.com/github/codeql-go/extractor",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//go/extractor/dbscheme",
+        "//go/extractor/diagnostics",
+        "//go/extractor/srcarchive",
+        "//go/extractor/trap",
+        "//go/extractor/util",
+        "//go/extractor/vendor/golang.org/x/mod/modfile",
+        "//go/extractor/vendor/golang.org/x/tools/go/packages",
+    ],
+)
+
+# notice that these other targets are not generated
+java_library(
+    name = "tokenizer-deps",
+    srcs = [
+        "net/sourceforge/pmd/cpd/AbstractLanguage.java",
+        "net/sourceforge/pmd/cpd/SourceCode.java",
+        "net/sourceforge/pmd/cpd/TokenEntry.java",
+        "net/sourceforge/pmd/cpd/Tokenizer.java",
+    ],
+)
+
+java_library(
+    name = "tokenizer-jar",
+    srcs = [
+        "net/sourceforge/pmd/cpd/GoLanguage.java",
+        "opencsv/CSVParser.java",
+        "opencsv/CSVReader.java",
+    ],
+    deps = [":tokenizer-deps"],
+)
+
+pkg_files(
+    name = "tokenizer",
+    srcs = [":tokenizer-jar"],
+    prefix = "tools",
+    renames = {
+        ":tokenizer-jar": "tokenizer.jar",
+    },
+    visibility = ["//go:__pkg__"],
+)

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -487,7 +487,9 @@ func extract(workspace project.GoWorkspace) bool {
 
 // Build the project and run the extractor.
 func installDependenciesAndBuild() {
-	log.Printf("Autobuilder was built with %s, environment has %s\n", runtime.Version(), toolchain.GetEnvGoVersion())
+	// do not print experiments the autobuilder was built with if any, only the version
+	version := strings.SplitN(runtime.Version(), " ", 2)[0]
+	log.Printf("Autobuilder was built with %s, environment has %s\n", version, toolchain.GetEnvGoVersion())
 
 	srcdir := getSourceDir()