Merge pull request #16317 from github/redsun82/go

Go: add bazel build

.bazelrc

@@ -14,6 +14,10 @@ build:linux --cxxopt=-std=c++20
 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
 
+# this requires developer mode, but is required to have pack installer functioning
+startup --windows_enable_symlinks
+common --enable_runfiles
+
 common --registry=file:///%workspace%/misc/bazel/registry
 common --registry=https://bcr.bazel.build
 

.pre-commit-config.yaml

@@ -29,6 +29,13 @@ repos:
         entry: bazel run //misc/bazel:buildifier
         pass_filenames: false
 
+      - id: go-gen
+        name: Check checked in generated files in go
+        files: ^go/.*
+        language: system
+        entry: bazel run //go:gen
+        pass_filenames: false
+
       - id: codeql-format
         name: Fix QL file formatting
         files: \.qll?$

MODULE.bazel

@@ -13,14 +13,16 @@ local_path_override(
 
 # see https://registry.bazel.build/ for a list of available packages
 
-bazel_dep(name = "platforms", version = "0.0.8")
+bazel_dep(name = "platforms", version = "0.0.9")
+bazel_dep(name = "rules_go", version = "0.47.0")
 bazel_dep(name = "rules_pkg", version = "0.10.1")
 bazel_dep(name = "rules_nodejs", version = "6.0.3")
 bazel_dep(name = "rules_python", version = "0.31.0")
 bazel_dep(name = "bazel_skylib", version = "1.5.0")
 bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl")
 bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
 bazel_dep(name = "fmt", version = "10.0.0")
+bazel_dep(name = "gazelle", version = "0.36.0")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
@@ -52,6 +54,9 @@ node.toolchain(
 )
 use_repo(node, "nodejs", "nodejs_toolchains")
 
+go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk")
+go_sdk.download(version = "1.22.2")
+
 register_toolchains(
     "@nodejs_toolchains//:all",
 )

go/BUILD.bazel

@@ -0,0 +1,93 @@
+load("@bazel_skylib//rules:native_binary.bzl", "native_binary")
+load("@rules_pkg//pkg:install.bzl", "pkg_install")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files")
+load("//:defs.bzl", "codeql_platform")
+
+native_binary(
+    name = "gazelle",
+    src = "@gazelle//cmd/gazelle",
+    out = "gazelle.exe",
+    args = ["go/extractor"],
+)
+
+_gen_binaries = [
+    "@rules_go//go",
+    ":gazelle",
+    "//go/extractor/cli/go-gen-dbscheme",
+]
+
+py_binary(
+    name = "gen",
+    srcs = ["gen.py"],
+    args = ["$(rlocationpath %s)" % bin for bin in _gen_binaries],
+    data = _gen_binaries,
+    deps = ["@rules_python//python/runfiles"],
+)
+
+# this is an instance of the dbscheme kept in the bazel build tree
+# this allows everything that bazel builds to be up-to-date,
+# independently from whether //go:gen was already run to update the checked in files
+genrule(
+    name = "dbscheme",
+    outs = ["go.dbscheme"],
+    cmd = "$(execpath //go/extractor/cli/go-gen-dbscheme) $@",
+    tools = ["//go/extractor/cli/go-gen-dbscheme"],
+)
+
+pkg_files(
+    name = "resources",
+    srcs = [
+        "LICENSE",
+        "codeql-extractor.yml",
+        "ql/lib/go.dbscheme.stats",
+        ":dbscheme",
+    ],
+)
+
+pkg_filegroup(
+    name = "extractor-pack-generic",
+    srcs = [
+        ":resources",
+        "//go/codeql-tools",
+        "//go/downgrades",
+        "//go/extractor:tokenizer",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+pkg_files(
+    name = "extractor-pack-arch",
+    srcs = [
+        "//go/extractor/cli/go-autobuilder",
+        "//go/extractor/cli/go-bootstrap",
+        "//go/extractor/cli/go-build-runner",
+        "//go/extractor/cli/go-extractor",
+        "//go/extractor/cli/go-gen-dbscheme",
+        "//go/extractor/cli/go-tokenizer",
+    ],
+    attributes = pkg_attributes(mode = "0755"),
+    prefix = "tools/" + codeql_platform,
+    visibility = ["//visibility:public"],
+)
+
+pkg_filegroup(
+    name = "extractor-pack",
+    srcs = [
+        ":extractor-pack-arch",
+        ":extractor-pack-generic",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+pkg_install(
+    name = "_extractor_pack",
+    srcs = [":extractor-pack"],
+)
+
+py_binary(
+    name = "create-extractor-pack",
+    srcs = ["create_extractor_pack.py"],
+    env = {"REPO_NAME": repo_name()},
+    main = "create_extractor_pack.py",
+    deps = ["_extractor_pack"],
+)

go/codeql-tools/BUILD.bazel

@@ -0,0 +1,28 @@
+load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files")
+
+pkg_files(
+    name = "sh-files",
+    srcs = glob(["*.sh"]),
+    attributes = pkg_attributes(mode = "0755"),
+)
+
+pkg_files(
+    name = "non-sh-files",
+    srcs = glob(
+        ["*"],
+        exclude = [
+            "*.sh",
+            "BUILD.bazel",
+        ],
+    ),
+)
+
+pkg_filegroup(
+    name = "codeql-tools",
+    srcs = [
+        ":non-sh-files",
+        ":sh-files",
+    ],
+    prefix = "tools",
+    visibility = ["//go:__pkg__"],
+)

go/create_extractor_pack.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+import os
+import pathlib
+import shutil
+import sys
+import subprocess
+
+try:
+    workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY'])
+except KeyError:
+    res = subprocess.run(["bazel", "run", ":create-extractor-pack"], cwd=pathlib.Path(__file__).parent)
+    sys.exit(res.returncode)
+
+from go._extractor_pack_install_script import main
+
+build_dir = workspace_dir / 'go' / 'build'
+
+if not build_dir.exists():
+    # we probably are in the internal repo
+    workspace_dir /= 'ql'
+    build_dir = workspace_dir / 'go' / 'build'
+
+dest_dir = build_dir / 'codeql-extractor-pack'
+shutil.rmtree(dest_dir, ignore_errors=True)
+os.environ['DESTDIR'] = str(dest_dir)
+main(sys.argv)

go/downgrades/BUILD.bazel

@@ -0,0 +1,12 @@
+load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix")
+
+pkg_files(
+    name = "downgrades",
+    srcs = glob(
+        ["**"],
+        exclude = ["BUILD.bazel"],
+    ),
+    prefix = "downgrades",
+    strip_prefix = strip_prefix.from_pkg(),
+    visibility = ["//go:__pkg__"],
+)

go/extractor/.gitattributes

@@ -0,0 +1 @@
+/*/**/BUILD.bazel linguist-generated=true

go/extractor/BUILD.bazel

@@ -0,0 +1,60 @@
+load("@rules_go//go:def.bzl", "go_library")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_files")
+
+# gazelle:prefix github.com/github/codeql-go/extractor
+# gazelle:map_kind go_binary codeql_go_binary //go:rules.bzl
+
+# the immediately following `extractor` target is kept up to date by `bazel run //go:gen`, do not edit directly
+go_library(
+    name = "extractor",
+    srcs = [
+        "extractor.go",
+        "gomodextractor.go",
+        "semaphore.go",
+    ],
+    importpath = "github.com/github/codeql-go/extractor",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//go/extractor/dbscheme",
+        "//go/extractor/diagnostics",
+        "//go/extractor/srcarchive",
+        "//go/extractor/trap",
+        "//go/extractor/util",
+        "//go/extractor/vendor/golang.org/x/mod/modfile",
+        "//go/extractor/vendor/golang.org/x/tools/go/packages",
+    ],
+)
+
+# the other targets are not generated by gazelle
+
+# this is separate from `tokenizer-jar` below because we don't want these compiled class files in the pack
+java_library(
+    name = "tokenizer-deps",
+    srcs = [
+        "net/sourceforge/pmd/cpd/AbstractLanguage.java",
+        "net/sourceforge/pmd/cpd/SourceCode.java",
+        "net/sourceforge/pmd/cpd/TokenEntry.java",
+        "net/sourceforge/pmd/cpd/Tokenizer.java",
+    ],
+)
+
+# we only need these compiled class files in the pack
+java_library(
+    name = "tokenizer-jar",
+    srcs = [
+        "net/sourceforge/pmd/cpd/GoLanguage.java",
+        "opencsv/CSVParser.java",
+        "opencsv/CSVReader.java",
+    ],
+    deps = [":tokenizer-deps"],
+)
+
+pkg_files(
+    name = "tokenizer",
+    srcs = [":tokenizer-jar"],
+    prefix = "tools",
+    renames = {
+        ":tokenizer-jar": "tokenizer.jar",  # name is `libtokenizer.jar` by default
+    },
+    visibility = ["//go:__pkg__"],
+)

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -487,7 +487,9 @@ func extract(workspace project.GoWorkspace) bool {
 
 // Build the project and run the extractor.
 func installDependenciesAndBuild() {
-	log.Printf("Autobuilder was built with %s, environment has %s\n", runtime.Version(), toolchain.GetEnvGoVersion())
+	// do not print experiments the autobuilder was built with if any, only the version
+	version := strings.SplitN(runtime.Version(), " ", 2)[0]
+	log.Printf("Autobuilder was built with %s, environment has %s\n", version, toolchain.GetEnvGoVersion())
 
 	srcdir := getSourceDir()