From 393f6b76664ee13517cd9c9cee918867f3432d1b Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 24 Apr 2024 17:04:57 +0200 Subject: [PATCH 01/41] Go: add gazelle-generated `BUILD` files --- .pre-commit-config.yaml | 7 +++++ MODULE.bazel | 7 +++++ go/extractor/.gitattributes | 1 + go/extractor/BUILD.bazel | 23 ++++++++++++++++ go/extractor/autobuilder/BUILD.bazel | 24 +++++++++++++++++ go/extractor/cli/go-autobuilder/BUILD.bazel | 22 +++++++++++++++ go/extractor/cli/go-bootstrap/BUILD.bazel | 14 ++++++++++ go/extractor/cli/go-build-runner/BUILD.bazel | 18 +++++++++++++ go/extractor/cli/go-extractor/BUILD.bazel | 18 +++++++++++++ go/extractor/cli/go-gen-dbscheme/BUILD.bazel | 15 +++++++++++ go/extractor/cli/go-tokenizer/BUILD.bazel | 14 ++++++++++ go/extractor/dbscheme/BUILD.bazel | 15 +++++++++++ go/extractor/diagnostics/BUILD.bazel | 8 ++++++ go/extractor/project/BUILD.bazel | 21 +++++++++++++++ go/extractor/srcarchive/BUILD.bazel | 17 ++++++++++++ go/extractor/toolchain/BUILD.bazel | 18 +++++++++++++ go/extractor/trap/BUILD.bazel | 23 ++++++++++++++++ go/extractor/util/BUILD.bazel | 14 ++++++++++ .../x/mod/internal/lazyregexp/BUILD.bazel | 9 +++++++ .../golang.org/x/mod/modfile/BUILD.bazel | 19 +++++++++++++ .../golang.org/x/mod/module/BUILD.bazel | 16 +++++++++++ .../golang.org/x/mod/semver/BUILD.bazel | 9 +++++++ .../x/tools/go/gcexportdata/BUILD.bazel | 13 +++++++++ .../go/internal/packagesdriver/BUILD.bazel | 10 +++++++ .../x/tools/go/packages/BUILD.bazel | 25 +++++++++++++++++ .../x/tools/go/types/objectpath/BUILD.bazel | 10 +++++++ .../x/tools/internal/event/BUILD.bazel | 17 ++++++++++++ .../x/tools/internal/event/core/BUILD.bazel | 17 ++++++++++++ .../x/tools/internal/event/keys/BUILD.bazel | 14 ++++++++++ .../x/tools/internal/event/label/BUILD.bazel | 9 +++++++ .../x/tools/internal/event/tag/BUILD.bazel | 10 +++++++ .../x/tools/internal/gcimporter/BUILD.bazel | 27 +++++++++++++++++++ .../x/tools/internal/gocommand/BUILD.bazel | 20 ++++++++++++++ .../internal/packagesinternal/BUILD.bazel | 9 +++++++ .../x/tools/internal/pkgbits/BUILD.bazel | 21 +++++++++++++++ .../tools/internal/tokeninternal/BUILD.bazel | 9 +++++++ .../x/tools/internal/typeparams/BUILD.bazel | 15 +++++++++++ .../tools/internal/typesinternal/BUILD.bazel | 14 ++++++++++ .../x/tools/internal/versions/BUILD.bazel | 15 +++++++++++ go/gazelle/BUILD.bazel | 8 ++++++ go/rules.bzl | 0 41 files changed, 595 insertions(+) create mode 100644 go/extractor/.gitattributes create mode 100644 go/extractor/BUILD.bazel create mode 100644 go/extractor/autobuilder/BUILD.bazel create mode 100644 go/extractor/cli/go-autobuilder/BUILD.bazel create mode 100644 go/extractor/cli/go-bootstrap/BUILD.bazel create mode 100644 go/extractor/cli/go-build-runner/BUILD.bazel create mode 100644 go/extractor/cli/go-extractor/BUILD.bazel create mode 100644 go/extractor/cli/go-gen-dbscheme/BUILD.bazel create mode 100644 go/extractor/cli/go-tokenizer/BUILD.bazel create mode 100644 go/extractor/dbscheme/BUILD.bazel create mode 100644 go/extractor/diagnostics/BUILD.bazel create mode 100644 go/extractor/project/BUILD.bazel create mode 100644 go/extractor/srcarchive/BUILD.bazel create mode 100644 go/extractor/toolchain/BUILD.bazel create mode 100644 go/extractor/trap/BUILD.bazel create mode 100644 go/extractor/util/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel create mode 100644 go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel create mode 100644 go/gazelle/BUILD.bazel create mode 100644 go/rules.bzl diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 383bc1103837..4e81bcc7711c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,6 +27,13 @@ repos: - repo: local hooks: + - id: gazelle + name: Check gazelle-generated BUILD files + files: go/extractor/.* + language: system + entry: bazel run //go/gazelle + pass_filenames: false + - id: codeql-format name: Fix QL file formatting files: \.qll?$ diff --git a/MODULE.bazel b/MODULE.bazel index 4e1fe0d9f7cb..2dc9dd0a7bb5 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -14,6 +14,7 @@ local_path_override( # see https://registry.bazel.build/ for a list of available packages bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "rules_go", version = "0.47.0") bazel_dep(name = "rules_pkg", version = "0.10.1") bazel_dep(name = "rules_nodejs", version = "6.0.3") bazel_dep(name = "rules_python", version = "0.31.0") @@ -22,6 +23,8 @@ bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl") bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") bazel_dep(name = "fmt", version = "10.0.0") +bazel_dep(name = "gazelle", version = "0.36.0", dev_dependency = True) + pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( hub_name = "codegen_deps", @@ -50,6 +53,10 @@ node.toolchain( ) use_repo(node, "nodejs", "nodejs_toolchains") +go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") +go_sdk.download(version = "1.22.2") +go_sdk.host() + register_toolchains( "@nodejs_toolchains//:all", ) diff --git a/go/extractor/.gitattributes b/go/extractor/.gitattributes new file mode 100644 index 000000000000..e406d4136778 --- /dev/null +++ b/go/extractor/.gitattributes @@ -0,0 +1 @@ +/*/**/BUILD.bazel linguist-generated=true diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel new file mode 100644 index 000000000000..ea99340e7d75 --- /dev/null +++ b/go/extractor/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_go//go:def.bzl", "go_library") + +# gazelle:prefix github.com/github/codeql-go/extractor + +go_library( + name = "extractor", + srcs = [ + "extractor.go", + "gomodextractor.go", + "semaphore.go", + ], + importpath = "github.com/github/codeql-go/extractor", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/dbscheme", + "//go/extractor/diagnostics", + "//go/extractor/srcarchive", + "//go/extractor/trap", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/modfile", + "//go/extractor/vendor/golang.org/x/tools/go/packages", + ], +) diff --git a/go/extractor/autobuilder/BUILD.bazel b/go/extractor/autobuilder/BUILD.bazel new file mode 100644 index 000000000000..b81b15816aa8 --- /dev/null +++ b/go/extractor/autobuilder/BUILD.bazel @@ -0,0 +1,24 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "autobuilder", + srcs = [ + "autobuilder.go", + "build-environment.go", + ], + importpath = "github.com/github/codeql-go/extractor/autobuilder", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/diagnostics", + "//go/extractor/project", + "//go/extractor/toolchain", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) + +go_test( + name = "autobuilder_test", + srcs = ["build-environment_test.go"], + embed = [":autobuilder"], +) diff --git a/go/extractor/cli/go-autobuilder/BUILD.bazel b/go/extractor/cli/go-autobuilder/BUILD.bazel new file mode 100644 index 000000000000..b53227b9f1bb --- /dev/null +++ b/go/extractor/cli/go-autobuilder/BUILD.bazel @@ -0,0 +1,22 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go-autobuilder_lib", + srcs = ["go-autobuilder.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-autobuilder", + visibility = ["//visibility:private"], + deps = [ + "//go/extractor/autobuilder", + "//go/extractor/diagnostics", + "//go/extractor/project", + "//go/extractor/toolchain", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) + +go_binary( + name = "go-autobuilder", + embed = [":go-autobuilder_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-bootstrap/BUILD.bazel b/go/extractor/cli/go-bootstrap/BUILD.bazel new file mode 100644 index 000000000000..15255227f9a4 --- /dev/null +++ b/go/extractor/cli/go-bootstrap/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go-bootstrap_lib", + srcs = ["go-bootstrap.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-bootstrap", + visibility = ["//visibility:private"], +) + +go_binary( + name = "go-bootstrap", + embed = [":go-bootstrap_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-build-runner/BUILD.bazel b/go/extractor/cli/go-build-runner/BUILD.bazel new file mode 100644 index 000000000000..fc407c3e1724 --- /dev/null +++ b/go/extractor/cli/go-build-runner/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go-build-runner_lib", + srcs = ["go-build-runner.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-build-runner", + visibility = ["//visibility:private"], + deps = [ + "//go/extractor/autobuilder", + "//go/extractor/util", + ], +) + +go_binary( + name = "go-build-runner", + embed = [":go-build-runner_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-extractor/BUILD.bazel b/go/extractor/cli/go-extractor/BUILD.bazel new file mode 100644 index 000000000000..9419c6f8ba93 --- /dev/null +++ b/go/extractor/cli/go-extractor/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go-extractor_lib", + srcs = ["go-extractor.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-extractor", + visibility = ["//visibility:private"], + deps = [ + "//go/extractor", + "//go/extractor/diagnostics", + ], +) + +go_binary( + name = "go-extractor", + embed = [":go-extractor_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-gen-dbscheme/BUILD.bazel b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel new file mode 100644 index 000000000000..1fbd75998c23 --- /dev/null +++ b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go-gen-dbscheme_lib", + srcs = ["go-gen-dbscheme.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-gen-dbscheme", + visibility = ["//visibility:private"], + deps = ["//go/extractor/dbscheme"], +) + +go_binary( + name = "go-gen-dbscheme", + embed = [":go-gen-dbscheme_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/cli/go-tokenizer/BUILD.bazel b/go/extractor/cli/go-tokenizer/BUILD.bazel new file mode 100644 index 000000000000..f55baecc69f3 --- /dev/null +++ b/go/extractor/cli/go-tokenizer/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "go-tokenizer_lib", + srcs = ["go-tokenizer.go"], + importpath = "github.com/github/codeql-go/extractor/cli/go-tokenizer", + visibility = ["//visibility:private"], +) + +go_binary( + name = "go-tokenizer", + embed = [":go-tokenizer_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/extractor/dbscheme/BUILD.bazel b/go/extractor/dbscheme/BUILD.bazel new file mode 100644 index 000000000000..efaf51684d74 --- /dev/null +++ b/go/extractor/dbscheme/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "dbscheme", + srcs = [ + "dbscheme.go", + "tables.go", + ], + importpath = "github.com/github/codeql-go/extractor/dbscheme", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/trap", + "//go/extractor/vendor/golang.org/x/tools/go/packages", + ], +) diff --git a/go/extractor/diagnostics/BUILD.bazel b/go/extractor/diagnostics/BUILD.bazel new file mode 100644 index 000000000000..8b218dc1317c --- /dev/null +++ b/go/extractor/diagnostics/BUILD.bazel @@ -0,0 +1,8 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "diagnostics", + srcs = ["diagnostics.go"], + importpath = "github.com/github/codeql-go/extractor/diagnostics", + visibility = ["//visibility:public"], +) diff --git a/go/extractor/project/BUILD.bazel b/go/extractor/project/BUILD.bazel new file mode 100644 index 000000000000..dd49b3b320f8 --- /dev/null +++ b/go/extractor/project/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "project", + srcs = ["project.go"], + importpath = "github.com/github/codeql-go/extractor/project", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/diagnostics", + "//go/extractor/toolchain", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/modfile", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) + +go_test( + name = "project_test", + srcs = ["project_test.go"], + embed = [":project"], +) diff --git a/go/extractor/srcarchive/BUILD.bazel b/go/extractor/srcarchive/BUILD.bazel new file mode 100644 index 000000000000..90664c90190e --- /dev/null +++ b/go/extractor/srcarchive/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "srcarchive", + srcs = [ + "projectlayout.go", + "srcarchive.go", + ], + importpath = "github.com/github/codeql-go/extractor/srcarchive", + visibility = ["//visibility:public"], +) + +go_test( + name = "srcarchive_test", + srcs = ["projectlayout_test.go"], + embed = [":srcarchive"], +) diff --git a/go/extractor/toolchain/BUILD.bazel b/go/extractor/toolchain/BUILD.bazel new file mode 100644 index 000000000000..d1ce09c26329 --- /dev/null +++ b/go/extractor/toolchain/BUILD.bazel @@ -0,0 +1,18 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "toolchain", + srcs = ["toolchain.go"], + importpath = "github.com/github/codeql-go/extractor/toolchain", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) + +go_test( + name = "toolchain_test", + srcs = ["toolchain_test.go"], + embed = [":toolchain"], +) diff --git a/go/extractor/trap/BUILD.bazel b/go/extractor/trap/BUILD.bazel new file mode 100644 index 000000000000..6c3f67b32478 --- /dev/null +++ b/go/extractor/trap/BUILD.bazel @@ -0,0 +1,23 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "trap", + srcs = [ + "labels.go", + "trapwriter.go", + "util.go", + ], + importpath = "github.com/github/codeql-go/extractor/trap", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/srcarchive", + "//go/extractor/util", + "//go/extractor/vendor/golang.org/x/tools/go/packages", + ], +) + +go_test( + name = "trap_test", + srcs = ["trapwriter_test.go"], + embed = [":trap"], +) diff --git a/go/extractor/util/BUILD.bazel b/go/extractor/util/BUILD.bazel new file mode 100644 index 000000000000..787a627b5bcc --- /dev/null +++ b/go/extractor/util/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "util", + srcs = ["util.go"], + importpath = "github.com/github/codeql-go/extractor/util", + visibility = ["//visibility:public"], +) + +go_test( + name = "util_test", + srcs = ["util_test.go"], + embed = [":util"], +) diff --git a/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel new file mode 100644 index 000000000000..33c06c34bec7 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "lazyregexp", + srcs = ["lazyre.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/mod/internal/lazyregexp", + importpath = "golang.org/x/mod/internal/lazyregexp", + visibility = ["//go/extractor/vendor/golang.org/x/mod:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel new file mode 100644 index 000000000000..7b5a78d785cf --- /dev/null +++ b/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel @@ -0,0 +1,19 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "modfile", + srcs = [ + "print.go", + "read.go", + "rule.go", + "work.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/mod/modfile", + importpath = "golang.org/x/mod/modfile", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/vendor/golang.org/x/mod/internal/lazyregexp", + "//go/extractor/vendor/golang.org/x/mod/module", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) diff --git a/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel new file mode 100644 index 000000000000..46ff4dbb7d14 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel @@ -0,0 +1,16 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "module", + srcs = [ + "module.go", + "pseudo.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/mod/module", + importpath = "golang.org/x/mod/module", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/vendor/golang.org/x/mod/internal/lazyregexp", + "//go/extractor/vendor/golang.org/x/mod/semver", + ], +) diff --git a/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel new file mode 100644 index 000000000000..5b54efe19818 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "semver", + srcs = ["semver.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/mod/semver", + importpath = "golang.org/x/mod/semver", + visibility = ["//visibility:public"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel new file mode 100644 index 000000000000..57f503f1cb23 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel @@ -0,0 +1,13 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "gcexportdata", + srcs = [ + "gcexportdata.go", + "importer.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/go/gcexportdata", + importpath = "golang.org/x/tools/go/gcexportdata", + visibility = ["//visibility:public"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/gcimporter"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel new file mode 100644 index 000000000000..962442de1e00 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel @@ -0,0 +1,10 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "packagesdriver", + srcs = ["sizes.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver", + importpath = "golang.org/x/tools/go/internal/packagesdriver", + visibility = ["//go/extractor/vendor/golang.org/x/tools/go:__subpackages__"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/gocommand"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel new file mode 100644 index 000000000000..1cad8baca98a --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel @@ -0,0 +1,25 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "packages", + srcs = [ + "doc.go", + "external.go", + "golist.go", + "golist_overlay.go", + "loadmode_string.go", + "packages.go", + "visit.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/go/packages", + importpath = "golang.org/x/tools/go/packages", + visibility = ["//visibility:public"], + deps = [ + "//go/extractor/vendor/golang.org/x/tools/go/gcexportdata", + "//go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver", + "//go/extractor/vendor/golang.org/x/tools/internal/gocommand", + "//go/extractor/vendor/golang.org/x/tools/internal/packagesinternal", + "//go/extractor/vendor/golang.org/x/tools/internal/typesinternal", + "//go/extractor/vendor/golang.org/x/tools/internal/versions", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel new file mode 100644 index 000000000000..1029221f434c --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel @@ -0,0 +1,10 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "objectpath", + srcs = ["objectpath.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/go/types/objectpath", + importpath = "golang.org/x/tools/go/types/objectpath", + visibility = ["//visibility:public"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/typeparams"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel new file mode 100644 index 000000000000..b882fef0d8cc --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "event", + srcs = [ + "doc.go", + "event.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event", + importpath = "golang.org/x/tools/internal/event", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = [ + "//go/extractor/vendor/golang.org/x/tools/internal/event/core", + "//go/extractor/vendor/golang.org/x/tools/internal/event/keys", + "//go/extractor/vendor/golang.org/x/tools/internal/event/label", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel new file mode 100644 index 000000000000..36bd68eed9f9 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel @@ -0,0 +1,17 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "core", + srcs = [ + "event.go", + "export.go", + "fast.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event/core", + importpath = "golang.org/x/tools/internal/event/core", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = [ + "//go/extractor/vendor/golang.org/x/tools/internal/event/keys", + "//go/extractor/vendor/golang.org/x/tools/internal/event/label", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel new file mode 100644 index 000000000000..f1674735052e --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "keys", + srcs = [ + "keys.go", + "standard.go", + "util.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event/keys", + importpath = "golang.org/x/tools/internal/event/keys", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/event/label"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel new file mode 100644 index 000000000000..2329754d6cfc --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "label", + srcs = ["label.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event/label", + importpath = "golang.org/x/tools/internal/event/label", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel new file mode 100644 index 000000000000..276dc5f4489a --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel @@ -0,0 +1,10 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "tag", + srcs = ["tag.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/event/tag", + importpath = "golang.org/x/tools/internal/event/tag", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = ["//go/extractor/vendor/golang.org/x/tools/internal/event/keys"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel new file mode 100644 index 000000000000..1879fe0dadf5 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel @@ -0,0 +1,27 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "gcimporter", + srcs = [ + "bimport.go", + "exportdata.go", + "gcimporter.go", + "iexport.go", + "iimport.go", + "newInterface10.go", + "newInterface11.go", + "support_go117.go", + "support_go118.go", + "unified_no.go", + "ureader_no.go", + "ureader_yes.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/gcimporter", + importpath = "golang.org/x/tools/internal/gcimporter", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = [ + "//go/extractor/vendor/golang.org/x/tools/go/types/objectpath", + "//go/extractor/vendor/golang.org/x/tools/internal/pkgbits", + "//go/extractor/vendor/golang.org/x/tools/internal/tokeninternal", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel new file mode 100644 index 000000000000..58f7091b49c0 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel @@ -0,0 +1,20 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "gocommand", + srcs = [ + "invoke.go", + "vendor.go", + "version.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/gocommand", + importpath = "golang.org/x/tools/internal/gocommand", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], + deps = [ + "//go/extractor/vendor/golang.org/x/mod/semver", + "//go/extractor/vendor/golang.org/x/tools/internal/event", + "//go/extractor/vendor/golang.org/x/tools/internal/event/keys", + "//go/extractor/vendor/golang.org/x/tools/internal/event/label", + "//go/extractor/vendor/golang.org/x/tools/internal/event/tag", + ], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel new file mode 100644 index 000000000000..3ed918c94899 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "packagesinternal", + srcs = ["packages.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/packagesinternal", + importpath = "golang.org/x/tools/internal/packagesinternal", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel new file mode 100644 index 000000000000..820e8f04c9b5 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel @@ -0,0 +1,21 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "pkgbits", + srcs = [ + "codes.go", + "decoder.go", + "doc.go", + "encoder.go", + "flags.go", + "frames_go1.go", + "frames_go17.go", + "reloc.go", + "support.go", + "sync.go", + "syncmarker_string.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/pkgbits", + importpath = "golang.org/x/tools/internal/pkgbits", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel new file mode 100644 index 000000000000..ff66085dbdba --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel @@ -0,0 +1,9 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "tokeninternal", + srcs = ["tokeninternal.go"], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/tokeninternal", + importpath = "golang.org/x/tools/internal/tokeninternal", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel new file mode 100644 index 000000000000..266816d9ea27 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "typeparams", + srcs = [ + "common.go", + "coretype.go", + "normalize.go", + "termlist.go", + "typeterm.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/typeparams", + importpath = "golang.org/x/tools/internal/typeparams", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel new file mode 100644 index 000000000000..c03b8a36ef7a --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "typesinternal", + srcs = [ + "errorcode.go", + "errorcode_string.go", + "types.go", + "types_118.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/typesinternal", + importpath = "golang.org/x/tools/internal/typesinternal", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel new file mode 100644 index 000000000000..6d7ffd0ac686 --- /dev/null +++ b/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel @@ -0,0 +1,15 @@ +load("@rules_go//go:def.bzl", "go_library") + +go_library( + name = "versions", + srcs = [ + "gover.go", + "types.go", + "types_go121.go", + "types_go122.go", + "versions.go", + ], + importmap = "github.com/github/codeql-go/extractor/vendor/golang.org/x/tools/internal/versions", + importpath = "golang.org/x/tools/internal/versions", + visibility = ["//go/extractor/vendor/golang.org/x/tools:__subpackages__"], +) diff --git a/go/gazelle/BUILD.bazel b/go/gazelle/BUILD.bazel new file mode 100644 index 000000000000..7c9b34edb3a6 --- /dev/null +++ b/go/gazelle/BUILD.bazel @@ -0,0 +1,8 @@ +load("@gazelle//:def.bzl", "gazelle") + +gazelle( + name = "gazelle", + extra_args = [ + "go/extractor", + ], +) diff --git a/go/rules.bzl b/go/rules.bzl new file mode 100644 index 000000000000..e69de29bb2d1 From 4ca8faa9c944faa1005046c2c5cef3f6d68131b3 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 25 Apr 2024 09:14:59 +0200 Subject: [PATCH 02/41] Go: introduce universal binaries on macOS --- MODULE.bazel | 3 ++- go/BUILD.bazel | 0 go/extractor/BUILD.bazel | 1 + go/extractor/cli/go-autobuilder/BUILD.bazel | 5 ++-- go/extractor/cli/go-bootstrap/BUILD.bazel | 5 ++-- go/extractor/cli/go-build-runner/BUILD.bazel | 5 ++-- go/extractor/cli/go-extractor/BUILD.bazel | 5 ++-- go/extractor/cli/go-gen-dbscheme/BUILD.bazel | 5 ++-- go/extractor/cli/go-tokenizer/BUILD.bazel | 5 ++-- go/rules.bzl | 5 ++++ misc/bazel/universal_binary.bzl | 24 ++++++++++++++++++++ 11 files changed, 50 insertions(+), 13 deletions(-) create mode 100644 go/BUILD.bazel create mode 100644 misc/bazel/universal_binary.bzl diff --git a/MODULE.bazel b/MODULE.bazel index 2dc9dd0a7bb5..7a85ab51a96c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,7 +13,8 @@ local_path_override( # see https://registry.bazel.build/ for a list of available packages -bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "apple_support", version = "1.15.1") +bazel_dep(name = "platforms", version = "0.0.9") bazel_dep(name = "rules_go", version = "0.47.0") bazel_dep(name = "rules_pkg", version = "0.10.1") bazel_dep(name = "rules_nodejs", version = "6.0.3") diff --git a/go/BUILD.bazel b/go/BUILD.bazel new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel index ea99340e7d75..040b42273140 100644 --- a/go/extractor/BUILD.bazel +++ b/go/extractor/BUILD.bazel @@ -1,6 +1,7 @@ load("@rules_go//go:def.bzl", "go_library") # gazelle:prefix github.com/github/codeql-go/extractor +# gazelle:map_kind go_binary codeql_go_binary //go:rules.bzl go_library( name = "extractor", diff --git a/go/extractor/cli/go-autobuilder/BUILD.bazel b/go/extractor/cli/go-autobuilder/BUILD.bazel index b53227b9f1bb..bf1235b33aa0 100644 --- a/go/extractor/cli/go-autobuilder/BUILD.bazel +++ b/go/extractor/cli/go-autobuilder/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") go_library( name = "go-autobuilder_lib", @@ -15,7 +16,7 @@ go_library( ], ) -go_binary( +codeql_go_binary( name = "go-autobuilder", embed = [":go-autobuilder_lib"], visibility = ["//visibility:public"], diff --git a/go/extractor/cli/go-bootstrap/BUILD.bazel b/go/extractor/cli/go-bootstrap/BUILD.bazel index 15255227f9a4..7bdd6d6e70f9 100644 --- a/go/extractor/cli/go-bootstrap/BUILD.bazel +++ b/go/extractor/cli/go-bootstrap/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") go_library( name = "go-bootstrap_lib", @@ -7,7 +8,7 @@ go_library( visibility = ["//visibility:private"], ) -go_binary( +codeql_go_binary( name = "go-bootstrap", embed = [":go-bootstrap_lib"], visibility = ["//visibility:public"], diff --git a/go/extractor/cli/go-build-runner/BUILD.bazel b/go/extractor/cli/go-build-runner/BUILD.bazel index fc407c3e1724..15557d751cf4 100644 --- a/go/extractor/cli/go-build-runner/BUILD.bazel +++ b/go/extractor/cli/go-build-runner/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") go_library( name = "go-build-runner_lib", @@ -11,7 +12,7 @@ go_library( ], ) -go_binary( +codeql_go_binary( name = "go-build-runner", embed = [":go-build-runner_lib"], visibility = ["//visibility:public"], diff --git a/go/extractor/cli/go-extractor/BUILD.bazel b/go/extractor/cli/go-extractor/BUILD.bazel index 9419c6f8ba93..be426331868c 100644 --- a/go/extractor/cli/go-extractor/BUILD.bazel +++ b/go/extractor/cli/go-extractor/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") go_library( name = "go-extractor_lib", @@ -11,7 +12,7 @@ go_library( ], ) -go_binary( +codeql_go_binary( name = "go-extractor", embed = [":go-extractor_lib"], visibility = ["//visibility:public"], diff --git a/go/extractor/cli/go-gen-dbscheme/BUILD.bazel b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel index 1fbd75998c23..06c0d0f61f18 100644 --- a/go/extractor/cli/go-gen-dbscheme/BUILD.bazel +++ b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") go_library( name = "go-gen-dbscheme_lib", @@ -8,7 +9,7 @@ go_library( deps = ["//go/extractor/dbscheme"], ) -go_binary( +codeql_go_binary( name = "go-gen-dbscheme", embed = [":go-gen-dbscheme_lib"], visibility = ["//visibility:public"], diff --git a/go/extractor/cli/go-tokenizer/BUILD.bazel b/go/extractor/cli/go-tokenizer/BUILD.bazel index f55baecc69f3..3fc5b464c7c5 100644 --- a/go/extractor/cli/go-tokenizer/BUILD.bazel +++ b/go/extractor/cli/go-tokenizer/BUILD.bazel @@ -1,4 +1,5 @@ -load("@rules_go//go:def.bzl", "go_binary", "go_library") +load("@rules_go//go:def.bzl", "go_library") +load("//go:rules.bzl", "codeql_go_binary") go_library( name = "go-tokenizer_lib", @@ -7,7 +8,7 @@ go_library( visibility = ["//visibility:private"], ) -go_binary( +codeql_go_binary( name = "go-tokenizer", embed = [":go-tokenizer_lib"], visibility = ["//visibility:public"], diff --git a/go/rules.bzl b/go/rules.bzl index e69de29bb2d1..4ef798001d2a 100644 --- a/go/rules.bzl +++ b/go/rules.bzl @@ -0,0 +1,5 @@ +load("@rules_go//go:def.bzl", "go_binary") +load("//misc/bazel:universal_binary.bzl", "wrap_as_universal_binary") + +def codeql_go_binary(**kwargs): + wrap_as_universal_binary(go_binary, **kwargs) diff --git a/misc/bazel/universal_binary.bzl b/misc/bazel/universal_binary.bzl new file mode 100644 index 000000000000..85881356d0eb --- /dev/null +++ b/misc/bazel/universal_binary.bzl @@ -0,0 +1,24 @@ +load("@apple_support//rules:universal_binary.bzl", _universal_binary = "universal_binary") + +def wrap_as_universal_binary(rule, *, name, visibility = None, **kwargs): + internal_name = "internal/%s" % name + universal_name = "universal/%s" % name + rule( + name = internal_name, + visibility = ["//visibility:private"], + **kwargs + ) + _universal_binary( + name = universal_name, + target_compatible_with = ["@platforms//os:macos"], + binary = internal_name, + visibility = ["//visibility:private"], + ) + native.alias( + name = name, + actual = select({ + "@platforms//os:macos": universal_name, + "//conditions:default": internal_name, + }), + visibility = visibility, + ) From 7d9a68bf173511af23c94bbd04560c42a123dc09 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 11:19:22 +0200 Subject: [PATCH 03/41] Go: wrap gazelle to regenerate from scratch and add header --- go/extractor/autobuilder/BUILD.bazel | 2 ++ go/extractor/cli/go-autobuilder/BUILD.bazel | 2 ++ go/extractor/cli/go-bootstrap/BUILD.bazel | 2 ++ go/extractor/cli/go-build-runner/BUILD.bazel | 2 ++ go/extractor/cli/go-extractor/BUILD.bazel | 2 ++ go/extractor/cli/go-gen-dbscheme/BUILD.bazel | 2 ++ go/extractor/cli/go-tokenizer/BUILD.bazel | 2 ++ go/extractor/dbscheme/BUILD.bazel | 2 ++ go/extractor/diagnostics/BUILD.bazel | 2 ++ go/extractor/project/BUILD.bazel | 2 ++ go/extractor/srcarchive/BUILD.bazel | 2 ++ go/extractor/toolchain/BUILD.bazel | 2 ++ go/extractor/trap/BUILD.bazel | 2 ++ go/extractor/util/BUILD.bazel | 2 ++ .../x/mod/internal/lazyregexp/BUILD.bazel | 2 ++ .../vendor/golang.org/x/mod/modfile/BUILD.bazel | 2 ++ .../vendor/golang.org/x/mod/module/BUILD.bazel | 2 ++ .../vendor/golang.org/x/mod/semver/BUILD.bazel | 2 ++ .../x/tools/go/gcexportdata/BUILD.bazel | 2 ++ .../tools/go/internal/packagesdriver/BUILD.bazel | 2 ++ .../golang.org/x/tools/go/packages/BUILD.bazel | 2 ++ .../x/tools/go/types/objectpath/BUILD.bazel | 2 ++ .../x/tools/internal/event/BUILD.bazel | 2 ++ .../x/tools/internal/event/core/BUILD.bazel | 2 ++ .../x/tools/internal/event/keys/BUILD.bazel | 2 ++ .../x/tools/internal/event/label/BUILD.bazel | 2 ++ .../x/tools/internal/event/tag/BUILD.bazel | 2 ++ .../x/tools/internal/gcimporter/BUILD.bazel | 2 ++ .../x/tools/internal/gocommand/BUILD.bazel | 2 ++ .../tools/internal/packagesinternal/BUILD.bazel | 2 ++ .../x/tools/internal/pkgbits/BUILD.bazel | 2 ++ .../x/tools/internal/tokeninternal/BUILD.bazel | 2 ++ .../x/tools/internal/typeparams/BUILD.bazel | 2 ++ .../x/tools/internal/typesinternal/BUILD.bazel | 2 ++ .../x/tools/internal/versions/BUILD.bazel | 2 ++ go/gazelle/BUILD.bazel | 11 ++++++++--- go/gazelle/gazelle.py | 16 ++++++++++++++++ 37 files changed, 94 insertions(+), 3 deletions(-) create mode 100644 go/gazelle/gazelle.py diff --git a/go/extractor/autobuilder/BUILD.bazel b/go/extractor/autobuilder/BUILD.bazel index b81b15816aa8..e40dc3a03215 100644 --- a/go/extractor/autobuilder/BUILD.bazel +++ b/go/extractor/autobuilder/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( diff --git a/go/extractor/cli/go-autobuilder/BUILD.bazel b/go/extractor/cli/go-autobuilder/BUILD.bazel index bf1235b33aa0..7abf4600d946 100644 --- a/go/extractor/cli/go-autobuilder/BUILD.bazel +++ b/go/extractor/cli/go-autobuilder/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") load("//go:rules.bzl", "codeql_go_binary") diff --git a/go/extractor/cli/go-bootstrap/BUILD.bazel b/go/extractor/cli/go-bootstrap/BUILD.bazel index 7bdd6d6e70f9..86b08dfa1210 100644 --- a/go/extractor/cli/go-bootstrap/BUILD.bazel +++ b/go/extractor/cli/go-bootstrap/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") load("//go:rules.bzl", "codeql_go_binary") diff --git a/go/extractor/cli/go-build-runner/BUILD.bazel b/go/extractor/cli/go-build-runner/BUILD.bazel index 15557d751cf4..e91c45306106 100644 --- a/go/extractor/cli/go-build-runner/BUILD.bazel +++ b/go/extractor/cli/go-build-runner/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") load("//go:rules.bzl", "codeql_go_binary") diff --git a/go/extractor/cli/go-extractor/BUILD.bazel b/go/extractor/cli/go-extractor/BUILD.bazel index be426331868c..769e4a7b09bc 100644 --- a/go/extractor/cli/go-extractor/BUILD.bazel +++ b/go/extractor/cli/go-extractor/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") load("//go:rules.bzl", "codeql_go_binary") diff --git a/go/extractor/cli/go-gen-dbscheme/BUILD.bazel b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel index 06c0d0f61f18..9de6d2198f81 100644 --- a/go/extractor/cli/go-gen-dbscheme/BUILD.bazel +++ b/go/extractor/cli/go-gen-dbscheme/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") load("//go:rules.bzl", "codeql_go_binary") diff --git a/go/extractor/cli/go-tokenizer/BUILD.bazel b/go/extractor/cli/go-tokenizer/BUILD.bazel index 3fc5b464c7c5..8ce0c76be266 100644 --- a/go/extractor/cli/go-tokenizer/BUILD.bazel +++ b/go/extractor/cli/go-tokenizer/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") load("//go:rules.bzl", "codeql_go_binary") diff --git a/go/extractor/dbscheme/BUILD.bazel b/go/extractor/dbscheme/BUILD.bazel index efaf51684d74..496a5ccdf49d 100644 --- a/go/extractor/dbscheme/BUILD.bazel +++ b/go/extractor/dbscheme/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/diagnostics/BUILD.bazel b/go/extractor/diagnostics/BUILD.bazel index 8b218dc1317c..9b0c148db320 100644 --- a/go/extractor/diagnostics/BUILD.bazel +++ b/go/extractor/diagnostics/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/project/BUILD.bazel b/go/extractor/project/BUILD.bazel index dd49b3b320f8..0048adf91641 100644 --- a/go/extractor/project/BUILD.bazel +++ b/go/extractor/project/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( diff --git a/go/extractor/srcarchive/BUILD.bazel b/go/extractor/srcarchive/BUILD.bazel index 90664c90190e..e72e2e7ca08a 100644 --- a/go/extractor/srcarchive/BUILD.bazel +++ b/go/extractor/srcarchive/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( diff --git a/go/extractor/toolchain/BUILD.bazel b/go/extractor/toolchain/BUILD.bazel index d1ce09c26329..fde8d327e9e2 100644 --- a/go/extractor/toolchain/BUILD.bazel +++ b/go/extractor/toolchain/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( diff --git a/go/extractor/trap/BUILD.bazel b/go/extractor/trap/BUILD.bazel index 6c3f67b32478..6cc7c4983b2e 100644 --- a/go/extractor/trap/BUILD.bazel +++ b/go/extractor/trap/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( diff --git a/go/extractor/util/BUILD.bazel b/go/extractor/util/BUILD.bazel index 787a627b5bcc..8b8869cac524 100644 --- a/go/extractor/util/BUILD.bazel +++ b/go/extractor/util/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library", "go_test") go_library( diff --git a/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel index 33c06c34bec7..deb5dc2b019a 100644 --- a/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/mod/internal/lazyregexp/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel index 7b5a78d785cf..097bacb107ce 100644 --- a/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/mod/modfile/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel index 46ff4dbb7d14..3bf5ae9997d1 100644 --- a/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/mod/module/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel b/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel index 5b54efe19818..760be56c9e08 100644 --- a/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/mod/semver/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel index 57f503f1cb23..5d68c2fe989e 100644 --- a/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/go/gcexportdata/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel index 962442de1e00..2ef27e2c88af 100644 --- a/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/go/internal/packagesdriver/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel index 1cad8baca98a..03d3e3b01585 100644 --- a/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/go/packages/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel index 1029221f434c..374c5c601bc8 100644 --- a/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/go/types/objectpath/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel index b882fef0d8cc..200e436fcd4b 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel index 36bd68eed9f9..a16713f536ce 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/core/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel index f1674735052e..1feefdf1a834 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/keys/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel index 2329754d6cfc..a4430ba0a17c 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/label/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel index 276dc5f4489a..d2c87f41a8ae 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/event/tag/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel index 1879fe0dadf5..56da3b0130e2 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/gcimporter/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel index 58f7091b49c0..7e64f94b95cb 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/gocommand/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel index 3ed918c94899..2d2b7dc5b33c 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/packagesinternal/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel index 820e8f04c9b5..cce327470517 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/pkgbits/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel index ff66085dbdba..c0f6cc8fb13a 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/tokeninternal/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel index 266816d9ea27..9c2dc20b6c65 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/typeparams/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel index c03b8a36ef7a..653752ab7152 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/typesinternal/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel b/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel index 6d7ffd0ac686..85d428debf54 100644 --- a/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel +++ b/go/extractor/vendor/golang.org/x/tools/internal/versions/BUILD.bazel @@ -1,3 +1,5 @@ +# generated running `bazel run //go/gazelle`, do not edit + load("@rules_go//go:def.bzl", "go_library") go_library( diff --git a/go/gazelle/BUILD.bazel b/go/gazelle/BUILD.bazel index 7c9b34edb3a6..3d81b50590a9 100644 --- a/go/gazelle/BUILD.bazel +++ b/go/gazelle/BUILD.bazel @@ -1,8 +1,13 @@ load("@gazelle//:def.bzl", "gazelle") gazelle( + name = "_gazelle", +) + +py_binary( name = "gazelle", - extra_args = [ - "go/extractor", - ], + srcs = ["gazelle.py"], + args = ["$(rlocationpath :_gazelle)"], + data = [":_gazelle"], + deps = ["@rules_python//python/runfiles"], ) diff --git a/go/gazelle/gazelle.py b/go/gazelle/gazelle.py new file mode 100644 index 000000000000..200f3c3ed6b1 --- /dev/null +++ b/go/gazelle/gazelle.py @@ -0,0 +1,16 @@ +import sys +import pathlib +import subprocess +from python.runfiles import runfiles + +this = pathlib.Path(__file__).resolve() +go_extractor_dir = this.parents[1] / "extractor" +gazelle = runfiles.Create().Rlocation(sys.argv[1]) +for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): + build_file.unlink() + +subprocess.check_call([gazelle, "go/extractor"]) + +for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): + contents = build_file.read_text() + build_file.write_text(f"# generated running `bazel run //go/gazelle`, do not edit\n\n{contents}") From 3ad9c026a5c59b8a23d5541a1251fc611dd89988 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 11:20:47 +0200 Subject: [PATCH 04/41] Go: remove `go_sdk.host` It's not required, and it can't work from the internal repository. --- MODULE.bazel | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/MODULE.bazel b/MODULE.bazel index 7a85ab51a96c..16697403dccc 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -23,8 +23,7 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl") bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") bazel_dep(name = "fmt", version = "10.0.0") - -bazel_dep(name = "gazelle", version = "0.36.0", dev_dependency = True) +bazel_dep(name = "gazelle", version = "0.36.0") pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( @@ -56,7 +55,6 @@ use_repo(node, "nodejs", "nodejs_toolchains") go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") go_sdk.download(version = "1.22.2") -go_sdk.host() register_toolchains( "@nodejs_toolchains//:all", From 925a2cca7e6a5c8231b37f67df3de469ddf50925 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 12:01:23 +0200 Subject: [PATCH 05/41] Go: create whole extractor pack with bazel --- go/BUILD.bazel | 59 +++++++++++++++++++++++++++++++++++++ go/codeql-tools/BUILD.bazel | 28 ++++++++++++++++++ go/create_extractor_pack.py | 16 ++++++++++ go/downgrades/BUILD.bazel | 12 ++++++++ go/extractor/BUILD.bazel | 31 +++++++++++++++++++ 5 files changed, 146 insertions(+) create mode 100644 go/codeql-tools/BUILD.bazel create mode 100644 go/create_extractor_pack.py create mode 100644 go/downgrades/BUILD.bazel diff --git a/go/BUILD.bazel b/go/BUILD.bazel index e69de29bb2d1..5736912310a8 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -0,0 +1,59 @@ +load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files") +load("@rules_pkg//pkg:install.bzl", "pkg_install") +load("//:defs.bzl", "codeql_platform") + +pkg_files( + name = "resources", + srcs = [ + "LICENSE", + "codeql-extractor.yml", + "ql/lib/go.dbscheme", + "ql/lib/go.dbscheme.stats", + ], +) + +pkg_filegroup( + name = "extractor-pack-generic", + srcs = [ + ":resources", + "//go/codeql-tools", + "//go/downgrades", + "//go/extractor:tokenizer", + ], + visibility = ["//visibility:public"], +) + +pkg_files( + name = "extractor-pack-arch", + srcs = [ + "//go/extractor/cli/go-autobuilder", + "//go/extractor/cli/go-bootstrap", + "//go/extractor/cli/go-build-runner", + "//go/extractor/cli/go-extractor", + "//go/extractor/cli/go-gen-dbscheme", + "//go/extractor/cli/go-tokenizer", + ], + prefix = "tools/" + codeql_platform, + visibility = ["//visibility:public"], +) + +pkg_filegroup( + name = "extractor-pack", + srcs = [ + ":extractor-pack-arch", + ":extractor-pack-generic", + ], + visibility = ["//visibility:public"], +) + +pkg_install( + name = "_create_extractor_pack", + srcs = [":extractor-pack"], +) + +py_binary( + name = "create-extractor-pack", + srcs = ["create_extractor_pack.py"], + main = "create_extractor_pack.py", + deps = [":_create_extractor_pack"], +) diff --git a/go/codeql-tools/BUILD.bazel b/go/codeql-tools/BUILD.bazel new file mode 100644 index 000000000000..4e839b4774b5 --- /dev/null +++ b/go/codeql-tools/BUILD.bazel @@ -0,0 +1,28 @@ +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") + +pkg_files( + name = "executables", + srcs = glob(["*.sh"]), + attributes = pkg_attributes(mode = "0755"), +) + +pkg_files( + name = "non-executables", + srcs = glob( + ["*"], + exclude = [ + "*.sh", + "BUILD.bazel", + ], + ), +) + +pkg_filegroup( + name = "codeql-tools", + srcs = [ + ":executables", + ":non-executables", + ], + prefix = "tools", + visibility = ["//go:__pkg__"], +) diff --git a/go/create_extractor_pack.py b/go/create_extractor_pack.py new file mode 100644 index 000000000000..08665a2d8dc9 --- /dev/null +++ b/go/create_extractor_pack.py @@ -0,0 +1,16 @@ +import os +import pathlib +import shutil +import sys +from go._create_extractor_pack_install_script import main + +try: + workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) +except KeyError: + print("this should be run with bazel run", file=sys.stderr) + sys.exit(1) + +dest_dir = workspace_dir / 'go' / 'build' / 'codeql-extractor-go' +shutil.rmtree(dest_dir, ignore_errors=True) +os.environ['DESTDIR'] = str(dest_dir) +main(sys.argv) diff --git a/go/downgrades/BUILD.bazel b/go/downgrades/BUILD.bazel new file mode 100644 index 000000000000..68c15741a9f6 --- /dev/null +++ b/go/downgrades/BUILD.bazel @@ -0,0 +1,12 @@ +load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") + +pkg_files( + name = "downgrades", + srcs = glob( + ["**"], + exclude = ["BUILD.bazel"], + ), + prefix = "downgrades", + strip_prefix = strip_prefix.from_pkg(), + visibility = ["//go:__pkg__"], +) diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel index 040b42273140..cb7bc3ac8ef2 100644 --- a/go/extractor/BUILD.bazel +++ b/go/extractor/BUILD.bazel @@ -1,4 +1,5 @@ load("@rules_go//go:def.bzl", "go_library") +load("@rules_pkg//pkg:mappings.bzl", "pkg_files") # gazelle:prefix github.com/github/codeql-go/extractor # gazelle:map_kind go_binary codeql_go_binary //go:rules.bzl @@ -22,3 +23,33 @@ go_library( "//go/extractor/vendor/golang.org/x/tools/go/packages", ], ) + +java_library( + name = "tokenizer-deps", + srcs = [ + "net/sourceforge/pmd/cpd/AbstractLanguage.java", + "net/sourceforge/pmd/cpd/SourceCode.java", + "net/sourceforge/pmd/cpd/TokenEntry.java", + "net/sourceforge/pmd/cpd/Tokenizer.java", + ], +) + +java_library( + name = "tokenizer-jar", + srcs = [ + "net/sourceforge/pmd/cpd/GoLanguage.java", + "opencsv/CSVParser.java", + "opencsv/CSVReader.java", + ], + deps = [":tokenizer-deps"], +) + +pkg_files( + name = "tokenizer", + srcs = [":tokenizer-jar"], + prefix = "tools", + renames = { + ":tokenizer-jar": "tokenizer.jar", + }, + visibility = ["//go:__pkg__"], +) From 19b2e56d0276518c1c6559536338b61e2451acf6 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 13:38:10 +0200 Subject: [PATCH 06/41] Go: group BUILD and dbscheme generation target --- .pre-commit-config.yaml | 8 ++++---- go/BUILD.bazel | 28 ++++++++++++++++++++++++++++ go/gazelle/BUILD.bazel | 13 ------------- go/{gazelle/gazelle.py => gen.py} | 12 ++++++++++-- 4 files changed, 42 insertions(+), 19 deletions(-) delete mode 100644 go/gazelle/BUILD.bazel rename go/{gazelle/gazelle.py => gen.py} (56%) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4e81bcc7711c..899ddd71b898 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -27,11 +27,11 @@ repos: - repo: local hooks: - - id: gazelle - name: Check gazelle-generated BUILD files - files: go/extractor/.* + - id: go-gen + name: Check checked in generated files in go + files: go/.* language: system - entry: bazel run //go/gazelle + entry: bazel run //go:gen pass_filenames: false - id: codeql-format diff --git a/go/BUILD.bazel b/go/BUILD.bazel index 5736912310a8..c5f32c5309f9 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -1,7 +1,26 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files") load("@rules_pkg//pkg:install.bzl", "pkg_install") +load("@bazel_skylib//rules:native_binary.bzl", "native_binary") +load("@gazelle//:def.bzl", "gazelle") load("//:defs.bzl", "codeql_platform") +gazelle( + name = "_gazelle", +) + +_gen_binaries = [ + ":_gazelle", + "//go/extractor/cli/go-gen-dbscheme", +] + +py_binary( + name = "gen", + srcs = ["gen.py"], + args = ["$(rlocationpath %s)" % bin for bin in _gen_binaries], + data = _gen_binaries, + deps = ["@rules_python//python/runfiles"], +) + pkg_files( name = "resources", srcs = [ @@ -57,3 +76,12 @@ py_binary( main = "create_extractor_pack.py", deps = [":_create_extractor_pack"], ) + +native_binary( + name = "gen-dbscheme", + src = "//go/extractor/cli/go-gen-dbscheme", + out = "go-gen-dbscheme", + args = [ + "$$BUILD_WORKSPACE_DIRECTORY/go/ql/lib/go.dbscheme", + ], +) diff --git a/go/gazelle/BUILD.bazel b/go/gazelle/BUILD.bazel deleted file mode 100644 index 3d81b50590a9..000000000000 --- a/go/gazelle/BUILD.bazel +++ /dev/null @@ -1,13 +0,0 @@ -load("@gazelle//:def.bzl", "gazelle") - -gazelle( - name = "_gazelle", -) - -py_binary( - name = "gazelle", - srcs = ["gazelle.py"], - args = ["$(rlocationpath :_gazelle)"], - data = [":_gazelle"], - deps = ["@rules_python//python/runfiles"], -) diff --git a/go/gazelle/gazelle.py b/go/gen.py similarity index 56% rename from go/gazelle/gazelle.py rename to go/gen.py index 200f3c3ed6b1..7b7e9d732376 100644 --- a/go/gazelle/gazelle.py +++ b/go/gen.py @@ -4,13 +4,21 @@ from python.runfiles import runfiles this = pathlib.Path(__file__).resolve() -go_extractor_dir = this.parents[1] / "extractor" -gazelle = runfiles.Create().Rlocation(sys.argv[1]) +go_extractor_dir = this.parent / "extractor" +go_dbscheme = this.parent / "ql" / "lib" / "go.dbscheme" +r = runfiles.Create() +gazelle, go_gen_dbscheme = map(r.Rlocation, sys.argv[1:]) + +print("clearing generated BUILD files") for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): build_file.unlink() +print("running gazelle") subprocess.check_call([gazelle, "go/extractor"]) +print("adding header to generated BUILD files") for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): contents = build_file.read_text() build_file.write_text(f"# generated running `bazel run //go/gazelle`, do not edit\n\n{contents}") + +subprocess.check_call([go_gen_dbscheme, go_dbscheme]) From bfa189e2ac2c251518cb61f9665f0970b36e099d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 13:48:44 +0200 Subject: [PATCH 07/41] Go: use a dbscheme generated during the build in `extractor-pack` --- go/BUILD.bazel | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/go/BUILD.bazel b/go/BUILD.bazel index c5f32c5309f9..d327a406f5fa 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -21,13 +21,23 @@ py_binary( deps = ["@rules_python//python/runfiles"], ) +# this is an internal copy of the dbscheme to be used by extractor-pack +# this allows the extractor-pack target to be independent and up-to-date with respect to +# having run //go:gen to update the checked in files +genrule( + name = "dbscheme", + outs = ["go.dbscheme"], + cmd = "$(execpath //go/extractor/cli/go-gen-dbscheme) $@", + tools = ["//go/extractor/cli/go-gen-dbscheme"], +) + pkg_files( name = "resources", srcs = [ "LICENSE", "codeql-extractor.yml", - "ql/lib/go.dbscheme", "ql/lib/go.dbscheme.stats", + ":dbscheme", ], ) From 146d84bbf8c0012cb3e496a4deb7470afb61a0dd Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 15:20:33 +0200 Subject: [PATCH 08/41] Go: rework makefile --- go/Makefile | 75 +++++------------------------------------------------ 1 file changed, 7 insertions(+), 68 deletions(-) diff --git a/go/Makefile b/go/Makefile index d0289a093a52..d9ba2c7e2d48 100644 --- a/go/Makefile +++ b/go/Makefile @@ -1,4 +1,4 @@ -all: extractor ql/lib/go.dbscheme +all: gen extractor ifeq ($(OS),Windows_NT) EXE = .exe @@ -14,17 +14,11 @@ CODEQL_PLATFORM = osx64 endif endif -CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh baseline-config-empty.json baseline-config-vendor.json configure-baseline.cmd configure-baseline.sh identify-environment.cmd identify-environment.sh index.cmd index.sh pre-finalize.cmd pre-finalize.sh tracing-config.lua) - EXTRACTOR_PACK_OUT = build/codeql-extractor-go -BINARIES = go-extractor go-tokenizer go-autobuilder go-build-runner go-bootstrap go-gen-dbscheme - -.PHONY: tools tools-codeql tools-codeql-full clean autoformat \ - tools-linux64 tools-osx64 tools-win64 check-formatting +.PHONY: extractor gen clean autoformat check-formatting clean: - rm -rf tools/bin tools/linux64 tools/osx64 tools/win64 tools/net tools/opencsv rm -rf $(EXTRACTOR_PACK_OUT) build/stats build/testdb autoformat: @@ -47,66 +41,11 @@ endif qhelp-to-markdown: scripts/qhelp-to-markdown.sh ql/src "$(QHELP_OUT_DIR)" -tools: tools-codeql tools/tokenizer.jar - -.PHONY: $(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))) -$(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))): - go build -C extractor -mod=vendor -o ../$@ ./cli/$(basename $(@F)) - -tools-codeql: tools-$(CODEQL_PLATFORM) - -tools-codeql-full: tools-linux64 tools-osx64 tools-win64 - -tools-linux64: $(addprefix tools/linux64/,$(BINARIES)) - -.PHONY: $(addprefix tools/linux64/,$(BINARIES)) -$(addprefix tools/linux64/,$(BINARIES)): - GOOS=linux GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@ ./cli/$(@F) - -tools-osx64: $(addprefix tools/osx64/,$(BINARIES)) - -.PHONY: $(addprefix tools/osx64/,$(BINARIES)) -$(addprefix tools/osx64/,$(BINARIES)): - GOOS=darwin GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@.amd64 ./cli/$(@F) - GOOS=darwin GOARCH=arm64 go build -C extractor -mod=vendor -o ../$@.arm64 ./cli/$(@F) - lipo -create $@.amd64 $@.arm64 -output $@ - rm $@.amd64 $@.arm64 - -tools-win64: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) - -.PHONY: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) -$(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))): - env GOOS=windows GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@ ./cli/$(basename $(@F)) - -.PHONY: extractor-common extractor extractor-full -extractor-common: codeql-extractor.yml LICENSE ql/lib/go.dbscheme \ - tools/tokenizer.jar $(CODEQL_TOOLS) - rm -rf $(EXTRACTOR_PACK_OUT) - mkdir -p $(EXTRACTOR_PACK_OUT) - cp codeql-extractor.yml LICENSE ql/lib/go.dbscheme ql/lib/go.dbscheme.stats $(EXTRACTOR_PACK_OUT) - mkdir $(EXTRACTOR_PACK_OUT)/tools - cp -r tools/tokenizer.jar $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools - cp -r downgrades $(EXTRACTOR_PACK_OUT) - -extractor: extractor-common tools-codeql - cp -r tools/$(CODEQL_PLATFORM) $(EXTRACTOR_PACK_OUT)/tools - -extractor-full: extractor-common tools-codeql-full - cp -r $(addprefix tools/,linux64 osx64 win64) $(EXTRACTOR_PACK_OUT)/tools - -tools/tokenizer.jar: tools/net/sourceforge/pmd/cpd/GoLanguage.class - jar cf $@ -C tools net - jar uf $@ -C tools opencsv - -tools/net/sourceforge/pmd/cpd/GoLanguage.class: extractor/net/sourceforge/pmd/cpd/GoLanguage.java - javac -cp extractor -d tools $< - rm tools/net/sourceforge/pmd/cpd/AbstractLanguage.class - rm tools/net/sourceforge/pmd/cpd/SourceCode.class - rm tools/net/sourceforge/pmd/cpd/TokenEntry.class - rm tools/net/sourceforge/pmd/cpd/Tokenizer.class +extractor: + bazel run :create-extractor-pack -ql/lib/go.dbscheme: tools/$(CODEQL_PLATFORM)/go-gen-dbscheme$(EXE) - $< $@ +gen: + bazel run :gen build/stats/src.stamp: mkdir -p $(@D)/src @@ -123,7 +62,7 @@ test: all build/testdb/check-upgrade-path codeql test run -j0 ql/test --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache) # use GOOS=linux because GOOS=darwin GOARCH=386 is no longer supported env GOOS=linux GOARCH=386 codeql$(EXE) test run -j0 ql/test/query-tests/Security/CWE-681 --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache) - cd extractor; go test -mod=vendor ./... + cd extractor; bazel test ... bash extractor-smoke-test/test.sh || (echo "Extractor smoke test FAILED"; exit 1) .PHONY: build/testdb/check-upgrade-path From d98ccdfa066544c75c3278c7499e12986cfd11e5 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 16:02:22 +0200 Subject: [PATCH 09/41] Go: update workflow --- .github/workflows/go-tests.yml | 46 ++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 6d9cac5dae91..75c34f4f5de2 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -28,13 +28,6 @@ jobs: name: Test Linux (Ubuntu) runs-on: ubuntu-latest-xl steps: - - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION }} - cache: false - id: go - - name: Check out code uses: actions/checkout@v4 @@ -46,15 +39,41 @@ jobs: run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' - name: Build + run: | + bazel run //go:create-extractor-pack + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + + - name: Test run: | cd go - make + make test cache="${{ steps.query-cache.outputs.cache-dir }}" + + check-code: + name: Check code + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v4 - name: Check that all Go code is autoformatted run: | cd go make check-formatting + - name: Check checked-in generated code + run: | + bazel run //go:gen + git add . + git diff --exit-code HEAD || ( + echo "please run bazel run //go:gen" + exit 1 + ) + - name: Compile qhelp files to markdown run: | cd go @@ -65,14 +84,3 @@ jobs: with: name: qhelp-markdown path: go/qhelp-out/**/*.md - - - name: Cache compilation cache - id: query-cache - uses: ./.github/actions/cache-query-compilation - with: - key: go-qltest - - - name: Test - run: | - cd go - make test cache="${{ steps.query-cache.outputs.cache-dir }}" From 0f387eeac298cb8c30c11a13ba9f9dcd027220e9 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 16:31:04 +0200 Subject: [PATCH 10/41] Go: add vendor update to `//go:gen` --- go/BUILD.bazel | 9 ++++++--- go/extractor/vendor/modules.txt | 1 + go/gen.py | 7 +++++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/go/BUILD.bazel b/go/BUILD.bazel index e769afeea1dd..50e2bc0d447d 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -1,15 +1,17 @@ load("@bazel_skylib//rules:native_binary.bzl", "native_binary") load("@gazelle//:def.bzl", "gazelle") load("@rules_pkg//pkg:install.bzl", "pkg_install") -load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files") +load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") load("//:defs.bzl", "codeql_platform") gazelle( - name = "_gazelle", + name = "gazelle", + extra_args = ["go/extractor"], ) _gen_binaries = [ - ":_gazelle", + "@rules_go//go", + ":gazelle", "//go/extractor/cli/go-gen-dbscheme", ] @@ -62,6 +64,7 @@ pkg_files( "//go/extractor/cli/go-gen-dbscheme", "//go/extractor/cli/go-tokenizer", ], + attributes = pkg_attributes(mode = "0755"), prefix = "tools/" + codeql_platform, visibility = ["//visibility:public"], ) diff --git a/go/extractor/vendor/modules.txt b/go/extractor/vendor/modules.txt index 5687615f62c3..37c4a68d828d 100644 --- a/go/extractor/vendor/modules.txt +++ b/go/extractor/vendor/modules.txt @@ -1,3 +1,4 @@ +## workspace # golang.org/x/mod v0.15.0 ## explicit; go 1.18 golang.org/x/mod/internal/lazyregexp diff --git a/go/gen.py b/go/gen.py index 7b7e9d732376..4b4cac0e46f2 100644 --- a/go/gen.py +++ b/go/gen.py @@ -7,14 +7,17 @@ go_extractor_dir = this.parent / "extractor" go_dbscheme = this.parent / "ql" / "lib" / "go.dbscheme" r = runfiles.Create() -gazelle, go_gen_dbscheme = map(r.Rlocation, sys.argv[1:]) +go, gazelle, go_gen_dbscheme = map(r.Rlocation, sys.argv[1:]) + +print("updating vendor") +subprocess.check_call([go, "-C", go_extractor_dir, "work", "vendor"]) print("clearing generated BUILD files") for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): build_file.unlink() print("running gazelle") -subprocess.check_call([gazelle, "go/extractor"]) +subprocess.check_call([gazelle]) print("adding header to generated BUILD files") for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): From 86d6b8ef21c8a1d82fb23c4530aa31b07de0a305 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 16:39:51 +0200 Subject: [PATCH 11/41] Go: put back go setup --- .github/workflows/go-tests.yml | 47 +++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 75c34f4f5de2..5c9204f8405f 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -16,9 +16,6 @@ on: - .github/actions/** - codeql-workspace.yml -env: - GO_VERSION: '~1.22.0' - permissions: contents: read @@ -31,6 +28,21 @@ jobs: - name: Check out code uses: actions/checkout@v4 + - name: Get go version + shell: bash + run: | + ( + echo -n "GO_VERSION=" + bazel run @rules_go//go -- version | sed 's/go version go\(\S*\) .*/\1/' + ) | tee -a "$GITHUB_ENV" + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + id: go + - name: Set up CodeQL CLI uses: ./.github/actions/fetch-codeql @@ -42,24 +54,6 @@ jobs: run: | bazel run //go:create-extractor-pack - - name: Cache compilation cache - id: query-cache - uses: ./.github/actions/cache-query-compilation - with: - key: go-qltest - - - name: Test - run: | - cd go - make test cache="${{ steps.query-cache.outputs.cache-dir }}" - - check-code: - name: Check code - runs-on: ubuntu-latest - steps: - - name: Check out code - uses: actions/checkout@v4 - - name: Check that all Go code is autoformatted run: | cd go @@ -84,3 +78,14 @@ jobs: with: name: qhelp-markdown path: go/qhelp-out/**/*.md + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + + - name: Test + run: | + cd go + make test cache="${{ steps.query-cache.outputs.cache-dir }}" From d66494dcb0b2b23010794a052a2315f19de10f30 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 26 Apr 2024 17:03:56 +0200 Subject: [PATCH 12/41] Go: update `go-tests-other-os.yml` --- .github/workflows/go-tests-other-os.yml | 58 +++++++------------------ 1 file changed, 15 insertions(+), 43 deletions(-) diff --git a/.github/workflows/go-tests-other-os.yml b/.github/workflows/go-tests-other-os.yml index ded53f868b70..ba3dd0335ad4 100644 --- a/.github/workflows/go-tests-other-os.yml +++ b/.github/workflows/go-tests-other-os.yml @@ -7,64 +7,38 @@ on: - .github/workflows/go-tests-other-os.yml - .github/actions/** - codeql-workspace.yml -env: - GO_VERSION: '~1.22.0' permissions: contents: read jobs: - test-mac: - name: Test MacOS - runs-on: macos-latest + test: + name: Test + strategy: + fail-fast: false + matrix: + os: [macos-latest, windows-latest-xl] + if: matrix.os == 'macos-latest' || github.repository_owner == 'github' + runs-on: ${{ matrix.os }} steps: - - name: Set up Go ${{ env.GO_VERSION }} - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION }} - cache: false - id: go - - name: Check out code uses: actions/checkout@v4 - - name: Set up CodeQL CLI - uses: ./.github/actions/fetch-codeql - - - name: Enable problem matchers in repository + - name: Get go version shell: bash - run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' - - - name: Build - run: | - cd go - make - - - name: Cache compilation cache - id: query-cache - uses: ./.github/actions/cache-query-compilation - with: - key: go-qltest - - name: Test run: | - cd go - make test cache="${{ steps.query-cache.outputs.cache-dir }}" + ( + echo -n "GO_VERSION=" + bazel run @rules_go//go -- version | sed 's/go version go\(\S*\) .*/\1/' + ) | tee -a "$GITHUB_ENV" - test-win: - if: github.repository_owner == 'github' - name: Test Windows - runs-on: windows-latest-xl - steps: - - name: Set up Go ${{ env.GO_VERSION }} + - name: Set up Go uses: actions/setup-go@v5 with: go-version: ${{ env.GO_VERSION }} cache: false id: go - - name: Check out code - uses: actions/checkout@v4 - - name: Set up CodeQL CLI uses: ./.github/actions/fetch-codeql @@ -74,15 +48,13 @@ jobs: - name: Build run: | - cd go - make + bazel run //go:create-extractor-pack - name: Cache compilation cache id: query-cache uses: ./.github/actions/cache-query-compilation with: key: go-qltest - - name: Test run: | cd go From b0758fd1097191313fc62a95cb61edff994707ee Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 10:29:53 +0200 Subject: [PATCH 13/41] Go: workaround for gazelle on macOS See https://github.com/bazelbuild/bazel-gazelle/issues/1793 for details. --- MODULE.bazel | 6 +++++- go/BUILD.bazel | 13 +++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index 6daa5a98215b..5fd99f767404 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -56,7 +56,11 @@ node.toolchain( use_repo(node, "nodejs", "nodejs_toolchains") go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") -go_sdk.download(version = "1.22.2") +go_sdk.download(version = "1.22.2") # default + +# following is needed for gazelle on macOS +# see https://github.com/bazelbuild/bazel-gazelle/issues/1793 +go_sdk.download(version = "1.21.9") register_toolchains( "@nodejs_toolchains//:all", diff --git a/go/BUILD.bazel b/go/BUILD.bazel index 50e2bc0d447d..41113d45f518 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -1,12 +1,25 @@ load("@bazel_skylib//rules:native_binary.bzl", "native_binary") load("@gazelle//:def.bzl", "gazelle") +load("@rules_go//go:def.bzl", "go_cross_binary") load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") load("//:defs.bzl", "codeql_platform") +# following is needed for running gazelle on macOS +# see https://github.com/bazelbuild/bazel-gazelle/issues/1793 +go_cross_binary( + name = "gazelle-1.21.9", + sdk_version = "1.21.9", + target = "@gazelle//cmd/gazelle", +) + gazelle( name = "gazelle", extra_args = ["go/extractor"], + gazelle = select({ + "@platforms//os:macos": ":gazelle-1.21.9", + "//conditions:default": "@gazelle//cmd/gazelle", + }), ) _gen_binaries = [ From 0dfd3367291da2e84b2dbe95f57d3ae054992bf3 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 10:34:22 +0200 Subject: [PATCH 14/41] Go: fix `//go:gen` on windows --- go/gen.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/go/gen.py b/go/gen.py index 4b4cac0e46f2..b9394cd15b28 100644 --- a/go/gen.py +++ b/go/gen.py @@ -1,11 +1,17 @@ import sys import pathlib import subprocess +import os from python.runfiles import runfiles -this = pathlib.Path(__file__).resolve() -go_extractor_dir = this.parent / "extractor" -go_dbscheme = this.parent / "ql" / "lib" / "go.dbscheme" +try: + workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) +except KeyError: + print("this should be run with bazel run", file=sys.stderr) + sys.exit(1) + +go_extractor_dir = workspace_dir / "go" / "extractor" +go_dbscheme = workspace_dir / "go" / "ql" / "lib" / "go.dbscheme" r = runfiles.Create() go, gazelle, go_gen_dbscheme = map(r.Rlocation, sys.argv[1:]) From 6ec223c5150efbacdb14157f7e5f0815bac7d62a Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 11:00:21 +0200 Subject: [PATCH 15/41] Go: small cleanup in `Makefile` --- go/Makefile | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/go/Makefile b/go/Makefile index d9ba2c7e2d48..3ebeb69fcaf1 100644 --- a/go/Makefile +++ b/go/Makefile @@ -1,19 +1,5 @@ all: gen extractor -ifeq ($(OS),Windows_NT) -EXE = .exe -CODEQL_PLATFORM = win64 -else -EXE = -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Linux) -CODEQL_PLATFORM = linux64 -endif -ifeq ($(UNAME_S),Darwin) -CODEQL_PLATFORM = osx64 -endif -endif - EXTRACTOR_PACK_OUT = build/codeql-extractor-go .PHONY: extractor gen clean autoformat check-formatting From 2f6dd2ab819c59425a4c636cdb65930d6c6405fb Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 11:11:23 +0200 Subject: [PATCH 16/41] Go: refactor workflows with shared action --- .github/workflows/go-tests-other-os.yml | 57 +++++------------- .github/workflows/go-tests.yml | 64 +------------------- go/actions/test/action.yml | 80 +++++++++++++++++++++++++ 3 files changed, 97 insertions(+), 104 deletions(-) create mode 100644 go/actions/test/action.yml diff --git a/.github/workflows/go-tests-other-os.yml b/.github/workflows/go-tests-other-os.yml index ba3dd0335ad4..9915b0869db7 100644 --- a/.github/workflows/go-tests-other-os.yml +++ b/.github/workflows/go-tests-other-os.yml @@ -12,50 +12,21 @@ permissions: contents: read jobs: - test: - name: Test - strategy: - fail-fast: false - matrix: - os: [macos-latest, windows-latest-xl] - if: matrix.os == 'macos-latest' || github.repository_owner == 'github' - runs-on: ${{ matrix.os }} + test-mac: + name: Test MacOS + runs-on: macos-latest steps: - name: Check out code uses: actions/checkout@v4 + - name: Run tests + uses: ./go/actions/test - - name: Get go version - shell: bash - run: | - ( - echo -n "GO_VERSION=" - bazel run @rules_go//go -- version | sed 's/go version go\(\S*\) .*/\1/' - ) | tee -a "$GITHUB_ENV" - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION }} - cache: false - id: go - - - name: Set up CodeQL CLI - uses: ./.github/actions/fetch-codeql - - - name: Enable problem matchers in repository - shell: bash - run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' - - - name: Build - run: | - bazel run //go:create-extractor-pack - - - name: Cache compilation cache - id: query-cache - uses: ./.github/actions/cache-query-compilation - with: - key: go-qltest - - name: Test - run: | - cd go - make test cache="${{ steps.query-cache.outputs.cache-dir }}" + test-win: + if: github.repository_owner == 'github' + name: Test Windows + runs-on: windows-latest-xl + steps: + - name: Check out code + uses: actions/checkout@v4 + - name: Run tests + uses: ./go/actions/test diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 5c9204f8405f..63e2b7c49740 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -27,65 +27,7 @@ jobs: steps: - name: Check out code uses: actions/checkout@v4 - - - name: Get go version - shell: bash - run: | - ( - echo -n "GO_VERSION=" - bazel run @rules_go//go -- version | sed 's/go version go\(\S*\) .*/\1/' - ) | tee -a "$GITHUB_ENV" - - - name: Set up Go - uses: actions/setup-go@v5 + - name: Run tests + uses: ./go/actions/test with: - go-version: ${{ env.GO_VERSION }} - cache: false - id: go - - - name: Set up CodeQL CLI - uses: ./.github/actions/fetch-codeql - - - name: Enable problem matchers in repository - shell: bash - run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' - - - name: Build - run: | - bazel run //go:create-extractor-pack - - - name: Check that all Go code is autoformatted - run: | - cd go - make check-formatting - - - name: Check checked-in generated code - run: | - bazel run //go:gen - git add . - git diff --exit-code HEAD || ( - echo "please run bazel run //go:gen" - exit 1 - ) - - - name: Compile qhelp files to markdown - run: | - cd go - env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown - - - name: Upload qhelp markdown - uses: actions/upload-artifact@v3 - with: - name: qhelp-markdown - path: go/qhelp-out/**/*.md - - - name: Cache compilation cache - id: query-cache - uses: ./.github/actions/cache-query-compilation - with: - key: go-qltest - - - name: Test - run: | - cd go - make test cache="${{ steps.query-cache.outputs.cache-dir }}" + run-code-checks: true diff --git a/go/actions/test/action.yml b/go/actions/test/action.yml new file mode 100644 index 000000000000..f9bdee5fe0c0 --- /dev/null +++ b/go/actions/test/action.yml @@ -0,0 +1,80 @@ +name: Test go extractor +description: Run build, QL tests and optionally basic code sanity checks (formatting and generation) +inputs: + run-code-checks: + description: Whether to run formatting, code and qhelp generation checks + required: false + default: false +runs: + using: composite + steps: + - name: Get go version + shell: bash + run: | + ( + echo -n "GO_VERSION=" + bazel run @rules_go//go -- version | sed 's/go version go\(\S*\) .*/\1/' + ) | tee -a "$GITHUB_ENV" + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + id: go + + - name: Set up CodeQL CLI + uses: ./.github/actions/fetch-codeql + + - name: Enable problem matchers in repository + shell: bash + run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' + + - name: Build + shell: bash + run: | + bazel run //go:create-extractor-pack + + - name: Check that all Go code is autoformatted + if: inputs.run-code-checks == 'true' + shell: bash + run: | + cd go + make check-formatting + + - name: Check checked-in generated code + if: inputs.run-code-checks == 'true' + shell: bash + run: | + bazel run //go:gen + git add . + git diff --exit-code HEAD || ( + echo "please run bazel run //go:gen" + exit 1 + ) + + - name: Compile qhelp files to markdown + if: inputs.run-code-checks == 'true' + shell: bash + run: | + cd go + env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown + + - name: Upload qhelp markdown + if: inputs.run-code-checks == 'true' + uses: actions/upload-artifact@v3 + with: + name: qhelp-markdown + path: go/qhelp-out/**/*.md + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + + - name: Test + shell: bash + run: | + cd go + make test cache="${{ steps.query-cache.outputs.cache-dir }}" From f0f6c229f6d10c6f3f770777a5bc23b1bd808439 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 11:16:30 +0200 Subject: [PATCH 17/41] Go: fix regex in action for macOS --- go/actions/test/action.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/actions/test/action.yml b/go/actions/test/action.yml index f9bdee5fe0c0..ade22410d957 100644 --- a/go/actions/test/action.yml +++ b/go/actions/test/action.yml @@ -13,7 +13,7 @@ runs: run: | ( echo -n "GO_VERSION=" - bazel run @rules_go//go -- version | sed 's/go version go\(\S*\) .*/\1/' + bazel run @rules_go//go -- version | sed 's/go version go\(.*\) .*/\1/' ) | tee -a "$GITHUB_ENV" - name: Set up Go From 1f78882cdc80011a86dfeb866b08dd46f3004121 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 11:27:16 +0200 Subject: [PATCH 18/41] Go: make windows checks happy --- go/actions/test/action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/actions/test/action.yml b/go/actions/test/action.yml index ade22410d957..a351c4382ac9 100644 --- a/go/actions/test/action.yml +++ b/go/actions/test/action.yml @@ -33,7 +33,7 @@ runs: - name: Build shell: bash run: | - bazel run //go:create-extractor-pack + bazel run go:create-extractor-pack - name: Check that all Go code is autoformatted if: inputs.run-code-checks == 'true' @@ -46,7 +46,7 @@ runs: if: inputs.run-code-checks == 'true' shell: bash run: | - bazel run //go:gen + bazel run go:gen git add . git diff --exit-code HEAD || ( echo "please run bazel run //go:gen" From 15bb846a5fb9a00748219475e4a2d436181e5e1a Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 12:17:47 +0200 Subject: [PATCH 19/41] Go: add workaround for extractor pack windows installer --- go/BUILD.bazel | 23 +++++++++++++++++++++-- go/create_extractor_pack.py | 16 +++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/go/BUILD.bazel b/go/BUILD.bazel index 41113d45f518..4251ebd228f2 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -3,6 +3,7 @@ load("@gazelle//:def.bzl", "gazelle") load("@rules_go//go:def.bzl", "go_cross_binary") load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") +load("@rules_pkg//pkg:zip.bzl", "pkg_zip") load("//:defs.bzl", "codeql_platform") # following is needed for running gazelle on macOS @@ -92,15 +93,33 @@ pkg_filegroup( ) pkg_install( - name = "_create_extractor_pack", + name = "_extractor-pack-installer", srcs = [":extractor-pack"], ) +# rules_pkg installer is currently broken on Windows +# see https://github.com/bazelbuild/rules_pkg/issues/387 +# for now, work around it using an archive +pkg_zip( + name = "_extractor-pack-zip", + srcs = [":extractor-pack"], +) + +alias( + name = "_create-extractor-pack-arg", + actual = select({ + "@platforms//os:windows": ":_extractor-pack-zip", + "//conditions:default": ":_extractor-pack-installer", + }), +) + py_binary( name = "create-extractor-pack", srcs = ["create_extractor_pack.py"], + args = ["$(rlocationpath :_create-extractor-pack-arg)"], + data = [":_create-extractor-pack-arg"], main = "create_extractor_pack.py", - deps = [":_create_extractor_pack"], + deps = ["@rules_python//python/runfiles"], ) native_binary( diff --git a/go/create_extractor_pack.py b/go/create_extractor_pack.py index 08665a2d8dc9..a1154a777d85 100644 --- a/go/create_extractor_pack.py +++ b/go/create_extractor_pack.py @@ -2,7 +2,9 @@ import pathlib import shutil import sys -from go._create_extractor_pack_install_script import main +import subprocess +import zipfile +from python.runfiles import runfiles try: workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) @@ -11,6 +13,14 @@ sys.exit(1) dest_dir = workspace_dir / 'go' / 'build' / 'codeql-extractor-go' +installer_or_zip = pathlib.Path(runfiles.Create().Rlocation(sys.argv[1])) + shutil.rmtree(dest_dir, ignore_errors=True) -os.environ['DESTDIR'] = str(dest_dir) -main(sys.argv) + +if installer_or_zip.suffix == '.zip': + dest_dir.mkdir() + with zipfile.ZipFile(installer_or_zip) as pack: + pack.extractall(dest_dir) +else: + os.environ['DESTDIR'] = str(dest_dir) + subprocess.check_call([installer_or_zip]) From e7886d0e572a3b13e073dce8a121ef44dd9e6e26 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 13:31:48 +0200 Subject: [PATCH 20/41] Bazel: add empty registry override --- .bazelrc | 3 + .bazelrc.internal | 4 + misc/bazel/registry/AUTHORS | 7 + misc/bazel/registry/LICENSE | 202 ++++++++++++++++++++++++ misc/bazel/registry/NOTICE | 3 + misc/bazel/registry/README.md | 3 + misc/bazel/registry/bazel_registry.json | 3 + misc/bazel/registry/fix.py | 55 +++++++ 8 files changed, 280 insertions(+) create mode 100644 .bazelrc.internal create mode 100644 misc/bazel/registry/AUTHORS create mode 100644 misc/bazel/registry/LICENSE create mode 100644 misc/bazel/registry/NOTICE create mode 100644 misc/bazel/registry/README.md create mode 100644 misc/bazel/registry/bazel_registry.json create mode 100755 misc/bazel/registry/fix.py diff --git a/.bazelrc b/.bazelrc index 12232b4bbd68..0a49f682da37 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,4 +14,7 @@ build:linux --cxxopt=-std=c++20 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor +common --registry=file:///%workspace%/misc/bazel/registry +common --registry=https://bcr.bazel.build + try-import %workspace%/local.bazelrc diff --git a/.bazelrc.internal b/.bazelrc.internal new file mode 100644 index 000000000000..cdffa9ccdea6 --- /dev/null +++ b/.bazelrc.internal @@ -0,0 +1,4 @@ +# this file should contain bazel settings required to build things from `semmle-code` + +common --registry=file:///%workspace%/ql/misc/bazel/registry +common --registry=https://bcr.bazel.build diff --git a/misc/bazel/registry/AUTHORS b/misc/bazel/registry/AUTHORS new file mode 100644 index 000000000000..42818b292e7c --- /dev/null +++ b/misc/bazel/registry/AUTHORS @@ -0,0 +1,7 @@ +# This is the list of Bazel's significant contributors. +# +# This does not necessarily list everyone who has contributed code, +# especially since many employees of one corporation may be contributing. +# To see the full list of contributors, see the revision history in +# source control. +Google LLC diff --git a/misc/bazel/registry/LICENSE b/misc/bazel/registry/LICENSE new file mode 100644 index 000000000000..d64569567334 --- /dev/null +++ b/misc/bazel/registry/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/misc/bazel/registry/NOTICE b/misc/bazel/registry/NOTICE new file mode 100644 index 000000000000..95329ae18f60 --- /dev/null +++ b/misc/bazel/registry/NOTICE @@ -0,0 +1,3 @@ +The files in this directory where originally taken from http://github.com/bazelbuild/bazel-central-registry and are +a derivative work thereof, distributed under the Apache 2.0 license, with the following exceptions: +* the `fix.py` file was added under the MIT license as the rest of the `codeql` repository. diff --git a/misc/bazel/registry/README.md b/misc/bazel/registry/README.md new file mode 100644 index 000000000000..5d1723d0eacb --- /dev/null +++ b/misc/bazel/registry/README.md @@ -0,0 +1,3 @@ +Versions to be patched can be taken from https://github.com/bazelbuild/bazel-central-repository. After adding patches +inside `//patches`, and eventually renaming ``, run [`fix.py`](./fix.py) to align all metadata +to the renamed version and added patches. diff --git a/misc/bazel/registry/bazel_registry.json b/misc/bazel/registry/bazel_registry.json new file mode 100644 index 000000000000..ea3f94f7a1e4 --- /dev/null +++ b/misc/bazel/registry/bazel_registry.json @@ -0,0 +1,3 @@ +{ + "mirrors": [] +} diff --git a/misc/bazel/registry/fix.py b/misc/bazel/registry/fix.py new file mode 100755 index 000000000000..a2b947e19e20 --- /dev/null +++ b/misc/bazel/registry/fix.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2024 GitHub, Inc. + +""" +Fix metadata in overridden registry, updating `metadata.json` to list correct versions and `source.json` +to list correct patches with sha256 hashes. +""" + +import pathlib +import json +import base64 +import hashlib +import re + +this_dir = pathlib.Path(__file__).resolve().parent + + +def sha256(file): + with open(file, 'rb') as input: + hash = hashlib.sha256(input.read()).digest() + hash = base64.b64encode(hash).decode() + return f"sha256-{hash}" + + +def patch_file(file, f): + try: + data = file.read_text() + except FileNotFoundError: + data = None + file.write_text(f(data)) + + +def patch_json(file, **kwargs): + def update(data): + data = json.loads(data) if data else {} + data.update(kwargs) + return json.dumps(data, indent=4) + "\n" + + patch_file(file, update) + + +for entry in this_dir.joinpath("modules").iterdir(): + if not entry.is_dir(): + continue + versions = [e for e in entry.iterdir() if e.is_dir()] + + patch_json(entry / "metadata.json", versions=[v.name for v in versions]) + + for version in versions: + patch_json(version / "source.json", patches={ + p.name: sha256(p) for p in version.joinpath("patches").iterdir() + }) + patch_file(version / "MODULE.bazel", + lambda s: re.sub(r'''version\s*=\s*['"].*['"]''', f'version = "{version.name}"', s, 1)) From cb85a756a053b4f7526e3ef7ab88e4c044baa39d Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 29 Apr 2024 13:42:05 +0200 Subject: [PATCH 21/41] Bazel: patch `apple_support` to avoid registering its toolchain This is done in order to avoid requiring a full Xcode installation, but still being able to use other `apple_support` facilities, like `universal_binary`. --- MODULE.bazel | 2 +- .../1.15.1-codeql.1/MODULE.bazel | 17 ++++++++++++++++ .../1.15.1-codeql.1/patches/module.patch | 20 +++++++++++++++++++ .../apple_support/1.15.1-codeql.1/source.json | 9 +++++++++ .../modules/apple_support/metadata.json | 5 +++++ 5 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel create mode 100644 misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch create mode 100644 misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json create mode 100644 misc/bazel/registry/modules/apple_support/metadata.json diff --git a/MODULE.bazel b/MODULE.bazel index 5fd99f767404..875e61da383a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,7 +13,7 @@ local_path_override( # see https://registry.bazel.build/ for a list of available packages -bazel_dep(name = "apple_support", version = "1.15.1") +bazel_dep(name = "apple_support", version = "1.15.1-codeql.1") bazel_dep(name = "platforms", version = "0.0.9") bazel_dep(name = "rules_go", version = "0.47.0") bazel_dep(name = "rules_pkg", version = "0.10.1") diff --git a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel new file mode 100644 index 000000000000..a58d520fee20 --- /dev/null +++ b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel @@ -0,0 +1,17 @@ +module( + name = "apple_support", + version = "1.15.1-codeql.1", + bazel_compatibility = [">=6.0.0"], + compatibility_level = 1, + repo_name = "build_bazel_apple_support", +) + +bazel_dep(name = "bazel_skylib", version = "1.3.0") +bazel_dep(name = "platforms", version = "0.0.9") + +bazel_dep( + name = "stardoc", + version = "0.6.2", + dev_dependency = True, + repo_name = "io_bazel_stardoc", +) diff --git a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch new file mode 100644 index 000000000000..59fc49ec7b37 --- /dev/null +++ b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch @@ -0,0 +1,20 @@ +diff --git a/MODULE.bazel b/MODULE.bazel +index 6b06c3b..99bc7c6 100644 +--- a/MODULE.bazel ++++ b/MODULE.bazel +@@ -1,6 +1,6 @@ + module( + name = "apple_support", +- version = "0", ++ version = "1.15.1-codeql.1", + bazel_compatibility = [">=6.0.0"], + compatibility_level = 1, + repo_name = "build_bazel_apple_support", +@@ -16,7 +16,3 @@ bazel_dep( + repo_name = "io_bazel_stardoc", + ) + +-apple_cc_configure = use_extension("//crosstool:setup.bzl", "apple_cc_configure_extension") +-use_repo(apple_cc_configure, "local_config_apple_cc", "local_config_apple_cc_toolchains") +- +-register_toolchains("@local_config_apple_cc_toolchains//:all") diff --git a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json new file mode 100644 index 000000000000..57a1f2137fea --- /dev/null +++ b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json @@ -0,0 +1,9 @@ +{ + "integrity": "sha256-xLsrc2fEhDgjAK7nW+WYuS+EeJb7MbvSLzojRq32aoA=", + "strip_prefix": "", + "url": "https://github.com/bazelbuild/apple_support/releases/download/1.15.1/apple_support.1.15.1.tar.gz", + "patches": { + "module.patch": "sha256-K06B2W9t6nKcU8S5u6cWeNIdw/vGWWKAoJdGiI8CSS0=" + }, + "patch_strip": 1 +} diff --git a/misc/bazel/registry/modules/apple_support/metadata.json b/misc/bazel/registry/modules/apple_support/metadata.json new file mode 100644 index 000000000000..8a0e54c9abd9 --- /dev/null +++ b/misc/bazel/registry/modules/apple_support/metadata.json @@ -0,0 +1,5 @@ +{ + "versions": [ + "1.15.1-codeql.1" + ] +} From 608791fd7f99166b6f6daba5b27f9abb7c37bac9 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 09:13:40 +0200 Subject: [PATCH 22/41] Bazel/Go: use native cross compilation for fat binaries --- go/rules.bzl | 40 +++++++++++++++++++++++++++++---- misc/bazel/universal_binary.bzl | 24 -------------------- 2 files changed, 36 insertions(+), 28 deletions(-) delete mode 100644 misc/bazel/universal_binary.bzl diff --git a/go/rules.bzl b/go/rules.bzl index 4ef798001d2a..e26dd57bf449 100644 --- a/go/rules.bzl +++ b/go/rules.bzl @@ -1,5 +1,37 @@ -load("@rules_go//go:def.bzl", "go_binary") -load("//misc/bazel:universal_binary.bzl", "wrap_as_universal_binary") +load("@rules_go//go:def.bzl", "go_binary", "go_cross_binary") -def codeql_go_binary(**kwargs): - wrap_as_universal_binary(go_binary, **kwargs) +def codeql_go_binary(*, name, visibility = None, **kwargs): + def internal(prefix = "internal"): + return "%s/%s" % (prefix, name) + + go_binary( + name = internal(), + visibility = ["//visibility:private"], + **kwargs + ) + macos_targets = ("darwin_arm64", "darwin_amd64") + for target in macos_targets: + go_cross_binary( + name = internal(target), + platform = "@rules_go//go/toolchain:%s" % target, + target = internal(), + target_compatible_with = ["@platforms//os:macos"], + visibility = ["//visibility:private"], + ) + native.genrule( + name = internal("universal"), + outs = [internal("universal_")], + srcs = [internal(t) for t in macos_targets], + target_compatible_with = ["@platforms//os:macos"], + executable = True, + visibility = ["//visibility:private"], + cmd = "lipo -create $(SRCS) -output $@", + ) + native.alias( + name = name, + actual = select({ + "@platforms//os:macos": internal("universal"), + "//conditions:default": internal(), + }), + visibility = visibility, + ) diff --git a/misc/bazel/universal_binary.bzl b/misc/bazel/universal_binary.bzl deleted file mode 100644 index 85881356d0eb..000000000000 --- a/misc/bazel/universal_binary.bzl +++ /dev/null @@ -1,24 +0,0 @@ -load("@apple_support//rules:universal_binary.bzl", _universal_binary = "universal_binary") - -def wrap_as_universal_binary(rule, *, name, visibility = None, **kwargs): - internal_name = "internal/%s" % name - universal_name = "universal/%s" % name - rule( - name = internal_name, - visibility = ["//visibility:private"], - **kwargs - ) - _universal_binary( - name = universal_name, - target_compatible_with = ["@platforms//os:macos"], - binary = internal_name, - visibility = ["//visibility:private"], - ) - native.alias( - name = name, - actual = select({ - "@platforms//os:macos": universal_name, - "//conditions:default": internal_name, - }), - visibility = visibility, - ) From 94212d103ec3cffd91f5efcffa668db36f99b7a7 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 09:14:15 +0200 Subject: [PATCH 23/41] Bazel/Go: remove `apple_support` This was actually unnecessary, and requried a full Xcode installation that we'd rather avoid. --- MODULE.bazel | 5 ----- go/BUILD.bazel | 13 ------------ .../1.15.1-codeql.1/MODULE.bazel | 17 ---------------- .../1.15.1-codeql.1/patches/module.patch | 20 ------------------- .../apple_support/1.15.1-codeql.1/source.json | 9 --------- .../modules/apple_support/metadata.json | 5 ----- 6 files changed, 69 deletions(-) delete mode 100644 misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel delete mode 100644 misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch delete mode 100644 misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json delete mode 100644 misc/bazel/registry/modules/apple_support/metadata.json diff --git a/MODULE.bazel b/MODULE.bazel index 875e61da383a..d069d320f07a 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,7 +13,6 @@ local_path_override( # see https://registry.bazel.build/ for a list of available packages -bazel_dep(name = "apple_support", version = "1.15.1-codeql.1") bazel_dep(name = "platforms", version = "0.0.9") bazel_dep(name = "rules_go", version = "0.47.0") bazel_dep(name = "rules_pkg", version = "0.10.1") @@ -58,10 +57,6 @@ use_repo(node, "nodejs", "nodejs_toolchains") go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") go_sdk.download(version = "1.22.2") # default -# following is needed for gazelle on macOS -# see https://github.com/bazelbuild/bazel-gazelle/issues/1793 -go_sdk.download(version = "1.21.9") - register_toolchains( "@nodejs_toolchains//:all", ) diff --git a/go/BUILD.bazel b/go/BUILD.bazel index 4251ebd228f2..6c80e7cb258c 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -1,26 +1,13 @@ load("@bazel_skylib//rules:native_binary.bzl", "native_binary") load("@gazelle//:def.bzl", "gazelle") -load("@rules_go//go:def.bzl", "go_cross_binary") load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") load("//:defs.bzl", "codeql_platform") -# following is needed for running gazelle on macOS -# see https://github.com/bazelbuild/bazel-gazelle/issues/1793 -go_cross_binary( - name = "gazelle-1.21.9", - sdk_version = "1.21.9", - target = "@gazelle//cmd/gazelle", -) - gazelle( name = "gazelle", extra_args = ["go/extractor"], - gazelle = select({ - "@platforms//os:macos": ":gazelle-1.21.9", - "//conditions:default": "@gazelle//cmd/gazelle", - }), ) _gen_binaries = [ diff --git a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel deleted file mode 100644 index a58d520fee20..000000000000 --- a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/MODULE.bazel +++ /dev/null @@ -1,17 +0,0 @@ -module( - name = "apple_support", - version = "1.15.1-codeql.1", - bazel_compatibility = [">=6.0.0"], - compatibility_level = 1, - repo_name = "build_bazel_apple_support", -) - -bazel_dep(name = "bazel_skylib", version = "1.3.0") -bazel_dep(name = "platforms", version = "0.0.9") - -bazel_dep( - name = "stardoc", - version = "0.6.2", - dev_dependency = True, - repo_name = "io_bazel_stardoc", -) diff --git a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch deleted file mode 100644 index 59fc49ec7b37..000000000000 --- a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/patches/module.patch +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/MODULE.bazel b/MODULE.bazel -index 6b06c3b..99bc7c6 100644 ---- a/MODULE.bazel -+++ b/MODULE.bazel -@@ -1,6 +1,6 @@ - module( - name = "apple_support", -- version = "0", -+ version = "1.15.1-codeql.1", - bazel_compatibility = [">=6.0.0"], - compatibility_level = 1, - repo_name = "build_bazel_apple_support", -@@ -16,7 +16,3 @@ bazel_dep( - repo_name = "io_bazel_stardoc", - ) - --apple_cc_configure = use_extension("//crosstool:setup.bzl", "apple_cc_configure_extension") --use_repo(apple_cc_configure, "local_config_apple_cc", "local_config_apple_cc_toolchains") -- --register_toolchains("@local_config_apple_cc_toolchains//:all") diff --git a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json b/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json deleted file mode 100644 index 57a1f2137fea..000000000000 --- a/misc/bazel/registry/modules/apple_support/1.15.1-codeql.1/source.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "integrity": "sha256-xLsrc2fEhDgjAK7nW+WYuS+EeJb7MbvSLzojRq32aoA=", - "strip_prefix": "", - "url": "https://github.com/bazelbuild/apple_support/releases/download/1.15.1/apple_support.1.15.1.tar.gz", - "patches": { - "module.patch": "sha256-K06B2W9t6nKcU8S5u6cWeNIdw/vGWWKAoJdGiI8CSS0=" - }, - "patch_strip": 1 -} diff --git a/misc/bazel/registry/modules/apple_support/metadata.json b/misc/bazel/registry/modules/apple_support/metadata.json deleted file mode 100644 index 8a0e54c9abd9..000000000000 --- a/misc/bazel/registry/modules/apple_support/metadata.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "versions": [ - "1.15.1-codeql.1" - ] -} From 12b9b805e2f40fd72bf30c0c8178c6b04e647891 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 12:37:47 +0200 Subject: [PATCH 24/41] Go: revert changes to `make` and CI to postpone them in a separate PR --- .github/workflows/go-tests-other-os.yml | 65 ++++++++++++++++-- .github/workflows/go-tests.yml | 51 +++++++++++++- go/Makefile | 89 +++++++++++++++++++++++-- 3 files changed, 191 insertions(+), 14 deletions(-) diff --git a/.github/workflows/go-tests-other-os.yml b/.github/workflows/go-tests-other-os.yml index 9915b0869db7..ded53f868b70 100644 --- a/.github/workflows/go-tests-other-os.yml +++ b/.github/workflows/go-tests-other-os.yml @@ -7,6 +7,8 @@ on: - .github/workflows/go-tests-other-os.yml - .github/actions/** - codeql-workspace.yml +env: + GO_VERSION: '~1.22.0' permissions: contents: read @@ -16,17 +18,72 @@ jobs: name: Test MacOS runs-on: macos-latest steps: + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + id: go + - name: Check out code uses: actions/checkout@v4 - - name: Run tests - uses: ./go/actions/test + + - name: Set up CodeQL CLI + uses: ./.github/actions/fetch-codeql + + - name: Enable problem matchers in repository + shell: bash + run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' + + - name: Build + run: | + cd go + make + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + - name: Test + run: | + cd go + make test cache="${{ steps.query-cache.outputs.cache-dir }}" test-win: if: github.repository_owner == 'github' name: Test Windows runs-on: windows-latest-xl steps: + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + id: go + - name: Check out code uses: actions/checkout@v4 - - name: Run tests - uses: ./go/actions/test + + - name: Set up CodeQL CLI + uses: ./.github/actions/fetch-codeql + + - name: Enable problem matchers in repository + shell: bash + run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' + + - name: Build + run: | + cd go + make + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + + - name: Test + run: | + cd go + make test cache="${{ steps.query-cache.outputs.cache-dir }}" diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 63e2b7c49740..6d9cac5dae91 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -16,6 +16,9 @@ on: - .github/actions/** - codeql-workspace.yml +env: + GO_VERSION: '~1.22.0' + permissions: contents: read @@ -25,9 +28,51 @@ jobs: name: Test Linux (Ubuntu) runs-on: ubuntu-latest-xl steps: + - name: Set up Go ${{ env.GO_VERSION }} + uses: actions/setup-go@v5 + with: + go-version: ${{ env.GO_VERSION }} + cache: false + id: go + - name: Check out code uses: actions/checkout@v4 - - name: Run tests - uses: ./go/actions/test + + - name: Set up CodeQL CLI + uses: ./.github/actions/fetch-codeql + + - name: Enable problem matchers in repository + shell: bash + run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' + + - name: Build + run: | + cd go + make + + - name: Check that all Go code is autoformatted + run: | + cd go + make check-formatting + + - name: Compile qhelp files to markdown + run: | + cd go + env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown + + - name: Upload qhelp markdown + uses: actions/upload-artifact@v3 with: - run-code-checks: true + name: qhelp-markdown + path: go/qhelp-out/**/*.md + + - name: Cache compilation cache + id: query-cache + uses: ./.github/actions/cache-query-compilation + with: + key: go-qltest + + - name: Test + run: | + cd go + make test cache="${{ steps.query-cache.outputs.cache-dir }}" diff --git a/go/Makefile b/go/Makefile index 3ebeb69fcaf1..d0289a093a52 100644 --- a/go/Makefile +++ b/go/Makefile @@ -1,10 +1,30 @@ -all: gen extractor +all: extractor ql/lib/go.dbscheme + +ifeq ($(OS),Windows_NT) +EXE = .exe +CODEQL_PLATFORM = win64 +else +EXE = +UNAME_S := $(shell uname -s) +ifeq ($(UNAME_S),Linux) +CODEQL_PLATFORM = linux64 +endif +ifeq ($(UNAME_S),Darwin) +CODEQL_PLATFORM = osx64 +endif +endif + +CODEQL_TOOLS = $(addprefix codeql-tools/,autobuild.cmd autobuild.sh baseline-config-empty.json baseline-config-vendor.json configure-baseline.cmd configure-baseline.sh identify-environment.cmd identify-environment.sh index.cmd index.sh pre-finalize.cmd pre-finalize.sh tracing-config.lua) EXTRACTOR_PACK_OUT = build/codeql-extractor-go -.PHONY: extractor gen clean autoformat check-formatting +BINARIES = go-extractor go-tokenizer go-autobuilder go-build-runner go-bootstrap go-gen-dbscheme + +.PHONY: tools tools-codeql tools-codeql-full clean autoformat \ + tools-linux64 tools-osx64 tools-win64 check-formatting clean: + rm -rf tools/bin tools/linux64 tools/osx64 tools/win64 tools/net tools/opencsv rm -rf $(EXTRACTOR_PACK_OUT) build/stats build/testdb autoformat: @@ -27,11 +47,66 @@ endif qhelp-to-markdown: scripts/qhelp-to-markdown.sh ql/src "$(QHELP_OUT_DIR)" -extractor: - bazel run :create-extractor-pack +tools: tools-codeql tools/tokenizer.jar + +.PHONY: $(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))) +$(addsuffix $(EXE),$(addprefix tools/bin/,$(BINARIES))): + go build -C extractor -mod=vendor -o ../$@ ./cli/$(basename $(@F)) + +tools-codeql: tools-$(CODEQL_PLATFORM) + +tools-codeql-full: tools-linux64 tools-osx64 tools-win64 + +tools-linux64: $(addprefix tools/linux64/,$(BINARIES)) + +.PHONY: $(addprefix tools/linux64/,$(BINARIES)) +$(addprefix tools/linux64/,$(BINARIES)): + GOOS=linux GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@ ./cli/$(@F) + +tools-osx64: $(addprefix tools/osx64/,$(BINARIES)) + +.PHONY: $(addprefix tools/osx64/,$(BINARIES)) +$(addprefix tools/osx64/,$(BINARIES)): + GOOS=darwin GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@.amd64 ./cli/$(@F) + GOOS=darwin GOARCH=arm64 go build -C extractor -mod=vendor -o ../$@.arm64 ./cli/$(@F) + lipo -create $@.amd64 $@.arm64 -output $@ + rm $@.amd64 $@.arm64 + +tools-win64: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) + +.PHONY: $(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))) +$(addsuffix .exe,$(addprefix tools/win64/,$(BINARIES))): + env GOOS=windows GOARCH=amd64 go build -C extractor -mod=vendor -o ../$@ ./cli/$(basename $(@F)) + +.PHONY: extractor-common extractor extractor-full +extractor-common: codeql-extractor.yml LICENSE ql/lib/go.dbscheme \ + tools/tokenizer.jar $(CODEQL_TOOLS) + rm -rf $(EXTRACTOR_PACK_OUT) + mkdir -p $(EXTRACTOR_PACK_OUT) + cp codeql-extractor.yml LICENSE ql/lib/go.dbscheme ql/lib/go.dbscheme.stats $(EXTRACTOR_PACK_OUT) + mkdir $(EXTRACTOR_PACK_OUT)/tools + cp -r tools/tokenizer.jar $(CODEQL_TOOLS) $(EXTRACTOR_PACK_OUT)/tools + cp -r downgrades $(EXTRACTOR_PACK_OUT) + +extractor: extractor-common tools-codeql + cp -r tools/$(CODEQL_PLATFORM) $(EXTRACTOR_PACK_OUT)/tools + +extractor-full: extractor-common tools-codeql-full + cp -r $(addprefix tools/,linux64 osx64 win64) $(EXTRACTOR_PACK_OUT)/tools + +tools/tokenizer.jar: tools/net/sourceforge/pmd/cpd/GoLanguage.class + jar cf $@ -C tools net + jar uf $@ -C tools opencsv + +tools/net/sourceforge/pmd/cpd/GoLanguage.class: extractor/net/sourceforge/pmd/cpd/GoLanguage.java + javac -cp extractor -d tools $< + rm tools/net/sourceforge/pmd/cpd/AbstractLanguage.class + rm tools/net/sourceforge/pmd/cpd/SourceCode.class + rm tools/net/sourceforge/pmd/cpd/TokenEntry.class + rm tools/net/sourceforge/pmd/cpd/Tokenizer.class -gen: - bazel run :gen +ql/lib/go.dbscheme: tools/$(CODEQL_PLATFORM)/go-gen-dbscheme$(EXE) + $< $@ build/stats/src.stamp: mkdir -p $(@D)/src @@ -48,7 +123,7 @@ test: all build/testdb/check-upgrade-path codeql test run -j0 ql/test --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache) # use GOOS=linux because GOOS=darwin GOARCH=386 is no longer supported env GOOS=linux GOARCH=386 codeql$(EXE) test run -j0 ql/test/query-tests/Security/CWE-681 --search-path build/codeql-extractor-go --consistency-queries ql/test/consistency --compilation-cache=$(cache) - cd extractor; bazel test ... + cd extractor; go test -mod=vendor ./... bash extractor-smoke-test/test.sh || (echo "Extractor smoke test FAILED"; exit 1) .PHONY: build/testdb/check-upgrade-path From 9055d9567aa9e850e6c7836aa7086c847192d588 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 15:08:51 +0200 Subject: [PATCH 25/41] Go: remove unused action (will be re-added later) --- go/actions/test/action.yml | 80 -------------------------------------- 1 file changed, 80 deletions(-) delete mode 100644 go/actions/test/action.yml diff --git a/go/actions/test/action.yml b/go/actions/test/action.yml deleted file mode 100644 index a351c4382ac9..000000000000 --- a/go/actions/test/action.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Test go extractor -description: Run build, QL tests and optionally basic code sanity checks (formatting and generation) -inputs: - run-code-checks: - description: Whether to run formatting, code and qhelp generation checks - required: false - default: false -runs: - using: composite - steps: - - name: Get go version - shell: bash - run: | - ( - echo -n "GO_VERSION=" - bazel run @rules_go//go -- version | sed 's/go version go\(.*\) .*/\1/' - ) | tee -a "$GITHUB_ENV" - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version: ${{ env.GO_VERSION }} - cache: false - id: go - - - name: Set up CodeQL CLI - uses: ./.github/actions/fetch-codeql - - - name: Enable problem matchers in repository - shell: bash - run: 'find .github/problem-matchers -name \*.json -exec echo "::add-matcher::{}" \;' - - - name: Build - shell: bash - run: | - bazel run go:create-extractor-pack - - - name: Check that all Go code is autoformatted - if: inputs.run-code-checks == 'true' - shell: bash - run: | - cd go - make check-formatting - - - name: Check checked-in generated code - if: inputs.run-code-checks == 'true' - shell: bash - run: | - bazel run go:gen - git add . - git diff --exit-code HEAD || ( - echo "please run bazel run //go:gen" - exit 1 - ) - - - name: Compile qhelp files to markdown - if: inputs.run-code-checks == 'true' - shell: bash - run: | - cd go - env QHELP_OUT_DIR=qhelp-out make qhelp-to-markdown - - - name: Upload qhelp markdown - if: inputs.run-code-checks == 'true' - uses: actions/upload-artifact@v3 - with: - name: qhelp-markdown - path: go/qhelp-out/**/*.md - - - name: Cache compilation cache - id: query-cache - uses: ./.github/actions/cache-query-compilation - with: - key: go-qltest - - - name: Test - shell: bash - run: | - cd go - make test cache="${{ steps.query-cache.outputs.cache-dir }}" From ca2d94b297820ff93ef072419c19de31d6c6dd57 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 15:15:45 +0200 Subject: [PATCH 26/41] Fix go pattern in `.pre-commit-config.yaml` --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 055cab4ac05e..5a0ccc4938f8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,7 +31,7 @@ repos: - id: go-gen name: Check checked in generated files in go - files: go/.* + files: ^go/.* language: system entry: bazel run //go:gen pass_filenames: false From 318d954536bfca389d3c00812a84ff81f99ff64c Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 15:38:47 +0200 Subject: [PATCH 27/41] Go: make `//go:gen` not clear by default, and clean on `--force` --- go/gen.py | 36 ++++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/go/gen.py b/go/gen.py index b9394cd15b28..ebb57b2a685a 100644 --- a/go/gen.py +++ b/go/gen.py @@ -2,8 +2,18 @@ import pathlib import subprocess import os +import argparse +import shutil from python.runfiles import runfiles +def options(): + p = argparse.ArgumentParser(description="Update generated checked in files in the Go pack") + p.add_argument("--force", "-f", action="store_true", help="Regenerate all files from scratch rather than updating them") + p.add_argument("generators", nargs=3) + return p.parse_args() + +opts = options() + try: workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) except KeyError: @@ -13,20 +23,34 @@ go_extractor_dir = workspace_dir / "go" / "extractor" go_dbscheme = workspace_dir / "go" / "ql" / "lib" / "go.dbscheme" r = runfiles.Create() -go, gazelle, go_gen_dbscheme = map(r.Rlocation, sys.argv[1:]) +go, gazelle, go_gen_dbscheme = map(r.Rlocation, opts.generators) + -print("updating vendor") +if opts.force: + print("clearing vendor directory") + shutil.rmtree(go_extractor_dir / "vendor") + +existing_build_files = set(go_extractor_dir.glob("*/**/BUILD.bazel")) + +print("updating vendor directory") subprocess.check_call([go, "-C", go_extractor_dir, "work", "vendor"]) -print("clearing generated BUILD files") -for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): - build_file.unlink() +if opts.force: + print("clearing generated BUILD files") + for build_file in existing_build_files: + build_file.unlink() print("running gazelle") subprocess.check_call([gazelle]) +build_files_to_update = set(go_extractor_dir.glob("*/**/BUILD.bazel")) +if not opts.force: + build_files_to_update -= existing_build_files + # these are always refreshed + build_files_to_update.update(go_extractor_dir.glob("vendor/**/BUILD.bazel")) + print("adding header to generated BUILD files") -for build_file in go_extractor_dir.glob("*/**/BUILD.bazel"): +for build_file in build_files_to_update: contents = build_file.read_text() build_file.write_text(f"# generated running `bazel run //go/gazelle`, do not edit\n\n{contents}") From 4ae82ac2150cd2c90cb708f1e810381367fb0fa7 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 15:40:13 +0200 Subject: [PATCH 28/41] Go: add explanatory comment to `extractor` `BUILD` file --- go/extractor/BUILD.bazel | 1 + 1 file changed, 1 insertion(+) diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel index cb7bc3ac8ef2..32eaa8fda745 100644 --- a/go/extractor/BUILD.bazel +++ b/go/extractor/BUILD.bazel @@ -4,6 +4,7 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_files") # gazelle:prefix github.com/github/codeql-go/extractor # gazelle:map_kind go_binary codeql_go_binary //go:rules.bzl +# following target is kept up to date by `bazel run //go:gen`, do not edit directly go_library( name = "extractor", srcs = [ From 0bc6934bfc210d7efc2f87966ab13cae710a82a0 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 15:44:06 +0200 Subject: [PATCH 29/41] Go: rename `pkg_files` to something less confusing --- go/codeql-tools/BUILD.bazel | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/codeql-tools/BUILD.bazel b/go/codeql-tools/BUILD.bazel index 4e839b4774b5..8c3946b24ad0 100644 --- a/go/codeql-tools/BUILD.bazel +++ b/go/codeql-tools/BUILD.bazel @@ -1,13 +1,13 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") pkg_files( - name = "executables", + name = "sh-files", srcs = glob(["*.sh"]), attributes = pkg_attributes(mode = "0755"), ) pkg_files( - name = "non-executables", + name = "non-sh-files", srcs = glob( ["*"], exclude = [ @@ -20,8 +20,8 @@ pkg_files( pkg_filegroup( name = "codeql-tools", srcs = [ - ":executables", - ":non-executables", + ":non-sh-files", + ":sh-files", ], prefix = "tools", visibility = ["//go:__pkg__"], From abcd9165b4b9ded2c2b9c297502df24529f95561 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 16:08:17 +0200 Subject: [PATCH 30/41] Go: write test go runtime version in a specific file --- .github/workflows/go-tests-other-os.yml | 10 ++++------ .github/workflows/go-tests.yml | 7 ++----- go/ql/test/go.mod | 4 ---- go/test-runtime-version/go.work | 1 + 4 files changed, 7 insertions(+), 15 deletions(-) delete mode 100644 go/ql/test/go.mod create mode 100644 go/test-runtime-version/go.work diff --git a/.github/workflows/go-tests-other-os.yml b/.github/workflows/go-tests-other-os.yml index ded53f868b70..d38fd18f2728 100644 --- a/.github/workflows/go-tests-other-os.yml +++ b/.github/workflows/go-tests-other-os.yml @@ -7,8 +7,6 @@ on: - .github/workflows/go-tests-other-os.yml - .github/actions/** - codeql-workspace.yml -env: - GO_VERSION: '~1.22.0' permissions: contents: read @@ -18,10 +16,10 @@ jobs: name: Test MacOS runs-on: macos-latest steps: - - name: Set up Go ${{ env.GO_VERSION }} + - name: Set up Go uses: actions/setup-go@v5 with: - go-version: ${{ env.GO_VERSION }} + go-version-file: go/test-runtime-version/go.work cache: false id: go @@ -55,10 +53,10 @@ jobs: name: Test Windows runs-on: windows-latest-xl steps: - - name: Set up Go ${{ env.GO_VERSION }} + - name: Set up Go uses: actions/setup-go@v5 with: - go-version: ${{ env.GO_VERSION }} + go-version-file: go/test-runtime-version/go.work cache: false id: go diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 6d9cac5dae91..60182bb2c810 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -16,9 +16,6 @@ on: - .github/actions/** - codeql-workspace.yml -env: - GO_VERSION: '~1.22.0' - permissions: contents: read @@ -28,10 +25,10 @@ jobs: name: Test Linux (Ubuntu) runs-on: ubuntu-latest-xl steps: - - name: Set up Go ${{ env.GO_VERSION }} + - name: Set up Go uses: actions/setup-go@v5 with: - go-version: ${{ env.GO_VERSION }} + go-version-file: go/test-runtime-version/go.work cache: false id: go diff --git a/go/ql/test/go.mod b/go/ql/test/go.mod deleted file mode 100644 index 2420613ecee8..000000000000 --- a/go/ql/test/go.mod +++ /dev/null @@ -1,4 +0,0 @@ -module github.com/github/codeql-go/ql/test - -go 1.21 - diff --git a/go/test-runtime-version/go.work b/go/test-runtime-version/go.work new file mode 100644 index 000000000000..233b10082402 --- /dev/null +++ b/go/test-runtime-version/go.work @@ -0,0 +1 @@ +go 1.22 From 1aafc377adf9a809e59e3a66f24235e2671b1822 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 16:22:24 +0200 Subject: [PATCH 31/41] Revert "Go: write test go runtime version in a specific file" This reverts commit abcd9165b4b9ded2c2b9c297502df24529f95561. --- .github/workflows/go-tests-other-os.yml | 10 ++++++---- .github/workflows/go-tests.yml | 7 +++++-- go/ql/test/go.mod | 4 ++++ go/test-runtime-version/go.work | 1 - 4 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 go/ql/test/go.mod delete mode 100644 go/test-runtime-version/go.work diff --git a/.github/workflows/go-tests-other-os.yml b/.github/workflows/go-tests-other-os.yml index d38fd18f2728..ded53f868b70 100644 --- a/.github/workflows/go-tests-other-os.yml +++ b/.github/workflows/go-tests-other-os.yml @@ -7,6 +7,8 @@ on: - .github/workflows/go-tests-other-os.yml - .github/actions/** - codeql-workspace.yml +env: + GO_VERSION: '~1.22.0' permissions: contents: read @@ -16,10 +18,10 @@ jobs: name: Test MacOS runs-on: macos-latest steps: - - name: Set up Go + - name: Set up Go ${{ env.GO_VERSION }} uses: actions/setup-go@v5 with: - go-version-file: go/test-runtime-version/go.work + go-version: ${{ env.GO_VERSION }} cache: false id: go @@ -53,10 +55,10 @@ jobs: name: Test Windows runs-on: windows-latest-xl steps: - - name: Set up Go + - name: Set up Go ${{ env.GO_VERSION }} uses: actions/setup-go@v5 with: - go-version-file: go/test-runtime-version/go.work + go-version: ${{ env.GO_VERSION }} cache: false id: go diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 60182bb2c810..6d9cac5dae91 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -16,6 +16,9 @@ on: - .github/actions/** - codeql-workspace.yml +env: + GO_VERSION: '~1.22.0' + permissions: contents: read @@ -25,10 +28,10 @@ jobs: name: Test Linux (Ubuntu) runs-on: ubuntu-latest-xl steps: - - name: Set up Go + - name: Set up Go ${{ env.GO_VERSION }} uses: actions/setup-go@v5 with: - go-version-file: go/test-runtime-version/go.work + go-version: ${{ env.GO_VERSION }} cache: false id: go diff --git a/go/ql/test/go.mod b/go/ql/test/go.mod new file mode 100644 index 000000000000..2420613ecee8 --- /dev/null +++ b/go/ql/test/go.mod @@ -0,0 +1,4 @@ +module github.com/github/codeql-go/ql/test + +go 1.21 + diff --git a/go/test-runtime-version/go.work b/go/test-runtime-version/go.work deleted file mode 100644 index 233b10082402..000000000000 --- a/go/test-runtime-version/go.work +++ /dev/null @@ -1 +0,0 @@ -go 1.22 From 76067cb12d88fdd761ac4dbfbd85eada52415977 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 17:02:48 +0200 Subject: [PATCH 32/41] Go: skip `X:nocoverageredesign` printing by autobuilder built with bazel --- go/extractor/cli/go-autobuilder/go-autobuilder.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/go/extractor/cli/go-autobuilder/go-autobuilder.go b/go/extractor/cli/go-autobuilder/go-autobuilder.go index 08f8477cac7b..8df8b5069189 100644 --- a/go/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/go/extractor/cli/go-autobuilder/go-autobuilder.go @@ -487,7 +487,9 @@ func extract(workspace project.GoWorkspace) bool { // Build the project and run the extractor. func installDependenciesAndBuild() { - log.Printf("Autobuilder was built with %s, environment has %s\n", runtime.Version(), toolchain.GetEnvGoVersion()) + // do not print experiments the autobuilder was built with if any, only the version + version := strings.SplitN(runtime.Version(), " ", 2)[0] + log.Printf("Autobuilder was built with %s, environment has %s\n", version, toolchain.GetEnvGoVersion()) srcdir := getSourceDir() From 00baccbc152af091936983ba111d0f5cc3b46c46 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Thu, 2 May 2024 17:08:23 +0200 Subject: [PATCH 33/41] Go: autoformat --- go/extractor/cli/go-autobuilder/go-autobuilder.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go/extractor/cli/go-autobuilder/go-autobuilder.go b/go/extractor/cli/go-autobuilder/go-autobuilder.go index 8df8b5069189..2e9731c989b8 100644 --- a/go/extractor/cli/go-autobuilder/go-autobuilder.go +++ b/go/extractor/cli/go-autobuilder/go-autobuilder.go @@ -487,8 +487,8 @@ func extract(workspace project.GoWorkspace) bool { // Build the project and run the extractor. func installDependenciesAndBuild() { - // do not print experiments the autobuilder was built with if any, only the version - version := strings.SplitN(runtime.Version(), " ", 2)[0] + // do not print experiments the autobuilder was built with if any, only the version + version := strings.SplitN(runtime.Version(), " ", 2)[0] log.Printf("Autobuilder was built with %s, environment has %s\n", version, toolchain.GetEnvGoVersion()) srcdir := getSourceDir() From 31c427e64c030e0cdaeb26b7c571e05124ba00f0 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 3 May 2024 13:04:42 +0200 Subject: [PATCH 34/41] Bazel/Go: add more explanation in `gen.py` --- go/gen.py | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/go/gen.py b/go/gen.py index ebb57b2a685a..0e9005571ea4 100644 --- a/go/gen.py +++ b/go/gen.py @@ -1,3 +1,12 @@ +""" +Update generated files related to Go in the repo. Using --force will regenerate all files from scratch. + +In particular the script will: +1. update the `vendor` dir with `go work vendor` (using a go toolchain provided by bazel) +2. update `BUILD.bazel` files using gazelle +3. update `ql/lib/go.dbscheme` using a compiled `go-dbschemegen` +""" + import sys import pathlib import subprocess @@ -7,9 +16,9 @@ from python.runfiles import runfiles def options(): - p = argparse.ArgumentParser(description="Update generated checked in files in the Go pack") + p = argparse.ArgumentParser(description="Update generated files related to Go in the repo") p.add_argument("--force", "-f", action="store_true", help="Regenerate all files from scratch rather than updating them") - p.add_argument("generators", nargs=3) + p.add_argument("executables", nargs=3, help="Internally provided executables") return p.parse_args() opts = options() @@ -23,7 +32,7 @@ def options(): go_extractor_dir = workspace_dir / "go" / "extractor" go_dbscheme = workspace_dir / "go" / "ql" / "lib" / "go.dbscheme" r = runfiles.Create() -go, gazelle, go_gen_dbscheme = map(r.Rlocation, opts.generators) +go, gazelle, go_gen_dbscheme = map(r.Rlocation, opts.executables) if opts.force: @@ -43,13 +52,16 @@ def options(): print("running gazelle") subprocess.check_call([gazelle]) +# we want to stamp all newly generated `BUILD.bazel` files with a header build_files_to_update = set(go_extractor_dir.glob("*/**/BUILD.bazel")) +# if --force, all files are new if not opts.force: + # otherwise, subtract the files that existed at the start build_files_to_update -= existing_build_files - # these are always refreshed + # but bring back the `vendor` ones, as the vendor update step always clears them build_files_to_update.update(go_extractor_dir.glob("vendor/**/BUILD.bazel")) -print("adding header to generated BUILD files") +print("adding header to newly generated BUILD files") for build_file in build_files_to_update: contents = build_file.read_text() build_file.write_text(f"# generated running `bazel run //go/gazelle`, do not edit\n\n{contents}") From 8f0b88497a89ed9cce41c659528175427d61bf85 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 3 May 2024 13:07:56 +0200 Subject: [PATCH 35/41] Bazel/Go: be more specific in `go/extractor/BUILD.bazel` comments --- go/extractor/BUILD.bazel | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel index 32eaa8fda745..297ca52c8b80 100644 --- a/go/extractor/BUILD.bazel +++ b/go/extractor/BUILD.bazel @@ -4,7 +4,7 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_files") # gazelle:prefix github.com/github/codeql-go/extractor # gazelle:map_kind go_binary codeql_go_binary //go:rules.bzl -# following target is kept up to date by `bazel run //go:gen`, do not edit directly +# the immediately following `extractor` target is kept up to date by `bazel run //go:gen`, do not edit directly go_library( name = "extractor", srcs = [ @@ -25,6 +25,7 @@ go_library( ], ) +# notice that these other targets are not generated java_library( name = "tokenizer-deps", srcs = [ From 2132c7bf967e6975827ce8a35b3511cf8297d3f2 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 3 May 2024 14:52:17 +0200 Subject: [PATCH 36/41] Bazel/Go: make `@codeql//go:gen` runnable from internal repo --- go/BUILD.bazel | 22 +++++++--------------- go/gen.py | 12 +++++++++--- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/go/BUILD.bazel b/go/BUILD.bazel index 6c80e7cb258c..e0da93475a21 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -1,13 +1,14 @@ load("@bazel_skylib//rules:native_binary.bzl", "native_binary") -load("@gazelle//:def.bzl", "gazelle") load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") load("@rules_pkg//pkg:zip.bzl", "pkg_zip") load("//:defs.bzl", "codeql_platform") -gazelle( +native_binary( name = "gazelle", - extra_args = ["go/extractor"], + src = "@gazelle//cmd/gazelle", + out = "gazelle.exe", + args = ["go/extractor"], ) _gen_binaries = [ @@ -24,9 +25,9 @@ py_binary( deps = ["@rules_python//python/runfiles"], ) -# this is an internal copy of the dbscheme to be used by extractor-pack -# this allows the extractor-pack target to be independent and up-to-date with respect to -# having run //go:gen to update the checked in files +# this is an instance of the dbscheme kept in the bazel build tree +# this allows everything that bazel builds to be up-to-date, +# independently from whether //go:gen was already run to update the checked in files genrule( name = "dbscheme", outs = ["go.dbscheme"], @@ -108,12 +109,3 @@ py_binary( main = "create_extractor_pack.py", deps = ["@rules_python//python/runfiles"], ) - -native_binary( - name = "gen-dbscheme", - src = "//go/extractor/cli/go-gen-dbscheme", - out = "go-gen-dbscheme", - args = [ - "$$BUILD_WORKSPACE_DIRECTORY/go/ql/lib/go.dbscheme", - ], -) diff --git a/go/gen.py b/go/gen.py index 0e9005571ea4..6f8d47d70962 100644 --- a/go/gen.py +++ b/go/gen.py @@ -24,12 +24,18 @@ def options(): opts = options() try: - workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) + workspace_dir = pathlib.Path(os.environ.pop('BUILD_WORKSPACE_DIRECTORY')) except KeyError: print("this should be run with bazel run", file=sys.stderr) sys.exit(1) go_extractor_dir = workspace_dir / "go" / "extractor" + +if not go_extractor_dir.exists(): + # internal repo? + workspace_dir /= "ql" + go_extractor_dir = workspace_dir / "go" / "extractor" + go_dbscheme = workspace_dir / "go" / "ql" / "lib" / "go.dbscheme" r = runfiles.Create() go, gazelle, go_gen_dbscheme = map(r.Rlocation, opts.executables) @@ -49,8 +55,8 @@ def options(): for build_file in existing_build_files: build_file.unlink() -print("running gazelle") -subprocess.check_call([gazelle]) +print("running gazelle", gazelle, go_extractor_dir) +subprocess.check_call([gazelle, "go/extractor"], cwd=workspace_dir) # we want to stamp all newly generated `BUILD.bazel` files with a header build_files_to_update = set(go_extractor_dir.glob("*/**/BUILD.bazel")) From 471303bd7ce7e45182cc586db60344f7bae86858 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 3 May 2024 14:56:17 +0200 Subject: [PATCH 37/41] Bazel/Go: remove unneeded comment --- MODULE.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MODULE.bazel b/MODULE.bazel index d069d320f07a..27479e1978f7 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -55,7 +55,7 @@ node.toolchain( use_repo(node, "nodejs", "nodejs_toolchains") go_sdk = use_extension("@rules_go//go:extensions.bzl", "go_sdk") -go_sdk.download(version = "1.22.2") # default +go_sdk.download(version = "1.22.2") register_toolchains( "@nodejs_toolchains//:all", From 17990da205995d0fcd7f052287ba9ecb15616e69 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 3 May 2024 15:58:43 +0200 Subject: [PATCH 38/41] Update go/extractor/BUILD.bazel Co-authored-by: Cornelius Riemenschneider --- go/extractor/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel index 297ca52c8b80..6047eea6860c 100644 --- a/go/extractor/BUILD.bazel +++ b/go/extractor/BUILD.bazel @@ -25,7 +25,7 @@ go_library( ], ) -# notice that these other targets are not generated +# the other targets are not generated java_library( name = "tokenizer-deps", srcs = [ From 77128de105d35ada1e379a6dea61d3939e6cab6e Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Fri, 3 May 2024 17:44:29 +0200 Subject: [PATCH 39/41] Bazel/Go: make installer work from internal repo and on windows It turns out everything that is needed for the installer to work on windows is enabling runfiles. This also requires symlinks to avoid excessive copying of files. --- .bazelrc | 3 +++ go/BUILD.bazel | 24 +++--------------------- go/create_extractor_pack.py | 24 ++++++++++-------------- 3 files changed, 16 insertions(+), 35 deletions(-) mode change 100644 => 100755 go/create_extractor_pack.py diff --git a/.bazelrc b/.bazelrc index 0a49f682da37..36111310779c 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,6 +14,9 @@ build:linux --cxxopt=-std=c++20 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor +# this requires developer mode, but is required to have pack installer functioning +common:windows --windows_enable_symlinks --enable_runfiles + common --registry=file:///%workspace%/misc/bazel/registry common --registry=https://bcr.bazel.build diff --git a/go/BUILD.bazel b/go/BUILD.bazel index e0da93475a21..4fb73a51fb69 100644 --- a/go/BUILD.bazel +++ b/go/BUILD.bazel @@ -1,7 +1,6 @@ load("@bazel_skylib//rules:native_binary.bzl", "native_binary") load("@rules_pkg//pkg:install.bzl", "pkg_install") load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_filegroup", "pkg_files") -load("@rules_pkg//pkg:zip.bzl", "pkg_zip") load("//:defs.bzl", "codeql_platform") native_binary( @@ -81,31 +80,14 @@ pkg_filegroup( ) pkg_install( - name = "_extractor-pack-installer", + name = "_extractor_pack", srcs = [":extractor-pack"], ) -# rules_pkg installer is currently broken on Windows -# see https://github.com/bazelbuild/rules_pkg/issues/387 -# for now, work around it using an archive -pkg_zip( - name = "_extractor-pack-zip", - srcs = [":extractor-pack"], -) - -alias( - name = "_create-extractor-pack-arg", - actual = select({ - "@platforms//os:windows": ":_extractor-pack-zip", - "//conditions:default": ":_extractor-pack-installer", - }), -) - py_binary( name = "create-extractor-pack", srcs = ["create_extractor_pack.py"], - args = ["$(rlocationpath :_create-extractor-pack-arg)"], - data = [":_create-extractor-pack-arg"], + env = {"REPO_NAME": repo_name()}, main = "create_extractor_pack.py", - deps = ["@rules_python//python/runfiles"], + deps = ["_extractor_pack"], ) diff --git a/go/create_extractor_pack.py b/go/create_extractor_pack.py old mode 100644 new mode 100755 index a1154a777d85..4d194ab93e46 --- a/go/create_extractor_pack.py +++ b/go/create_extractor_pack.py @@ -1,26 +1,22 @@ +#!/usr/bin/env python3 import os import pathlib import shutil import sys import subprocess -import zipfile -from python.runfiles import runfiles try: workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']) except KeyError: - print("this should be run with bazel run", file=sys.stderr) - sys.exit(1) + res = subprocess.run(["bazel", "run", ":create-extractor-pack"], cwd=pathlib.Path(__file__).parent) + sys.exit(res.returncode) -dest_dir = workspace_dir / 'go' / 'build' / 'codeql-extractor-go' -installer_or_zip = pathlib.Path(runfiles.Create().Rlocation(sys.argv[1])) +from go._extractor_pack_install_script import main -shutil.rmtree(dest_dir, ignore_errors=True) +if os.environ['REPO_NAME'] == 'codeql~': + workspace_dir /= 'ql' -if installer_or_zip.suffix == '.zip': - dest_dir.mkdir() - with zipfile.ZipFile(installer_or_zip) as pack: - pack.extractall(dest_dir) -else: - os.environ['DESTDIR'] = str(dest_dir) - subprocess.check_call([installer_or_zip]) +dest_dir = workspace_dir / 'go' / 'build' / 'codeql-extractor-pack' +shutil.rmtree(dest_dir, ignore_errors=True) +os.environ['DESTDIR'] = str(dest_dir) +main(sys.argv) From 5b184c179adfcc4a5f3accf86cb617ebdf9a7275 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 6 May 2024 12:47:51 +0200 Subject: [PATCH 40/41] Bazel/Go: add some comments --- go/extractor/BUILD.bazel | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/go/extractor/BUILD.bazel b/go/extractor/BUILD.bazel index 6047eea6860c..7e576927f662 100644 --- a/go/extractor/BUILD.bazel +++ b/go/extractor/BUILD.bazel @@ -25,7 +25,9 @@ go_library( ], ) -# the other targets are not generated +# the other targets are not generated by gazelle + +# this is separate from `tokenizer-jar` below because we don't want these compiled class files in the pack java_library( name = "tokenizer-deps", srcs = [ @@ -36,6 +38,7 @@ java_library( ], ) +# we only need these compiled class files in the pack java_library( name = "tokenizer-jar", srcs = [ @@ -51,7 +54,7 @@ pkg_files( srcs = [":tokenizer-jar"], prefix = "tools", renames = { - ":tokenizer-jar": "tokenizer.jar", + ":tokenizer-jar": "tokenizer.jar", # name is `libtokenizer.jar` by default }, visibility = ["//go:__pkg__"], ) From 73df4fa920bfdef11cd8ecb094092f103bfa4850 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Mon, 6 May 2024 14:55:35 +0200 Subject: [PATCH 41/41] Go: fix Windows installation --- .bazelrc | 3 ++- go/create_extractor_pack.py | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.bazelrc b/.bazelrc index 36111310779c..c2b4d3b7f03e 100644 --- a/.bazelrc +++ b/.bazelrc @@ -15,7 +15,8 @@ build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor # this requires developer mode, but is required to have pack installer functioning -common:windows --windows_enable_symlinks --enable_runfiles +startup --windows_enable_symlinks +common --enable_runfiles common --registry=file:///%workspace%/misc/bazel/registry common --registry=https://bcr.bazel.build diff --git a/go/create_extractor_pack.py b/go/create_extractor_pack.py index 4d194ab93e46..427f0c37c36f 100755 --- a/go/create_extractor_pack.py +++ b/go/create_extractor_pack.py @@ -13,10 +13,14 @@ from go._extractor_pack_install_script import main -if os.environ['REPO_NAME'] == 'codeql~': +build_dir = workspace_dir / 'go' / 'build' + +if not build_dir.exists(): + # we probably are in the internal repo workspace_dir /= 'ql' + build_dir = workspace_dir / 'go' / 'build' -dest_dir = workspace_dir / 'go' / 'build' / 'codeql-extractor-pack' +dest_dir = build_dir / 'codeql-extractor-pack' shutil.rmtree(dest_dir, ignore_errors=True) os.environ['DESTDIR'] = str(dest_dir) main(sys.argv)