Merge pull request #15834 from github/post-release-prep/codeql-cli-2.…

…16.4

Post-release preparation for codeql-cli-2.16.4

cpp/ql/lib/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.12.7
+
+### Minor Analysis Improvements
+
+* Added destructors for named objects to the intermediate representation.
+
 ## 0.12.6
 
 ### New Features

cpp/ql/lib/change-notes/2024-02-26-ir-named-destructors.md → cpp/ql/lib/change-notes/released/0.12.7.md

@@ -1,4 +1,5 @@
----
-category: minorAnalysis
----
-* Added destructors for named objects to the intermediate representation.
+## 0.12.7
+
+### Minor Analysis Improvements
+
+* Added destructors for named objects to the intermediate representation.

cpp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.12.6
+lastReleaseVersion: 0.12.7

cpp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-all
-version: 0.12.7-dev
+version: 0.12.8-dev
 groups: cpp
 dbscheme: semmlecode.cpp.dbscheme
 extractor: cpp

cpp/ql/src/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.9.6
+
+### Minor Analysis Improvements
+
+* The "non-constant format string" query (`cpp/non-constant-format`) has been converted to a `path-problem` query.
+* The new C/C++ dataflow and taint-tracking libraries (`semmle.code.cpp.dataflow.new.DataFlow` and `semmle.code.cpp.dataflow.new.TaintTracking`) now implicitly assume that dataflow and taint modelled via `DataFlowFunction` and `TaintFunction` always fully overwrite their buffers and thus act as flow barriers. As a result, many dataflow and taint-tracking queries now produce fewer false positives. To remove this assumption and go back to the previous behavior for a given model, one can override the new `isPartialWrite` predicate.
+
 ## 0.9.5
 
 ### Minor Analysis Improvements

cpp/ql/src/change-notes/2024-02-16-modelled-functions-block-flow.md → cpp/ql/src/change-notes/released/0.9.6.md

@@ -1,4 +1,6 @@
----
-category: minorAnalysis
----
+## 0.9.6
+
+### Minor Analysis Improvements
+
+* The "non-constant format string" query (`cpp/non-constant-format`) has been converted to a `path-problem` query.
 * The new C/C++ dataflow and taint-tracking libraries (`semmle.code.cpp.dataflow.new.DataFlow` and `semmle.code.cpp.dataflow.new.TaintTracking`) now implicitly assume that dataflow and taint modelled via `DataFlowFunction` and `TaintFunction` always fully overwrite their buffers and thus act as flow barriers. As a result, many dataflow and taint-tracking queries now produce fewer false positives. To remove this assumption and go back to the previous behavior for a given model, one can override the new `isPartialWrite` predicate.

cpp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.9.5
+lastReleaseVersion: 0.9.6

cpp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/cpp-queries
-version: 0.9.6-dev
+version: 0.9.7-dev
 groups:
   - cpp
   - queries

csharp/ql/campaigns/Solorigate/lib/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.7.10
+
+No user-facing changes.
+
 ## 1.7.9
 
 No user-facing changes.

csharp/ql/campaigns/Solorigate/lib/change-notes/released/1.7.10.md

@@ -0,0 +1,3 @@
+## 1.7.10
+
+No user-facing changes.

csharp/ql/campaigns/Solorigate/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.7.9
+lastReleaseVersion: 1.7.10

csharp/ql/campaigns/Solorigate/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-all
-version: 1.7.10-dev
+version: 1.7.11-dev
 groups:
   - csharp
   - solorigate

csharp/ql/campaigns/Solorigate/src/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.7.10
+
+No user-facing changes.
+
 ## 1.7.9
 
 No user-facing changes.

csharp/ql/campaigns/Solorigate/src/change-notes/released/1.7.10.md

@@ -0,0 +1,3 @@
+## 1.7.10
+
+No user-facing changes.

csharp/ql/campaigns/Solorigate/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 1.7.9
+lastReleaseVersion: 1.7.10

csharp/ql/campaigns/Solorigate/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-solorigate-queries
-version: 1.7.10-dev
+version: 1.7.11-dev
 groups:
   - csharp
   - solorigate

csharp/ql/lib/CHANGELOG.md

@@ -1,3 +1,17 @@
+## 0.8.10
+
+### Major Analysis Improvements
+
+* Improved support for flow through captured variables that properly adheres to inter-procedural control flow.
+* We no longer make use of CodeQL database stats, which may affect join-orders in custom queries. It is therefore recommended to test performance of custom queries after upgrading to this version.
+
+### Minor Analysis Improvements
+
+* C# 12: Add QL library support (`ExperimentalAttribute`) for the experimental attribute.
+* C# 12: Add extractor and QL library support for `ref readonly` parameters.
+* C#: The table `expr_compiler_generated` has been deleted and its content has been added to `compiler_generated`.
+* Data flow via get only properties like `public object Obj { get; }` is now captured by the data flow library.
+
 ## 0.8.9
 
 ### Minor Analysis Improvements

csharp/ql/lib/change-notes/released/0.8.10.md

@@ -0,0 +1,13 @@
+## 0.8.10
+
+### Major Analysis Improvements
+
+* Improved support for flow through captured variables that properly adheres to inter-procedural control flow.
+* We no longer make use of CodeQL database stats, which may affect join-orders in custom queries. It is therefore recommended to test performance of custom queries after upgrading to this version.
+
+### Minor Analysis Improvements
+
+* C# 12: Add QL library support (`ExperimentalAttribute`) for the experimental attribute.
+* C# 12: Add extractor and QL library support for `ref readonly` parameters.
+* C#: The table `expr_compiler_generated` has been deleted and its content has been added to `compiler_generated`.
+* Data flow via get only properties like `public object Obj { get; }` is now captured by the data flow library.

csharp/ql/lib/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.8.9
+lastReleaseVersion: 0.8.10

csharp/ql/lib/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-all
-version: 0.8.10-dev
+version: 0.8.11-dev
 groups: csharp
 dbscheme: semmlecode.csharp.dbscheme
 extractor: csharp

csharp/ql/src/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.8.10
+
+### Minor Analysis Improvements
+
+* Most data flow queries that track flow from *remote* flow sources now use the current *threat model* configuration instead. This doesn't lead to any changes in the produced alerts (as the default configuration is *remote* flow sources) unless the threat model configuration is changed. The changed queries are `cs/code-injection`, `cs/command-line-injection`, `cs/user-controlled-bypass`, `cs/count-untrusted-data-external-api`, `cs/untrusted-data-to-external-api`, `cs/ldap-injection`, `cs/log-forging`, `cs/xml/missing-validation`, `cs/redos`, `cs/regex-injection`, `cs/resource-injection`, `cs/sql-injection`, `cs/path-injection`, `cs/unsafe-deserialization-untrusted-input`, `cs/web/unvalidated-url-redirection`, `cs/xml/insecure-dtd-handling`, `cs/xml/xpath-injection`, `cs/web/xss`, and `cs/uncontrolled-format-string`.
+
 ## 0.8.9
 
 ### Minor Analysis Improvements

csharp/ql/src/change-notes/2024-02-06-threat-models.md → csharp/ql/src/change-notes/released/0.8.10.md

@@ -1,4 +1,5 @@
----
-category: minorAnalysis
----
-* Most data flow queries that track flow from *remote* flow sources now use the current *threat model* configuration instead. This doesn't lead to any changes in the produced alerts (as the default configuration is *remote* flow sources) unless the threat model configuration is changed. The changed queries are `cs/code-injection`, `cs/command-line-injection`, `cs/user-controlled-bypass`, `cs/count-untrusted-data-external-api`, `cs/untrusted-data-to-external-api`, `cs/ldap-injection`, `cs/log-forging`, `cs/xml/missing-validation`, `cs/redos`, `cs/regex-injection`, `cs/resource-injection`, `cs/sql-injection`, `cs/path-injection`, `cs/unsafe-deserialization-untrusted-input`, `cs/web/unvalidated-url-redirection`, `cs/xml/insecure-dtd-handling`, `cs/xml/xpath-injection`, `cs/web/xss`, and `cs/uncontrolled-format-string`.
+## 0.8.10
+
+### Minor Analysis Improvements
+
+* Most data flow queries that track flow from *remote* flow sources now use the current *threat model* configuration instead. This doesn't lead to any changes in the produced alerts (as the default configuration is *remote* flow sources) unless the threat model configuration is changed. The changed queries are `cs/code-injection`, `cs/command-line-injection`, `cs/user-controlled-bypass`, `cs/count-untrusted-data-external-api`, `cs/untrusted-data-to-external-api`, `cs/ldap-injection`, `cs/log-forging`, `cs/xml/missing-validation`, `cs/redos`, `cs/regex-injection`, `cs/resource-injection`, `cs/sql-injection`, `cs/path-injection`, `cs/unsafe-deserialization-untrusted-input`, `cs/web/unvalidated-url-redirection`, `cs/xml/insecure-dtd-handling`, `cs/xml/xpath-injection`, `cs/web/xss`, and `cs/uncontrolled-format-string`.

csharp/ql/src/codeql-pack.release.yml

@@ -1,2 +1,2 @@
 ---
-lastReleaseVersion: 0.8.9
+lastReleaseVersion: 0.8.10

csharp/ql/src/qlpack.yml

@@ -1,5 +1,5 @@
 name: codeql/csharp-queries
-version: 0.8.10-dev
+version: 0.8.11-dev
 groups:
   - csharp
   - queries

go/extractor/cli/go-autobuilder/go-autobuilder.go

@@ -3,7 +3,6 @@ package main
 import (
 	"fmt"
 	"log"
-	"net/url"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -56,63 +55,6 @@ Build behavior:
 	fmt.Fprintf(os.Stderr, "Usage:\n\n  %s\n", os.Args[0])
 }
 
-// Returns the import path of the package being built, or "" if it cannot be determined.
-func getImportPath() (importpath string) {
-	importpath = os.Getenv("LGTM_INDEX_IMPORT_PATH")
-	if importpath == "" {
-		repourl := os.Getenv("SEMMLE_REPO_URL")
-		if repourl == "" {
-			githubrepo := os.Getenv("GITHUB_REPOSITORY")
-			if githubrepo == "" {
-				log.Printf("Unable to determine import path, as neither LGTM_INDEX_IMPORT_PATH nor GITHUB_REPOSITORY is set\n")
-				return ""
-			} else {
-				importpath = "github.com/" + githubrepo
-			}
-		} else {
-			importpath = getImportPathFromRepoURL(repourl)
-			if importpath == "" {
-				log.Printf("Failed to determine import path from SEMMLE_REPO_URL '%s'\n", repourl)
-				return
-			}
-		}
-	}
-	log.Printf("Import path is '%s'\n", importpath)
-	return
-}
-
-// Returns the import path of the package being built from `repourl`, or "" if it cannot be
-// determined.
-func getImportPathFromRepoURL(repourl string) string {
-	// check for scp-like URL as in "[email protected]:github/codeql-go.git"
-	shorturl := regexp.MustCompile(`^([^@]+@)?([^:]+):([^/].*?)(\.git)?$`)
-	m := shorturl.FindStringSubmatch(repourl)
-	if m != nil {
-		return m[2] + "/" + m[3]
-	}
-
-	// otherwise parse as proper URL
-	u, err := url.Parse(repourl)
-	if err != nil {
-		log.Fatalf("Malformed repository URL '%s'\n", repourl)
-	}
-
-	if u.Scheme == "file" {
-		// we can't determine import paths from file paths
-		return ""
-	}
-
-	if u.Hostname() == "" || u.Path == "" {
-		return ""
-	}
-
-	host := u.Hostname()
-	path := u.Path
-	// strip off leading slashes and trailing `.git` if present
-	path = regexp.MustCompile(`^/+|\.git$`).ReplaceAllString(path, "")
-	return host + "/" + path
-}
-
 func restoreRepoLayout(fromDir string, dirEntries []string, scratchDirName string, toDir string) {
 	for _, dirEntry := range dirEntries {
 		if dirEntry != scratchDirName {
@@ -568,7 +510,7 @@ func installDependenciesAndBuild() {
 	if len(workspaces) == 1 {
 		workspace := workspaces[0]
 
-		importpath := getImportPath()
+		importpath := util.GetImportPath()
 		needGopath := getNeedGopath(workspace, importpath)
 
 		inLGTM := os.Getenv("LGTM_SRC") != "" || os.Getenv("LGTM_INDEX_NEED_GOPATH") != ""

go/extractor/project/project.go

@@ -439,8 +439,9 @@ func getBuildRoots(emitDiagnostics bool) (goWorkspaces []GoWorkspace, totalModul
 			for _, component := range components {
 				path = filepath.Join(path, component)
 
-				// Try to initialize a `go.mod` file automatically for the stray source files.
-				if !slices.Contains(goModDirs, path) {
+				// Try to initialize a `go.mod` file automatically for the stray source files if
+				// doing so would not place it in a parent directory of an existing `go.mod` file.
+				if !startsWithAnyOf(path, goModDirs) {
 					goWorkspaces = append(goWorkspaces, GoWorkspace{
 						BaseDir: path,
 						DepMode: GoGetNoModules,
@@ -477,6 +478,16 @@ func getBuildRoots(emitDiagnostics bool) (goWorkspaces []GoWorkspace, totalModul
 	return
 }
 
+// Determines whether `str` starts with any of `prefixes`.
+func startsWithAnyOf(str string, prefixes []string) bool {
+	for _, prefix := range prefixes {
+		if relPath, err := filepath.Rel(str, prefix); err == nil && !strings.HasPrefix(relPath, "..") {
+			return true
+		}
+	}
+	return false
+}
+
 // Finds Go workspaces in the current working directory.
 func GetWorkspaceInfo(emitDiagnostics bool) []GoWorkspace {
 	bazelPaths := slices.Concat(

go/extractor/project/project_test.go

@@ -0,0 +1,27 @@
+package project
+
+import (
+	"path/filepath"
+	"testing"
+)
+
+func testStartsWithAnyOf(t *testing.T, path string, prefix string, expectation bool) {
+	result := startsWithAnyOf(path, []string{prefix})
+	if result != expectation {
+		t.Errorf("Expected startsWithAnyOf(%s, %s) to be %t, but it is %t.", path, prefix, expectation, result)
+	}
+}
+
+func TestStartsWithAnyOf(t *testing.T) {
+	testStartsWithAnyOf(t, ".", ".", true)
+	testStartsWithAnyOf(t, ".", "dir", true)
+	testStartsWithAnyOf(t, ".", filepath.Join("foo", "bar"), true)
+	testStartsWithAnyOf(t, "dir", "dir", true)
+	testStartsWithAnyOf(t, "foo", filepath.Join("foo", "bar"), true)
+	testStartsWithAnyOf(t, filepath.Join("foo", "bar"), filepath.Join("foo", "bar"), true)
+	testStartsWithAnyOf(t, filepath.Join("foo", "bar"), filepath.Join("foo", "bar", "baz"), true)
+
+	testStartsWithAnyOf(t, filepath.Join("foo", "bar"), "foo", false)
+	testStartsWithAnyOf(t, filepath.Join("foo", "bar"), "bar", false)
+	testStartsWithAnyOf(t, filepath.Join("foo", "bar"), filepath.Join("foo", "baz"), false)
+}

go/extractor/toolchain/toolchain.go

@@ -5,8 +5,10 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 
+	"github.com/github/codeql-go/extractor/util"
 	"golang.org/x/mod/semver"
 )
 
@@ -81,7 +83,20 @@ func TidyModule(path string) *exec.Cmd {
 
 // Run `go mod init` in the directory given by `path`.
 func InitModule(path string) *exec.Cmd {
-	modInit := exec.Command("go", "mod", "init", "codeql/auto-project")
+	moduleName := "codeql/auto-project"
+
+	if importpath := util.GetImportPath(); importpath != "" {
+		// This should be something like `github.com/user/repo`
+		moduleName = importpath
+
+		// If we are not initialising the new module in the root directory of the workspace,
+		// append the relative path to the module name.
+		if relPath, err := filepath.Rel(".", path); err != nil && relPath != "." {
+			moduleName = moduleName + "/" + relPath
+		}
+	}
+
+	modInit := exec.Command("go", "mod", "init", moduleName)
 	modInit.Dir = path
 	return modInit
 }