Swift: make extractor compilable with Swift 6

.gitattributes

@@ -86,4 +86,5 @@
 /misc/ripunzip/ripunzip-* filter=lfs diff=lfs merge=lfs -text
 
 # swift prebuilt resources
-/swift/third_party/resource-dir/*.zip filter=lfs diff=lfs merge=lfs -text
+/swift/third_party/resources/*.zip filter=lfs diff=lfs merge=lfs -text
+/swift/third_party/resources/*.tar.zst filter=lfs diff=lfs merge=lfs -text

.github/workflows/swift.yml

@@ -50,14 +50,14 @@ jobs:
       - uses: ./swift/actions/build-and-test
   build-and-test-linux:
     if: github.repository_owner == 'github'
-    runs-on: ubuntu-latest-xl
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - uses: ./swift/actions/build-and-test
   qltests-linux:
     if: github.repository_owner == 'github'
     needs: build-and-test-linux
-    runs-on: ubuntu-latest-xl
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - uses: ./swift/actions/run-ql-tests
@@ -109,3 +109,10 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/fetch-codeql
       - uses: ./swift/actions/database-upgrade-scripts
+  check-no-override:
+    if : github.event_name == 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - shell: bash
+        run: bazel test //swift/... --test_tag_filters=override --test_output=errors

MODULE.bazel

@@ -28,6 +28,7 @@ bazel_dep(name = "gazelle", version = "0.38.0")
 bazel_dep(name = "rules_dotnet", version = "0.16.1")
 bazel_dep(name = "googletest", version = "1.14.0.bcr.1")
 bazel_dep(name = "rules_rust", version = "0.52.2")
+bazel_dep(name = "zstd", version = "1.5.5.bcr.1")
 
 bazel_dep(name = "buildifier_prebuilt", version = "6.4.0", dev_dependency = True)
 
@@ -91,10 +92,10 @@ use_repo(
     swift_deps,
     "binlog",
     "picosha2",
-    "swift_prebuilt_darwin_x86_64",
-    "swift_prebuilt_linux",
-    "swift_toolchain_linux",
-    "swift_toolchain_macos",
+    "swift-prebuilt-linux",
+    "swift-prebuilt-macos",
+    "swift-resource-dir-linux",
+    "swift-resource-dir-macos",
 )
 
 node = use_extension("@rules_nodejs//nodejs:extensions.bzl", "node")
@@ -186,16 +187,6 @@ lfs_files(
     executable = True,
 )
 
-lfs_files(
-    name = "swift-resource-dir-linux",
-    srcs = ["//swift/third_party/resource-dir:resource-dir-linux.zip"],
-)
-
-lfs_files(
-    name = "swift-resource-dir-macos",
-    srcs = ["//swift/third_party/resource-dir:resource-dir-macos.zip"],
-)
-
 register_toolchains(
     "@nodejs_toolchains//:all",
 )

misc/bazel/lfs.bzl

@@ -77,6 +77,12 @@ def _download_lfs(repository_ctx):
         )
     repository_ctx.file("BUILD.bazel", build)
 
+    # this is for drop-in compatibility with `http_file`
+    repository_ctx.file(
+        "file/BUILD.bazel",
+        'alias(name = "file", actual = "//:%s", visibility = ["//visibility:public"])\n' % name,
+    )
+
 lfs_archive = repository_rule(
     doc = "Export the contents from an on-demand LFS archive. The corresponding path should be added to be ignored " +
           "in `.lfsconfig`.",

swift/BUILD.bazel

@@ -81,7 +81,7 @@ codeql_pack(
     zips = select({
         "@platforms//os:windows": {},
         "//conditions:default": {
-            "//swift/third_party/resource-dir": "resource-dir/{CODEQL_PLATFORM}",
+            "//swift/third_party/resources:dir": "resource-dir/{CODEQL_PLATFORM}",
         },
     }),
 )

swift/README.md

@@ -113,3 +113,8 @@ In particular for breakpoints to work you might need to setup the following remo
 ### Thread safety
 
 The extractor is single-threaded, and there was no effort to make anything in it thread-safe.
+
+### Updating the swift compiler version
+
+This can only be done with access to the internal repository at the moment. Some (incomplete) instructions are
+found [here](third_party/resources/updating.md).

swift/actions/build-and-test/action.yml

@@ -58,7 +58,7 @@ runs:
       if: ${{ github.event_name == 'pull_request' }}
       shell: bash
       run: |
-        bazel test //swift/...
+        bazel test //swift/... --test_tag_filters=-override --test_output=errors
     - name: Evict bazel cache
       if: ${{ github.event_name != 'pull_request' }}
       shell: bash

swift/extractor/BUILD.bazel

@@ -18,7 +18,7 @@ swift_cc_binary(
         "//swift/extractor/invocation",
         "//swift/extractor/remapping",
         "//swift/extractor/translators",
-        "//swift/third_party/swift-llvm-support",
+        "//swift/third_party/resources:prebuilt",
         "@absl//absl/strings",
     ],
 )

swift/extractor/SwiftExtractor.cpp

@@ -152,10 +152,10 @@ static std::unordered_set<swift::ModuleDecl*> extractDeclarations(
   }
 
   std::vector<swift::Token> comments;
-  if (primaryFile && primaryFile->getBufferID().hasValue()) {
+  if (primaryFile && primaryFile->getBufferID()) {
     auto& sourceManager = compiler.getSourceMgr();
     auto tokens = swift::tokenize(compiler.getInvocation().getLangOptions(), sourceManager,
-                                  primaryFile->getBufferID().getValue());
+                                  *primaryFile->getBufferID());
     for (auto& token : tokens) {
       if (token.getKind() == swift::tok::comment) {
         comments.push_back(token);
@@ -188,6 +188,7 @@ static std::unordered_set<std::string> collectInputFilenames(swift::CompilerInst
   std::unordered_set<std::string> sourceFiles;
   const auto& inOuts = compiler.getInvocation().getFrontendOptions().InputsAndOutputs;
   for (auto& input : inOuts.getAllInputs()) {
+    LOG_INFO("> {}", input.getFileName());
     if (input.getType() == swift::file_types::TY_Swift &&
         (!inOuts.hasPrimaryInputs() || input.isPrimary())) {
       sourceFiles.insert(input.getFileName());

swift/extractor/infra/BUILD.bazel

@@ -10,7 +10,7 @@ swift_cc_library(
         "//swift/extractor/infra/file",
         "//swift/extractor/trap",
         "//swift/logging",
-        "//swift/third_party/swift-llvm-support",
+        "//swift/third_party/resources:prebuilt",
         "@picosha2",
     ],
 )

swift/extractor/infra/SwiftTagTraits.h

@@ -175,6 +175,8 @@ MAP(swift::Expr, ExprTag)
     MAP(swift::LinearFunctionExtractOriginalExpr, LinearFunctionExtractOriginalExprTag)
     MAP(swift::LinearToDifferentiableFunctionExpr, LinearToDifferentiableFunctionExprTag)
     MAP(swift::ABISafeConversionExpr, AbiSafeConversionExprTag)  // different acronym convention
+    MAP(swift::ActorIsolationErasureExpr, void)  // TODO swift 6.0
+    MAP(swift::UnreachableExpr, void)  // TODO swift 6.0
   MAP(swift::ExplicitCastExpr, ExplicitCastExprTag)
     MAP(swift::CheckedCastExpr, CheckedCastExprTag)
       MAP(swift::ForcedCheckedCastExpr, ForcedCheckedCastExprTag)
@@ -200,7 +202,8 @@ MAP(swift::Expr, ExprTag)
   MAP(swift::ConsumeExpr, ConsumeExprTag)
   MAP(swift::MaterializePackExpr, MaterializePackExprTag)
   MAP(swift::SingleValueStmtExpr, SingleValueStmtExprTag)
-
+  MAP(swift::ExtractFunctionIsolationExpr, void)  // TODO swift 6.0
+  MAP(swift::CurrentContextIsolationExpr, void)  // TODO swift 6.0
 MAP(swift::Decl, DeclTag)
   MAP(swift::ValueDecl, ValueDeclTag)
     MAP(swift::TypeDecl, TypeDeclTag)
@@ -332,6 +335,7 @@ MAP(swift::TypeBase, TypeTag)
   MAP(swift::PackExpansionType, PackExpansionTypeTag)
   MAP(swift::PackElementType, PackElementTypeTag)
   MAP(swift::TypeVariableType, void)  // created during type checking and only used for constraint checking
+  MAP(swift::ErrorUnionType, void)  // TODO swift 6.0
   MAP(swift::SugarType, SugarTypeTag)
     MAP(swift::ParenType, ParenTypeTag)
     MAP(swift::TypeAliasType, TypeAliasTypeTag)

swift/extractor/main.cpp

@@ -20,6 +20,7 @@
 #include "swift/extractor/trap/TrapDomain.h"
 #include "swift/extractor/infra/file/Path.h"
 #include "swift/logging/SwiftAssert.h"
+#include "swift/Threading/Errors.h"
 
 using namespace std::string_literals;
 using namespace codeql::main_logger;
@@ -75,6 +76,13 @@ static void processFrontendOptions(codeql::SwiftExtractorState& state,
   }
 }
 
+static void turnOffSilVerifications(swift::SILOptions& options) {
+  options.VerifyAll = false;
+  options.VerifyExclusivity = false;
+  options.VerifyNone = true;
+  options.VerifySILOwnership = false;
+}
+
 codeql::TrapDomain invocationTrapDomain(codeql::SwiftExtractorState& state);
 
 // This is part of the swiftFrontendTool interface, we hook into the
@@ -89,6 +97,7 @@ class Observer : public swift::FrontendObserver {
     options.KeepASTContext = true;
     lockOutputSwiftModuleTraps(state, options);
     processFrontendOptions(state, options);
+    turnOffSilVerifications(invocation.getSILOptions());
   }
 
   void configuredCompiler(swift::CompilerInstance& instance) override {

swift/extractor/mangler/BUILD.bazel

@@ -8,6 +8,6 @@ swift_cc_library(
     deps = [
         "//swift/extractor/infra",
         "//swift/extractor/trap",
-        "//swift/third_party/swift-llvm-support",
+        "//swift/third_party/resources:prebuilt",
     ],
 )

swift/extractor/mangler/SwiftMangler.cpp

@@ -197,19 +197,30 @@ SwiftMangledName SwiftMangler::visitAnyFunctionType(const swift::AnyFunctionType
   auto ret = initMangled(type);
   for (const auto& param : type->getParams()) {
     ret << fetch(param.getPlainType());
-    if (param.isInOut()) {
-      ret << "_inout";
+    auto flags = param.getParameterFlags();
+    ret << "_" << getNameForParamSpecifier(flags.getOwnershipSpecifier());
+    if (flags.isIsolated()) {
+      ret << "_isolated";
     }
-    if (param.isOwned()) {
-      ret << "_owned";
+    if (flags.isAutoClosure()) {
+      ret << "_autoclosure";
     }
-    if (param.isShared()) {
-      ret << "_shared";
+    if (flags.isNonEphemeral()) {
+      ret << "_nonephermeral";
     }
-    if (param.isIsolated()) {
+    if (flags.isIsolated()) {
       ret << "_isolated";
     }
-    if (param.isVariadic()) {
+    if (flags.isSending()) {
+      ret << "_sending";
+    }
+    if (flags.isCompileTimeConst()) {
+      ret << "_compiletimeconst";
+    }
+    if (flags.isNoDerivative()) {
+      ret << "_noderivative";
+    }
+    if (flags.isVariadic()) {
       ret << "...";
     }
   }
@@ -219,6 +230,9 @@ SwiftMangledName SwiftMangler::visitAnyFunctionType(const swift::AnyFunctionType
   }
   if (type->isThrowing()) {
     ret << "_throws";
+    if (type->hasThrownError()) {
+      ret << "(" << fetch(type->getThrownError()) << ")";
+    }
   }
   if (type->isSendable()) {
     ret << "_sendable";
@@ -229,6 +243,9 @@ SwiftMangledName SwiftMangler::visitAnyFunctionType(const swift::AnyFunctionType
   if (type->hasGlobalActor()) {
     ret << "_actor" << fetch(type->getGlobalActor());
   }
+  if (type->getIsolation().isErased()) {
+    ret << "_isolated";
+  }
   // TODO: see if this needs to be used in identifying types, if not it needs to be removed from
   // type printing in the Swift compiler code
   assert(type->hasExtInfo() && "type must have ext info");

swift/extractor/remapping/BUILD.bazel

@@ -8,7 +8,7 @@ swift_cc_library(
     deps = [
         "//swift/extractor/config",
         "//swift/extractor/infra/file",
-        "//swift/third_party/swift-llvm-support",
+        "//swift/third_party/resources:prebuilt",
         "@picosha2",
     ],
 )

swift/extractor/translators/BUILD.bazel

@@ -8,6 +8,6 @@ swift_cc_library(
     deps = [
         "//swift/extractor/infra",
         "//swift/extractor/mangler",
-        "//swift/third_party/swift-llvm-support",
+        "//swift/third_party/resources:prebuilt",
     ],
 )

swift/extractor/translators/ExprTranslator.cpp

@@ -378,7 +378,7 @@ codeql::KeyPathExpr ExprTranslator::translateKeyPathExpr(const swift::KeyPathExp
     for (const auto& component : expr.getComponents()) {
       entry.components.push_back(emitKeyPathComponent(component));
     }
-    if (auto rootTypeRepr = expr.getRootType()) {
+    if (auto rootTypeRepr = expr.getExplicitRootType()) {
       auto keyPathType = expr.getType()->getAs<swift::BoundGenericClassType>();
       CODEQL_EXPECT_OR(return entry, keyPathType, "KeyPathExpr must have BoundGenericClassType");
       auto keyPathTypeArgs = keyPathType->getGenericArgs();

swift/ql/consistency-queries/UnresolvedAstNodes.ql

@@ -1,5 +1,8 @@
 import swift
 
-from AstNode n
-where n.getAPrimaryQlClass().matches("Unresolved%")
-select n
+from AstNode n, string cls
+where
+  cls = n.getAPrimaryQlClass() and
+  cls.matches("Unresolved%") and
+  not n.getLocation() instanceof UnknownLocation
+select n, cls

swift/ql/lib/codeql/swift/frameworks/StandardLibrary/Collection.qll

@@ -15,18 +15,35 @@
     row =
       [
         ";Collection;true;prefix(_:);;;Argument[-1];ReturnValue;taint",
+        ";Collection;true;prefix(_:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
         ";Collection;true;prefix(through:);;;Argument[-1];ReturnValue;taint",
+        ";Collection;true;prefix(through:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
         ";Collection;true;prefix(upTo:);;;Argument[-1];ReturnValue;taint",
+        ";Collection;true;prefix(upTo:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
         ";Collection;true;prefix(while:);;;Argument[-1];ReturnValue;taint",
+        ";Collection;true;prefix(while:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
         ";Collection;true;suffix(_:);;;Argument[-1];ReturnValue;taint",
+        ";Collection;true;suffix(_:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
         ";Collection;true;suffix(from:);;;Argument[-1];ReturnValue;taint",
+        ";Collection;true;suffix(_:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
         ";Collection;true;dropFirst(_:);;;Argument[-1];ReturnValue;taint",
+        ";Collection;true;dropFirst(_:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
         ";Collection;true;dropLast(_:);;;Argument[-1];ReturnValue;taint",
+        ";Collection;true;dropLast(_:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
+        ";Collection;true;flatMap(_:);;;Argument[-1];ReturnValue;taint",
+        ";Collection;true;flatMap(_:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
+        ";Collection;true;map(_:);;;Argument[-1];ReturnValue;taint",
+        ";Collection;true;map(_:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
         ";Collection;true;split(maxSplits:omittingEmptySubsequences:whereSeparator:);;;Argument[-1];ReturnValue;taint",
         ";Collection;true;split(separator:maxSplits:omittingEmptySubsequences:);;;Argument[-1];ReturnValue;taint",
         ";Collection;true;removeFirst();;;Argument[-1];ReturnValue;taint",
         ";Collection;true;popFirst();;;Argument[-1];ReturnValue;taint",
         ";Collection;true;randomElement();;;Argument[-1].CollectionElement;ReturnValue.OptionalSome;value",
+        ";Collection;true;randomElement(using:);;;Argument[-1].CollectionElement;ReturnValue.OptionalSome;value",
+        ";Collection;true;trimmingPrefix(_:);;;Argument[-1];ReturnValue;taint",
+        ";Collection;true;trimmingPrefix(_:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
+        ";Collection;true;trimmingPrefix(while:);;;Argument[-1];ReturnValue;taint",
+        ";Collection;true;trimmingPrefix(while:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
         ";RangeReplaceableCollection;true;init(_:);;;Argument[0];ReturnValue.CollectionElement;taint",
         ";RangeReplaceableCollection;true;init(_:);;;Argument[0].CollectionElement;ReturnValue.CollectionElement;value",
         ";RangeReplaceableCollection;true;init(repeating:count:);;;Argument[0];ReturnValue.CollectionElement;value",
@@ -38,9 +55,19 @@
         ";RangeReplaceableCollection;true;insert(_:at:);;;Argument[0];Argument[-1];taint",
         ";RangeReplaceableCollection;true;replaceSubrange(_:with:);;;Argument[1];Argument[-1];taint",
         ";RangeReplaceableCollection;true;replaceSubrange(_:with:);;;Argument[1].CollectionElement;Argument[-1].CollectionElement;value",
+        ";BidirectionalCollection;true;dropLast(_:);;;Argument[-1];ReturnValue;taint",
+        ";BidirectionalCollection;true;dropLast(_:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
         ";BidirectionalCollection;true;joined(separator:);;;Argument[-1..0];ReturnValue;taint",
+        ";BidirectionalCollection;true;joined(separator:);;;Argument[-1].CollectionElement;ReturnValue;taint",
+        ";BidirectionalCollection;true;joined(separator:);;;Argument[-1].CollectionElement.CollectionElement;ReturnValue.CollectionElement;value",
         ";BidirectionalCollection;true;last(where:);;;Argument[-1];ReturnValue;taint",
         ";BidirectionalCollection;true;popLast();;;Argument[-1];ReturnValue;taint",
+        ";BidirectionalCollection;true;reversed();;;Argument[-1];ReturnValue;taint",
+        ";BidirectionalCollection;true;reversed();;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
+        ";BidirectionalCollection;true;suffix(_:);;;Argument[-1];ReturnValue;taint",
+        ";BidirectionalCollection;true;suffix(_:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
+        ";BidirectionalCollection;true;suffix(from:);;;Argument[-1];ReturnValue;taint",
+        ";BidirectionalCollection;true;suffix(_:);;;Argument[-1].CollectionElement;ReturnValue.CollectionElement;value",
         ";MutableCollection;true;withContiguousMutableStorageIfAvailable(_:);;;Argument[-1];Argument[0].Parameter[0].CollectionElement;taint",
         ";MutableCollection;true;withContiguousMutableStorageIfAvailable(_:);;;Argument[-1].CollectionElement;Argument[0].Parameter[0].CollectionElement;value",
         ";MutableCollection;true;withContiguousMutableStorageIfAvailable(_:);;;Argument[0].Parameter[0].CollectionElement;Argument[-1].CollectionElement;value",

swift/ql/lib/codeql/swift/frameworks/StandardLibrary/Sequence.qll

@@ -45,6 +45,7 @@ private class SequenceSummaries extends SummaryModelCsv {
         ";Sequence;true;joined();;;Argument[-1].CollectionElement.CollectionElement;ReturnValue.CollectionElement;value",
         ";Sequence;true;joined(separator:);;;Argument[0..-1];ReturnValue;taint",
         ";Sequence;true;joined(separator:);;;Argument[-1].CollectionElement;ReturnValue;taint",
+        ";Sequence;true;joined(separator:);;;Argument[-1].CollectionElement.CollectionElement;ReturnValue.CollectionElement;value",
         ";Sequence;true;first(where:);;;Argument[-1].CollectionElement;ReturnValue;value",
         ";Sequence;true;withContiguousStorageIfAvailable(_:);;;Argument[-1];Argument[0].Parameter[0].CollectionElement;taint",
         ";Sequence;true;withContiguousStorageIfAvailable(_:);;;Argument[-1].CollectionElement;Argument[0].Parameter[0].CollectionElement;value",

swift/ql/lib/codeql/swift/frameworks/StandardLibrary/String.qll

@@ -173,6 +173,12 @@ private class StringFieldsInheritTaint extends TaintInheritingContent,
         namedTypeDecl.getFullName() = "CustomDebugStringConvertible" and
         fieldDecl.getName() = "debugDescription"
         or
+        namedTypeDecl.getFullName() = "CustomTestStringConvertible" and
+        fieldDecl.getName() = "testDescription"
+        or
+        namedTypeDecl.getFullName() = "CustomURLRepresentationParameterConvertible" and
+        fieldDecl.getName() = "urlRepresentationParameter"
+        or
         namedTypeDecl.getFullName() = "Substring" and
         fieldDecl.getName() = "base"
       ) and

swift/ql/test/extractor-tests/errors/CONSISTENCY/UnresolvedAstNodes.expected

@@ -1,2 +1 @@
-| file://:0:0:0:0 | ... .combine(_:) |
-| unresolved.swift:5:1:5:14 | UnresolvedSpecializeExpr |
+| unresolved.swift:5:1:5:14 | UnresolvedSpecializeExpr | UnresolvedSpecializeExpr |