Merge pull request #17178 from github/criemen/pytest-swift

Port Swift integration tests to pytest.

swift/ql/integration-tests/BUILD.bazel

@@ -1,7 +1,6 @@
 py_library(
     name = "utils",
     srcs = [
-        "create_database_utils.py",
         "diagnostics_test_utils.py",
     ],
     visibility = ["//swift:__subpackages__"],

swift/ql/integration-tests/autobuilder/unsupported-os/test.py

@@ -0,0 +1,6 @@
+import runs_on
+
+
+@(runs_on.linux or runs_on.windows)
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/linux-only/RegexLiteralExpr/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([
-    "swiftc -enable-bare-slash-regex regex.swift -o /dev/null",
-], lang="swift")
+
+@runs_on.linux
+def test(codeql, swift):
+    codeql.database.create(command="swiftc -enable-bare-slash-regex regex.swift -o /dev/null")

swift/ql/integration-tests/osx-only/autobuilder/failure/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/no-build-system/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/no-swift-with-spm/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/no-swift/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/no-xcode-with-spm/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/only-tests-with-spm/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/only-tests/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
-from diagnostics_test_utils import *
+import runs_on
 
-run_codeql_database_create([], lang='swift', keep_trap=True, db=None, runFunction=runUnsuccessfully)
-check_diagnostics()
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(_assert_failure=True)

swift/ql/integration-tests/osx-only/autobuilder/xcode-fails-spm-works/test.py

@@ -1,5 +1,8 @@
-from create_database_utils import *
+import runs_on
+import pytest
 
-runSuccessfully(['xcodebuild', 'clean'])
-runSuccessfully(['swift', 'package', 'clean'])
-run_codeql_database_create([], lang='swift', keep_trap=True)
+
+@runs_on.macos
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift):
+    codeql.database.create()

swift/ql/integration-tests/osx-only/canonical-case/test.py

@@ -1,5 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([
-    './build.sh',
-], lang='swift')
+
+@runs_on.macos
+def test(codeql, swift):
+    codeql.database.create(command="./build.sh")

swift/ql/integration-tests/osx-only/hello-xcode/test.py

@@ -1,9 +1,13 @@
-from create_database_utils import *
+import runs_on
+import pytest
 
-run_codeql_database_create([
-    'xcodebuild clean',
-    'xcodebuild build '
-    '-project codeql-swift-autobuild-test.xcodeproj '
-    '-target codeql-swift-autobuild-test '
-    'CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO',
-], lang='swift', keep_trap=True)
+
+@runs_on.macos
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift):
+    codeql.database.create(
+        command="xcodebuild build "
+        "-project codeql-swift-autobuild-test.xcodeproj "
+        "-target codeql-swift-autobuild-test "
+        "CODE_SIGNING_REQUIRED=NO CODE_SIGNING_ALLOWED=NO",
+    )

swift/ql/integration-tests/posix-only/cross-references/test.py

@@ -1,6 +1,8 @@
-from create_database_utils import *
+import runs_on
+import pytest
 
-run_codeql_database_create([
-    'swift package clean',
-    'swift build'
-], lang='swift')
+
+@runs_on.posix
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift):
+    codeql.database.create(command="swift build")

swift/ql/integration-tests/posix-only/deduplication/BuiltinTypes.expected

@@ -2,7 +2,6 @@
 | Builtin.Executor | BuiltinExecutorType |
 | Builtin.FPIEEE32 | BuiltinFloatType |
 | Builtin.FPIEEE64 | BuiltinFloatType |
-| Builtin.FPIEEE80 | BuiltinFloatType |
 | Builtin.Int1 | BuiltinIntegerType |
 | Builtin.Int8 | BuiltinIntegerType |
 | Builtin.Int16 | BuiltinIntegerType |

swift/ql/integration-tests/posix-only/deduplication/BuiltinTypes.ql

@@ -2,5 +2,8 @@ import swift
 
 from BuiltinType t
 // FPIEEE16 related stuff is not there on macOS
-where not t.toString().matches("%FPIEEE16")
+// FPIEEE80 is also missing on some CI runners
+where
+  not t.toString().matches("%FPIEEE16") and
+  not t.toString().matches("%FPIEEE80")
 select t, t.getPrimaryQlClasses()

swift/ql/integration-tests/posix-only/deduplication/test.py

@@ -1,6 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([
-    'swift package clean',
-    'swift build',
-], lang='swift', keep_trap=True)
+
+@runs_on.posix
+def test(codeql, swift):
+    codeql.database.create(command="swift build", keep_trap=True)

swift/ql/integration-tests/posix-only/frontend-invocations/test.py

@@ -1,22 +1,24 @@
-from create_database_utils import *
-from subprocess import check_call
 from hashlib import sha256
 from pathlib import Path
+import runs_on
+import pytest
 
-run_codeql_database_create([
-    './build.sh',
-], lang='swift',
-)
 
-with open('hashes.expected', 'w') as expected:
-    for f in sorted(Path().glob("*.swiftmodule")):
-        with open(f, 'rb') as module:
-            print(f.name, sha256(module.read()).hexdigest(), file=expected)
+@runs_on.posix
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift, expected_files):
+    codeql.database.create(command="./build.sh")
+    with open("hashes.expected", "w") as expected:
+        for f in sorted(Path().glob("*.swiftmodule")):
+            with open(f, "rb") as module:
+                print(f.name, sha256(module.read()).hexdigest(), file=expected)
 
-with open('hashes.actual', 'w') as actual:
-    hashes = [(s.name, s.resolve().name) for s in Path("test-db/working/swift-extraction-artifacts/store").iterdir()]
-    hashes.sort()
-    for module, hash in hashes:
-        print(module, hash, file=actual)
-
-check_call(['diff', '-u', 'hashes.expected', 'hashes.actual'])
+    with open("hashes.actual", "w") as actual:
+        hashes = [
+            (s.name, s.resolve().name)
+            for s in Path("test-db/working/swift-extraction-artifacts/store").iterdir()
+        ]
+        hashes.sort()
+        for module, hash in hashes:
+            print(module, hash, file=actual)
+    expected_files.add("hashes.expected")

swift/ql/integration-tests/posix-only/hello-world/test.py

@@ -1,6 +1,6 @@
-from create_database_utils import *
+import runs_on
 
-run_codeql_database_create([
-    'swift package clean',
-    'swift build'
-], lang='swift')
+
+@runs_on.posix
+def test(codeql, swift):
+    codeql.database.create(command="swift build")

swift/ql/integration-tests/posix-only/linkage-awareness/test.py

@@ -1,5 +1,8 @@
-from create_database_utils import *
+import runs_on
+import pytest
 
-run_codeql_database_create([
-    './build.sh',
-], lang='swift', keep_trap=True)
+
+@runs_on.posix
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift):
+    codeql.database.create(command="./build.sh")

swift/ql/integration-tests/posix-only/partial-modules/test.py

@@ -1,6 +1,8 @@
-from create_database_utils import *
+import runs_on
+import pytest
 
-run_codeql_database_create([
-    'swift package clean',
-    'swift build'
-], lang='swift', keep_trap=True)
+
+@runs_on.posix
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift):
+    codeql.database.create(command="swift build")

swift/ql/integration-tests/posix-only/symlinks/test.py

@@ -1,20 +1,21 @@
-from create_database_utils import *
 import os
+import runs_on
+import pytest
 
-symlinks = ['preserve/Sources/A.swift', 'resolve/Sources/A.swift']
 
-for s in symlinks:
-    try:
-        os.symlink(os.getcwd() + '/main.swift', s)
-    except:
-        pass
+@runs_on.posix
+@pytest.mark.ql_test("DB-CHECK", xfail=True)
+def test(codeql, swift):
+    symlinks = ["preserve/Sources/A.swift", "resolve/Sources/A.swift"]
 
-run_codeql_database_create([
-    'swift package clean --package-path resolve',
-    'swift build --package-path resolve',
-    'swift package clean --package-path preserve',
-    'env CODEQL_PRESERVE_SYMLINKS=true swift build --package-path preserve'
-], lang='swift', keep_trap=True)
-
-for s in symlinks:
-    os.unlink(s)
+    for s in symlinks:
+        try:
+            os.symlink(os.getcwd() + "/main.swift", s)
+        except:
+            pass
+    codeql.database.create(
+        command=[
+            "swift build --package-path resolve",
+            "env CODEQL_PRESERVE_SYMLINKS=true swift build --package-path preserve",
+        ]
+    )