Python: Build external extractor

codeql-workspace.yml

@@ -6,6 +6,7 @@ provide:
   - "*/ql/consistency-queries/qlpack.yml"
   - "*/ql/automodel/src/qlpack.yml"
   - "*/ql/automodel/test/qlpack.yml"
+  - "python/extractor/qlpack.yml"
   - "shared/**/qlpack.yml"
   - "cpp/ql/test/query-tests/Security/CWE/CWE-190/semmle/tainted/qlpack.yml"
   - "go/ql/config/legacy-support/qlpack.yml"

python/BUILD.bazel

@@ -1,4 +1,6 @@
-load("@rules_pkg//:mappings.bzl", "pkg_filegroup", "pkg_files")
+load("@rules_pkg//pkg:mappings.bzl", "pkg_filegroup", "pkg_files")
+load("@semmle_code//:dist.bzl", "dist", "pack_zip")
+load("//:defs.bzl", "codeql_platform")
 
 package(default_visibility = ["//visibility:public"])
 
@@ -28,3 +30,32 @@ pkg_filegroup(
         "//python/downgrades",
     ],
 )
+
+pkg_files(
+    name = "codeql-extractor-yml",
+    srcs = ["codeql-extractor.yml"],
+    strip_prefix = None,
+)
+
+dist(
+    name = "extractor-generic",
+    srcs = [
+        ":codeql-extractor-yml",
+        ":dbscheme-group",
+        "//python/downgrades",
+        "//python/extractor",
+        "//python/tools",
+    ],
+    prefix = "python",
+    visibility = ["//visibility:public"],
+)
+
+pack_zip(
+    name = "extractor-arch",
+    srcs = [
+        "//python/extractor/tsg-python",
+    ],
+    package_file_name = "extractor-" + codeql_platform + ".zip",
+    prefix = "python/tools/" + codeql_platform,
+    visibility = ["//visibility:public"],
+)

python/codeql-extractor.yml

@@ -0,0 +1,46 @@
+name: "python"
+display_name: "Python"
+version: 1.22.1
+column_kind: utf32
+build_modes:
+  - none
+github_api_languages:
+  - Python
+scc_languages:
+  - Python
+file_types:
+  - name: python
+    display_name: Python sources
+    extensions:
+      - .py
+legacy_qltest_extraction: true
+options:
+  logging:
+    title: Options pertaining to logging.
+    type: object
+    properties:
+      verbosity:
+        title: Python extractor logging verbosity level.
+        description: >
+          Controls the level of verbosity of the CodeQL Python extractor.
+
+          The supported levels are (in order of increasing verbosity):
+
+            - off
+            - errors
+            - warnings
+            - info or progress
+            - debug or progress+
+            - trace or progress++
+            - progress+++
+        type: string
+        pattern: "^(off|errors|warnings|(info|progress)|(debug|progress\\+)|(trace|progress\\+\\+)|progress\\+\\+\\+)$"
+  python_executable_name:
+    title: Controls the name of the Python executable used by the Python extractor.
+    description: >
+      The Python extractor uses platform-dependent heuristics to determine the name of the Python executable to use.
+      Specifying a value for this option overrides the name of the Python executable used by the extractor.
+      Accepted values are py, python and python3.
+      Use this setting with caution, the Python extractor requires Python 3 to run.
+    type: string
+    pattern: "^(py|python|python3)$"

python/extractor/BUILD.bazel

@@ -1,4 +1,4 @@
-load("//:dist.bzl", "pack_zip")
+load("@semmle_code//:dist.bzl", "pack_zip")
 
 py_binary(
     name = "make-zips-py",
@@ -33,7 +33,7 @@ genrule(
 )
 
 pack_zip(
-    name = "extractor-python",
+    name = "extractor",
     srcs = [
         "LICENSE-PSF.md",  # because we distribute imp.py
         "convert_setup.py",

python/extractor/cli-integration-test/writing-diagnostics/test_diagnostics_output.py

@@ -1,6 +1,6 @@
 import os
 import sys
-sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "integration-tests"))
+sys.path.append(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..", "integration-tests"))
 import diagnostics_test_utils
 
 test_db = "db"

python/extractor/qlpack.yml

@@ -1,6 +1,6 @@
 name: extractor-python
 dependencies:
-  codeql/python-all: "*"
-  codeql/python-queries: "*"
+  codeql/python-all: ${workspace}
+  codeql/python-queries: ${workspace}
 extractor: python
 warnOnImplicitThis: true

python/extractor/tests/test_patterns.py

@@ -12,7 +12,7 @@ def __init__(self, name):
 
     def test(self):
         repo_dir = subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE).communicate()[0].rstrip().decode("utf-8")
-        test_file_path = os.path.abspath(os.path.join(repo_dir, "unit-tests", "files", "pattern-matching", "patterns.json"))
+        test_file_path = os.path.abspath(os.path.join(repo_dir, "..", "unit-tests", "files", "pattern-matching", "patterns.json"))
         with open(test_file_path) as test_file:
             test_patterns = json.load(test_file)
         for test_pattern in test_patterns:

python/extractor/tsg-python/BUILD.bazel

@@ -1,5 +1,5 @@
-load("@tsg_python_crate_index//:defs.bzl", "aliases", "all_crate_deps")
-load("//:common.bzl", "codeql_rust_binary")
+load("@py_deps//:defs.bzl", "aliases", "all_crate_deps")
+load("@semmle_code//:common.bzl", "codeql_rust_binary")
 
 codeql_rust_binary(
     name = "tsg-python",
@@ -12,5 +12,5 @@ codeql_rust_binary(
     visibility = ["//visibility:public"],
     deps = all_crate_deps(
         normal = True,
-    ) + ["//extractor-python/tsg-python/tree-sitter-python"],
+    ) + ["//python/extractor/tsg-python/tsp"],
 )