diff --git a/csharp/.gitignore b/csharp/.gitignore index de2e4b63e18c..f3ea7470cb94 100644 --- a/csharp/.gitignore +++ b/csharp/.gitignore @@ -14,5 +14,4 @@ csharp.log .vscode/launch.json extractor/Semmle.Extraction.CSharp.Driver/Properties/launchSettings.json -extractor-pack paket-files/ diff --git a/csharp/tools/BUILD.bazel b/csharp/tools/BUILD.bazel index cad9d7793f01..d35f702d51ca 100644 --- a/csharp/tools/BUILD.bazel +++ b/csharp/tools/BUILD.bazel @@ -1,11 +1,11 @@ -load("@semmle_code//:dist.bzl", "pack_zip") +load("//misc/bazel:pkg.bzl", "codeql_pkg_files") -pack_zip( +codeql_pkg_files( name = "tools", - srcs = glob(["**/*"]), excludes = [ "BUILD.bazel", ], + exes = glob(["**/*"]), prefix = "tools", visibility = ["//csharp:__pkg__"], ) diff --git a/javascript/BUILD.bazel b/javascript/BUILD.bazel index 18315a098138..5081a2bcd7f3 100644 --- a/javascript/BUILD.bazel +++ b/javascript/BUILD.bazel @@ -1,6 +1,5 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_files") -load("@semmle_code//:dist.bzl", "dist") -load("@semmle_code//buildutils-internal:zipmerge.bzl", "zipmerge") +load("//misc/bazel:pkg.bzl", "codeql_pack") package(default_visibility = ["//visibility:public"]) @@ -23,8 +22,10 @@ pkg_files( strip_prefix = None, ) -dist( - name = "javascript-extractor-pack", +# We have to use a zip of the typescript parser wrapper, as it's generated by a genrule +# and we don't know a list of its output files. +codeql_pack( + name = "javascript", srcs = [ ":dbscheme-group", "//javascript/downgrades", @@ -32,17 +33,14 @@ dist( "//javascript/extractor:tools-extractor", "@semmle_code//language-packs/javascript:resources", ], - prefix = "javascript", + visibility = ["//visibility:public"], + zips = {"//javascript/extractor/lib/typescript": "tools"}, ) -# We have to zipmerge in the typescript parser wrapper, as it's generated by a genrule -# and we don't know a list of its output files. Therefore, we sidestep the -# rules_pkg tooling here, and generate the zip for the language pack manually. -zipmerge( - name = "javascript", - srcs = [ - ":javascript-extractor-pack.zip", - "//javascript/extractor/lib/typescript", - ], - out = "javascript.zip", +# TODO copy for internal repository backward compatibility +genrule( + name = "javascript.zip", + srcs = [":javascript-generic-zip"], + outs = ["javascript.zip"], + cmd = "cp $< $@", ) diff --git a/javascript/downgrades/BUILD.bazel b/javascript/downgrades/BUILD.bazel index 3d56c33c3594..97e1bbb617fa 100644 --- a/javascript/downgrades/BUILD.bazel +++ b/javascript/downgrades/BUILD.bazel @@ -1,11 +1,12 @@ -load("@semmle_code//:dist.bzl", "pack_zip") +load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix") -pack_zip( +codeql_pkg_files( name = "downgrades", srcs = glob( ["**/*"], exclude = ["BUILD.bazel"], ), prefix = "downgrades", - visibility = ["//visibility:public"], + strip_prefix = strip_prefix.from_pkg(), + visibility = ["//javascript:__pkg__"], ) diff --git a/javascript/externs/BUILD.bazel b/javascript/externs/BUILD.bazel index 233cf242b535..0f7c0300dbe5 100644 --- a/javascript/externs/BUILD.bazel +++ b/javascript/externs/BUILD.bazel @@ -1,11 +1,12 @@ -load("@semmle_code//:dist.bzl", "pack_zip") +load("//misc/bazel:pkg.bzl", "codeql_pkg_files", "strip_prefix") -pack_zip( +codeql_pkg_files( name = "externs", srcs = glob( ["**/*"], exclude = ["BUILD.bazel"], ), prefix = "tools/data/externs", - visibility = ["//visibility:public"], + strip_prefix = strip_prefix.from_pkg(), + visibility = ["//javascript:__pkg__"], ) diff --git a/javascript/extractor/lib/typescript/BUILD.bazel b/javascript/extractor/lib/typescript/BUILD.bazel index 904331e4c640..28003162673e 100644 --- a/javascript/extractor/lib/typescript/BUILD.bazel +++ b/javascript/extractor/lib/typescript/BUILD.bazel @@ -1,5 +1,3 @@ -load("@semmle_code//:common.bzl", "on_windows") - # Builds a zip file of the compiled typscript-parser-wrapper and its dependencies. genrule( name = "typescript", @@ -33,19 +31,16 @@ genrule( # Install again with only runtime deps "$$NPM install --prod", "mv node_modules build/", - "mkdir -p javascript/tools/typescript-parser-wrapper", - "mv build/* javascript/tools/typescript-parser-wrapper", + "mkdir -p typescript-parser-wrapper", + "mv build/* typescript-parser-wrapper", + "OUT=$$BAZEL_ROOT/$@", + "case $$OSTYPE in", + " cygwin|msys|win32) OUT=$$(cygpath -w $$OUT);;", + "esac", "", - ]) + on_windows( - " && ".join([ - "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$(cygpath -w $$BAZEL_ROOT/$@) $$(find javascript -name '*' -print)", - "rm -rf $$TEMP", - ]), - " && ".join([ - "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$BAZEL_ROOT/$@ $$(find javascript -name '*' -print)", - "rm -rf $$TEMP", - ]), - ), + "$$BAZEL_ROOT/$(execpath @bazel_tools//tools/zip:zipper) cC $$OUT $$(find typescript-parser-wrapper -name '*' -print)", + "rm -rf $$TEMP", + ]), tools = [ "@bazel_tools//tools/zip:zipper", "@nodejs//:node_bin", diff --git a/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java b/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java index 6a51b2e69c4d..4061f1a8651b 100644 --- a/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java +++ b/javascript/extractor/test/com/semmle/js/extractor/test/AllTests.java @@ -50,7 +50,7 @@ public static void setUp() throws Exception { entry = zis.getNextEntry(); } } - Path tsWrapper = tempDir.resolve("javascript/tools/typescript-parser-wrapper/main.js"); + Path tsWrapper = tempDir.resolve("typescript-parser-wrapper/main.js"); if (!Files.exists(tsWrapper)) { throw new RuntimeException("Could not find ts-wrapper at " + tsWrapper); } diff --git a/javascript/resources/BUILD.bazel b/javascript/resources/BUILD.bazel new file mode 100644 index 000000000000..c9ac9a734896 --- /dev/null +++ b/javascript/resources/BUILD.bazel @@ -0,0 +1,15 @@ +load("//misc/bazel:pkg.bzl", "codeql_pkg_files") + +codeql_pkg_files( + name = "resources", + srcs = glob( + ["**/*"], + exclude = [ + "tools/*.sh", + "BUILD.bazel", + ], + ), + exes = glob(["tools/*.sh"]), + strip_prefix = "", + visibility = ["//javascript:__pkg__"], +) diff --git a/javascript/resources/codeql-extractor.yml b/javascript/resources/codeql-extractor.yml new file mode 100644 index 000000000000..b82b2a965811 --- /dev/null +++ b/javascript/resources/codeql-extractor.yml @@ -0,0 +1,94 @@ +name: "javascript" +aliases: + - javascript-typescript + - typescript +display_name: "JavaScript/TypeScript" +version: 1.22.1 +column_kind: "utf16" +unicode_newlines: true +build_modes: + - none +file_coverage_languages: + - name: javascript + display_name: JavaScript + scc_languages: + - JavaScript + - name: typescript + display_name: TypeScript + scc_languages: + - TypeScript + - TypeScript Typings +github_api_languages: + - JavaScript + - TypeScript +scc_languages: + - JavaScript + - TypeScript + - TypeScript Typings +file_types: + - name: javascript + display_name: JavaScript + extensions: + - .js + - .jsx + - name: ecmascript + display_name: ECMAScript + extensions: + - .es + - .es6 + - .mjs + - name: typescript + display_name: TypeScript + extensions: + - .ts + - .tsx + - name: html + display_name: HTML + extensions: + - .html + - .htm + - .xhtm + - .xhtml + - name: vue + display_name: Vue.js component + extensions: + - .vue + - name: data + display_name: Data or configuration files + extensions: + - .json + - .yml + - .yaml + - .raml +legacy_qltest_extraction: true +options: + trap: + title: TRAP options + description: Options about how the extractor handles TRAP files + type: object + visibility: 3 + properties: + cache: + title: TRAP cache options + description: Options about how the extractor handles its TRAP cache + type: object + properties: + dir: + title: TRAP cache directory + description: The directory of the TRAP cache to use + type: string + bound: + title: TRAP cache bound + description: A soft limit (in MB) on the size of the TRAP cache + type: string + pattern: "[0-9]+" + write: + title: TRAP cache writeable + description: Whether to write to the TRAP cache as well as reading it + type: string + pattern: "(true|TRUE|false|FALSE)" + skip_types: + title: Skip type extraction for TypeScript + description: Whether to skip the extraction of types in a TypeScript application + type: string + pattern: "^(false|true)$" diff --git a/javascript/resources/tools/autobuild.cmd b/javascript/resources/tools/autobuild.cmd new file mode 100644 index 000000000000..c64898bea781 --- /dev/null +++ b/javascript/resources/tools/autobuild.cmd @@ -0,0 +1,30 @@ +@echo off +SETLOCAL EnableDelayedExpansion + +set jvm_args=-Xss16m + +rem If CODEQL_RAM is set, use half for Java and half for TS. +if NOT [%CODEQL_RAM%] == [] ( + set /a "half_ram=CODEQL_RAM/2" + set LGTM_TYPESCRIPT_RAM=%half_ram% + set jvm_args=!jvm_args! -Xmx!half_ram!m +) + +rem If CODEQL_THREADS is set, propagate via LGTM_THREADS. +if NOT [%CODEQL_THREADS%] == [] ( + set LGTM_THREADS=%CODEQL_THREADS% +) + +rem The JS autobuilder expects to find typescript modules under SEMMLE_DIST/tools. +rem They are included in the pack, but we need to set SEMMLE_DIST appropriately. +set SEMMLE_DIST=%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT% + +rem The JS autobuilder expects LGTM_SRC to be set to the source root. +set LGTM_SRC=%CD% + +type NUL && "%CODEQL_JAVA_HOME%\bin\java.exe" %jvm_args% ^ + -cp "%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT%\tools\extractor-javascript.jar" ^ + com.semmle.js.extractor.AutoBuild +exit /b %ERRORLEVEL% + +ENDLOCAL diff --git a/javascript/resources/tools/autobuild.sh b/javascript/resources/tools/autobuild.sh new file mode 100755 index 000000000000..7aaaf2878284 --- /dev/null +++ b/javascript/resources/tools/autobuild.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +set -eu + +jvm_args=-Xss16m + +# If CODEQL_RAM is set, use half for Java and half for TS. +if [ -n "${CODEQL_RAM:-}" ] ; then + half_ram="$(( CODEQL_RAM / 2 ))" + LGTM_TYPESCRIPT_RAM="$half_ram" + export LGTM_TYPESCRIPT_RAM + jvm_args="$jvm_args -Xmx${half_ram}m" +fi + +# If CODEQL_THREADS is set, propagate via LGTM_THREADS. +if [ -n "${CODEQL_THREADS:-}" ] ; then + LGTM_THREADS="$CODEQL_THREADS" + export LGTM_THREADS +fi + +# The JS autobuilder expects to find typescript modules under SEMMLE_DIST/tools. +# They are included in the pack, but we need to set SEMMLE_DIST appropriately. +# We want to word-split $jvm_args, so disable the shellcheck warning. +# shellcheck disable=SC2086 +env SEMMLE_DIST="$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT" \ + LGTM_SRC="$(pwd)" \ + "${CODEQL_JAVA_HOME}/bin/java" $jvm_args \ + -cp "$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT/tools/extractor-javascript.jar" \ + com.semmle.js.extractor.AutoBuild diff --git a/javascript/resources/tools/baseline-config.json b/javascript/resources/tools/baseline-config.json new file mode 100644 index 000000000000..e79f2b6f40bc --- /dev/null +++ b/javascript/resources/tools/baseline-config.json @@ -0,0 +1,8 @@ +{ + "paths-ignore": [ + "**/node_modules/**", + "**/bower_components/**", + "**/*.min.js", + "**/*-min.js" + ] +} diff --git a/javascript/resources/tools/configure-baseline.cmd b/javascript/resources/tools/configure-baseline.cmd new file mode 100644 index 000000000000..1398391a3794 --- /dev/null +++ b/javascript/resources/tools/configure-baseline.cmd @@ -0,0 +1,2 @@ +@echo off +type "%CODEQL_EXTRACTOR_JAVASCRIPT_ROOT%\tools\baseline-config.json" diff --git a/javascript/resources/tools/configure-baseline.sh b/javascript/resources/tools/configure-baseline.sh new file mode 100755 index 000000000000..0743da4453ca --- /dev/null +++ b/javascript/resources/tools/configure-baseline.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +cat "$CODEQL_EXTRACTOR_JAVASCRIPT_ROOT/tools/baseline-config.json" diff --git a/javascript/resources/tools/index-dir.sh b/javascript/resources/tools/index-dir.sh new file mode 100755 index 000000000000..c7ee9673b1fa --- /dev/null +++ b/javascript/resources/tools/index-dir.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +echo "Not implemented." 1>&2 +exit 1 diff --git a/ql/.gitignore b/ql/.gitignore index 5af0736b59f8..b8df693e8270 100644 --- a/ql/.gitignore +++ b/ql/.gitignore @@ -1,5 +1,4 @@ target -extractor-pack .vscode/launch.json .cache ql/test/**/*.testproj diff --git a/ruby/.gitignore b/ruby/.gitignore index f34732b9f207..f0b40d44e7a3 100644 --- a/ruby/.gitignore +++ b/ruby/.gitignore @@ -1,5 +1,4 @@ extractor/target -extractor-pack .vscode/launch.json .cache ql/test/**/*.testproj