fix: test per runtime (RD-12064)

.github/workflows/push-actions.yml

@@ -11,7 +11,7 @@ jobs:
       matrix:
         npm-task: ['prettier:ci', 'lint', 'test:unit']
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Node
         uses: actions/setup-node@v3
         with:
@@ -21,7 +21,7 @@ jobs:
       - run: npm ci
       - run: npm run ${{ matrix.npm-task }}
       - name: Upload distro as artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ matrix.npm-task == 'test:unit' }}
         with:
           name: distro-package
@@ -32,7 +32,7 @@ jobs:
     name: List instrumentations to test
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - id: list_instrumentations
       run: |
         instrumentations_as_json_list=$(ls -1 test/instrumentations | jq --raw-input --slurp 'split("\n") | map(select(. != ""))' | jq -r tostring)
@@ -57,7 +57,7 @@ jobs:
           # see https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners for
           # runner specs, default swap space is only 2GB
           swap-size-gb: 10
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Node
         uses: actions/setup-node@v3
         with:
@@ -83,7 +83,7 @@ jobs:
           node-version: ${{ matrix.node_cimg_tag }}
           architecture: x64
       - name: Download distro
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           name: distro-package
       - run: npm i ./distro.tgz

.github/workflows/version-testing.yml

@@ -10,10 +10,10 @@ jobs:
       matrix:
         # NOTE: if these are updated, don't forget to
         # update the download-artifacts versions as well
-        node_cimg_tag: [ 14, 16, 18 ]
+        node_cimg_tag: [ 14, 16, 18, 20 ]
     name: test-untested-versions
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Node
         uses: actions/setup-node@v3
         with:
@@ -40,7 +40,7 @@ jobs:
               cp "$path" versions_artifact;
             fi
           done
-      - uses: actions/upload-artifact@v2
+      - uses: actions/upload-artifact@v4
         with:
           name: ${{ matrix.node_cimg_tag }}
           path: versions_artifact
@@ -50,27 +50,32 @@ jobs:
     name: create-pr-for-new-versions
     needs: test-untested-versions
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
           python-version: 3.7
       - run: mkdir versions_artifacts
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v4
         continue-on-error: true  # when there aren't changes the artifact won't be uploaded and this step will fail
         with:
           name: 14
           path: versions_artifacts/14
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v4
         continue-on-error: true
         with:
           name: 16
           path: versions_artifacts/16
-      - uses: actions/download-artifact@v2
+      - uses: actions/download-artifact@v4
         continue-on-error: true
         with:
           name: 18
           path: versions_artifacts/18
+      - uses: actions/download-artifact@v4
+        continue-on-error: true
+        with:
+          name: 20
+          path: versions_artifacts/20
       - run: python3 -m pip install -r ./requirements-ci.txt
         # TODO replace this with a node.js script
       - run: PYTHONPATH=${PYTHONPATH}:$(pwd)/scripts python3 -m gather_version_artifacts
@@ -95,13 +100,13 @@ jobs:
           body: ${{ env.new_versions }}
           base: main
           labels: version-testing, automerge
-          reviewers: GuyMoses,saartochner,shanishiri
+          reviewers: GuyMoses,nadav3396,shanishiri
 
   delete-old-version-testing-branches:
     runs-on: ubuntu-latest
     name: delete-old-version-testing-branches
     needs: create-pr-for-new-versions
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - run: |  # remove all the version testing branches that aren't from today
           ./scripts/delete_old_version_testing_branches.sh

README.md

@@ -276,22 +276,26 @@ trace.getActiveSpan()?.addEvent('<error-message>', {'lumigo.type': '<error-type>
 
 ## Supported packages
 
-| Instrumentation | Package | Supported Versions |
-| --- | --- | --- |
-| grpc-js | [@grpc](https://www.npmjs.com/package/@grpc) | 1.8.0~1.8.20 |
-| amqplib | [amqplib](https://www.npmjs.com/package/amqplib) | 0.9.0~0.10.3 |
-| express | [express](https://www.npmjs.com/package/express) | 4.9.0~4.18.2 |
-| fastify | [fastify](https://www.npmjs.com/package/fastify) | 3.29.0~4.24.3 |
-| ioredis | [ioredis](https://www.npmjs.com/package/ioredis) | 4.0.0~4.28.5 |
-| | | 5.0.0~5.3.2 |
-| kafkajs | [kafkajs](https://www.npmjs.com/package/kafkajs) | 2.0.0~2.2.4 |
-| mongodb | [mongodb](https://www.npmjs.com/package/mongodb) | 3.6.6~3.7.3 |
-| | | 4.0.0~4.16.0 |
-| prisma | [prisma](https://www.npmjs.com/package/prisma) | 4.2.0~4.13.0 |
-| redis | [redis](https://www.npmjs.com/package/redis) | 4.0.0~4.6.8 |
+| Instrumentation | Package | Supported Versions | | | |
+| --- | --- | :---: | :---: | :---: | :---: |
+| | | 14 | 16 | 18 | 20 |
+| grpc-js | [grpc-js](https://www.npmjs.com/package/grpc-js) | 1.8.0~1.8.20|1.8.0~1.8.20|1.8.0~1.8.20|1.8.0~1.8.20|
+| amqplib | [amqplib](https://www.npmjs.com/package/amqplib) | 0.9.0~0.10.3|0.9.0~0.10.3|0.9.0~0.10.3|0.9.0~0.10.3|
+| express | [express](https://www.npmjs.com/package/express) | 4.9.0~4.18.2|4.9.0~4.18.2|4.9.0~4.18.2|4.9.0~4.18.2|
+| fastify | [fastify](https://www.npmjs.com/package/fastify) | 3.3.0~3.29.5|3.3.0~3.29.5|3.3.0~3.29.5|3.3.0~3.29.5|
+| | | 4.0.0| 4.0.0| 4.0.0| 4.0.0|
+| | | 4.0.1~4.26.1| 4.0.1~4.26.1| 4.0.1~4.26.1| 4.0.1~4.26.1|
+| ioredis | [ioredis](https://www.npmjs.com/package/ioredis) | 4.0.0~4.28.5|4.0.0~4.28.5|4.0.0~4.28.5|4.0.0~4.28.5|
+| | | 5.0.0~5.3.2| 5.0.0~5.3.2| 5.0.0~5.3.2| 5.0.0~5.3.2|
+| kafkajs | [kafkajs](https://www.npmjs.com/package/kafkajs) | 2.0.0~2.2.4|2.0.0~2.2.4|2.0.0~2.2.4|2.0.0~2.2.4|
+| mongodb | [mongodb](https://www.npmjs.com/package/mongodb) | 4.17.0|3.6.6~3.7.3|3.6.6~3.7.3|3.6.6~3.7.3|
+| | | 6.3.0| 4.0.0~4.17.2| 4.0.0~4.17.2| 4.0.0~4.17.2|
+| | | | 5.0.0~5.9.2| 5.0.0~5.9.2| 5.0.0~5.9.2|
+| prisma | [prisma](https://www.npmjs.com/package/prisma) | 4.2.0~4.13.0|4.2.0~4.13.0|4.2.0~4.13.0|4.2.0~4.13.0|
+| redis | [redis](https://www.npmjs.com/package/redis) | 4.0.0~4.6.8|4.0.0~4.6.13|4.0.0~4.6.13|4.0.0~4.6.13|
+| | | 4.6.10~4.6.13| | | |
 
 ### Activating your Prisma client's instrumentation
-
 If you're using [Prisma](https://www.npmjs.com/package/prisma) and you would like it instrumented, the *only* thing you will need to do (aside from [activating the tracer](#tracer-activation), of course) is ensure that your schema file's `generator client` has the `tracing` preview feature enabled prior to generating the client itself.
 
 ```prisma
@@ -303,6 +307,7 @@ generator client {
 
 **NOTE**: There have been reports of a possible bug that interferes with tracing when multiple Prisma clients have been instantiated, see [Prisma issue #20779](https://github.com/prisma/prisma/issues/20779).
 
+
 ## Automated dependency reporting
 
 To provide better support and better data-driven product decisions with respect to which packages to support next, the Lumigo OpenTelemetry Distro for JS will report to Lumigo SaaS on startup the packages and their versions used in this application, together with the OpenTelemetry resource data to enable analytics in terms of which platforms use which dependencies.

jest.instrumentations.setup.js

@@ -7,6 +7,7 @@ const {
   loadPackageVersionsFromBackup,
 } = require('./scripts/tested-versions-file-utils');
 
+const runtimeVersion = parseInt(process.version.slice(1).split('.')[0]);
 const oldEnv = Object.assign({}, process.env);
 beforeEach(() => {
   process.env = { ...oldEnv };
@@ -34,7 +35,7 @@ afterAll(() => {
       versions_keys.forEach((pkg) => {
         // updated supported versions file
         const testedVersionFolder = `./src/instrumentations/${pkg}/tested_versions`;
-        const testVersionsFile = `${testedVersionFolder}/${pkg}`;
+        const testVersionsFile = `${testedVersionFolder}/${runtimeVersion}/${pkg}`;
         fs.mkdirSync(testedVersionFolder, { recursive: true });
         const versionStrings = versions[pkg].unsupported
           .map((v) => `!${v}`)

scripts/describe_supported_versions_diff.sh

@@ -11,6 +11,8 @@ git diff --name-only -- "${ROOT_DIR}/src/" | \
     sort | \
     while read -r modified_version_file; do \
         package_name=$(basename "${modified_version_file}")
+        runtime_path=$(dirname "${modified_version_file}")
+        runtime=$(basename "${runtime_path}")
         new_versions=$(git diff HEAD --no-ext-diff --unified=0 --exit-code -a --no-prefix -- ${modified_version_file} | egrep "^\+" | tail -n +2 | sed 's/\+\(.*\)/\1/' | tr '\n' ' ')
-        echo "${package_name}: ${new_versions}"
+        echo "${package_name} (${runtime}): ${new_versions}"
     done

scripts/gather_version_artifacts.py

@@ -1,102 +1,67 @@
+# call this script from the project root with python3 -m scripts.gather_version_artifacts
+
 import os
 from glob import glob
-from typing import List, Dict, Tuple
+from typing import List
 
 from tested_versions_utils import TestedVersions
 
-ARTIFACT_DIR_NAME = "versions_artifacts"
-
+ARTIFACTS_PATH = "versions_artifacts"
+INSTRUMENTATIONS_PATH = "src/instrumentations/"
 
 # Before this script runs, the job downloads the artifacts into files with the following example structure:
 #
 #   versions_artifacts/
-#       12/
-#           express: (next lines are the data inside the file)
-#               4.17.2
-#               !4.17.3
-#           mongoose:
-#               6.4.4
-#               3.9.7
-#       14/
-#           express:
-#               4.17.2
-#               !4.17.3
-#           mongoose:
-#               3.9.7
-#               6.4.4
+#       3.7/
+#           boto3: (next lines are the data inside the file)
+#               1.0.0
+#              !2.0.0
+#       3.10/
+#           fastapi:
+#               5.1.0
+#               5.2.0
 #
 # Each file contains the original supported versions and the results from the tests in the previous job.
 
+
 def main() -> None:
-    runtime_to_files = {
-        python_runtime: sorted(
-            os.listdir(os.path.join(ARTIFACT_DIR_NAME, python_runtime))
-        )
-        for python_runtime in os.listdir(ARTIFACT_DIR_NAME)
+    files_by_runtime = {
+        python_runtime: sorted(os.listdir(os.path.join(ARTIFACTS_PATH, python_runtime)))
+        for python_runtime in os.listdir(ARTIFACTS_PATH)
     }
-    print("runtime_to_files:", runtime_to_files)
-    if not any(runtime_to_files.values()):
+    print(f"new results detected: {files_by_runtime}")
+
+    if not any(files_by_runtime.values()):
         print("No files were found so nothing to update, returning")
         return
-    files_names = list(runtime_to_files.values())[0]
-    if any([files != files_names for files in runtime_to_files.values()]):
-        raise Exception("Got different files from different runtimes")
-    origin_tested_files = glob(
-        "src/instrumentations/*/tested_versions/*"
-    )
-    for instrumentation_name in files_names:
-        handle_dependency(
-            instrumentation_name, origin_tested_files, tuple(runtime_to_files)
-        )
 
+    for runtime, dependencies in files_by_runtime.items():
+        update_dependencies(dependencies, runtime)
 
-def handle_dependency(
-    instrumentation_name: str, origin_tested_files: List[str], runtimes: Tuple[str, ...]
-) -> None:
-    print("working on:", instrumentation_name)
-    origin_path = next(
-        path
-        for path in origin_tested_files
-        if path.endswith(f"tested_versions/{instrumentation_name}")
-    )
 
-    runtime_to_tested_versions = calculate_runtime_to_tested_versions_dict(
-        instrumentation_name, runtimes
-    )
-
-    for version in list(runtime_to_tested_versions.values())[0].all_versions:
-        supported = all(
-            [
-                # A version is supported only if it works in all runtimes
-                (version in runtime_to_tested_versions[runtime].supported_versions)
-                for runtime in runtimes
-            ]
+def update_dependencies(dependencies: List[str], runtime: str) -> None:
+    for instrumentation_name in dependencies:
+        print(
+            f"processing {instrumentation_name} for {runtime}...",
         )
-
-        TestedVersions.add_version_to_file(origin_path, version, supported)
-
-
-def calculate_runtime_to_tested_versions_dict(
-    instrumentation_name: str, runtimes: Tuple[str, ...]
-) -> Dict[str, TestedVersions]:
-    runtime_to_tested_versions = {
-        runtime: TestedVersions.from_file(
-            os.path.join(ARTIFACT_DIR_NAME, runtime, instrumentation_name)
+        # find all the relevant files in the instrumentations path
+        # NOTE: this handles the case where the instrumentation's tested versions are
+        #       not in its own path, eg. uvicorn
+        original_tested_versions_files = glob(
+            f"{INSTRUMENTATIONS_PATH}/*/tested_versions/{runtime}/*"
         )
-        for runtime in runtimes
-    }
-    print("runtime_to_tested_versions:", runtime_to_tested_versions)
-
-    # Sanity check that we tested the same versions in all runtimes
-    all_versions = set(list(runtime_to_tested_versions.values())[0].all_versions)
-    if any(
-        [
-            set(tested_versions.all_versions) != all_versions
-            for tested_versions in runtime_to_tested_versions.values()
-        ]
-    ):
-        raise Exception("Got different versions from different runtimes")
-    return runtime_to_tested_versions
+        for original_tested_versions_file in original_tested_versions_files:
+            if original_tested_versions_file.endswith(
+                f"tested_versions/{runtime}/{instrumentation_name}"
+            ):
+                tested_versions = TestedVersions.from_file(
+                    os.path.join(ARTIFACTS_PATH, runtime, instrumentation_name)
+                )
+                for version in tested_versions.all_versions:
+                    supported = version in tested_versions.supported_versions
+                    TestedVersions.add_version_to_file(
+                        original_tested_versions_file, version, supported
+                    )
 
 
 if __name__ == "__main__":

scripts/init_tested_versions.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+PACKAGE_NAME="$1"
+SUPPORTED_VERSIONS=(14 16 18 20)
+
+if [ -z "$PACKAGE_NAME" ]; then
+    echo "Usage: $0 <PACKAGE_NAME>"
+    exit 1
+fi
+
+base_folder="src/instrumentations/${PACKAGE_NAME}/tested_versions"
+test_folder="src/test/integration/${PACKAGE_NAME}"
+
+echo "Creating tested versions folders under ${base_folder}..."
+for version in "${SUPPORTED_VERSIONS[@]}"; do
+    echo "Creating ${version} folder..."
+    version_folder="${base_folder}/${version}"
+    mkdir -p "$version_folder"
+    touch "${version_folder}/${PACKAGE_NAME}"
+done
+
+echo ""
+echo "Creating tested versions symlink in ${test_folder}..."
+mkdir -p "$test_folder"
+pushd "$test_folder"
+    if [ -e tested_versions ]; then
+        echo "Removing old tested_versions symlink..."
+        rm tested_versions
+    fi
+    echo "linking from ${test_folder} to ${base_folder}..."
+    ln -s ../../../../${base_folder} tested_versions
+popd
+
+echo ""
+echo "Done"