diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f42af328ef..29f1281767 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,21 @@ -ARG IMAGE=ghcr.io/newrelic-experimental/pyenv-devcontainer:latest - # To target other architectures, change the --platform directive in the Dockerfile. -FROM --platform=linux/amd64 ${IMAGE} +ARG IMAGE_TAG=latest +FROM ghcr.io/newrelic/newrelic-python-agent-ci:${IMAGE_TAG} + +# Setup non-root user +USER root +ARG UID=1001 +ARG GID=${UID} +ENV HOME=/home/vscode +RUN mkdir -p ${HOME} && \ + groupadd --gid ${GID} vscode && \ + useradd --uid ${UID} --gid ${GID} --home ${HOME} vscode && \ + chown -R ${UID}:${GID} /home/vscode + +# Fix pyenv installation +RUN echo 'eval "$(pyenv init -)"' >>${HOME}/.bashrc && \ + chown -R vscode:vscode "${PYENV_ROOT}" && \ + chown -R vscode:vscode "${PIPX_HOME}" + +# Set user +USER ${UID}:${GID} diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 92a8cdee48..fbefff476e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,7 +5,7 @@ // To target other architectures, change the --platform directive in the Dockerfile. "dockerfile": "Dockerfile", "args": { - "IMAGE": "ghcr.io/newrelic-experimental/pyenv-devcontainer:latest" + "IMAGE_TAG": "latest" } }, "remoteUser": "vscode", diff --git a/.github/actions/setup-python-matrix/action.yml b/.github/actions/setup-python-matrix/action.yml deleted file mode 100644 index 299dd2e7bd..0000000000 --- a/.github/actions/setup-python-matrix/action.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: "setup-python-matrix" -description: "Sets up all versions of python required for matrix testing in this repo." -runs: - using: "composite" - steps: - - uses: actions/setup-python@v4 - with: - python-version: "pypy-3.7" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "pypy-2.7" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.7" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.8" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.10" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - architecture: x64 - - - uses: actions/setup-python@v4 - with: - python-version: "2.7" - architecture: x64 - - - name: Install Dependencies - shell: bash - run: | - python3.10 -m pip install -U pip - python3.10 -m pip install -U wheel setuptools tox 'virtualenv<20.22.0' diff --git a/.github/containers/Dockerfile b/.github/containers/Dockerfile new file mode 100644 index 0000000000..deebcd4ef7 --- /dev/null +++ b/.github/containers/Dockerfile @@ -0,0 +1,117 @@ + +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM ubuntu:20.04 + +# Install OS packages +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y \ + bash \ + build-essential \ + curl \ + expat \ + fish \ + fontconfig \ + freetds-common \ + freetds-dev \ + gcc \ + git \ + libbz2-dev \ + libcurl4-openssl-dev \ + libffi-dev \ + libgmp-dev \ + libkrb5-dev \ + liblzma-dev \ + libmpfr-dev \ + libncurses-dev \ + libpq-dev \ + libreadline-dev \ + libsqlite3-dev \ + libssl-dev \ + locales \ + make \ + odbc-postgresql \ + openssl \ + python3-dev \ + python3-pip \ + sudo \ + tzdata \ + unixodbc-dev \ + unzip \ + vim \ + wget \ + zip \ + zlib1g \ + zlib1g-dev \ + zsh && \ + rm -rf /var/lib/apt/lists/* + +# Build librdkafka from source +ARG LIBRDKAFKA_VERSION=2.1.1 +RUN cd /tmp && \ + wget https://github.com/confluentinc/librdkafka/archive/refs/tags/v${LIBRDKAFKA_VERSION}.zip -O ./librdkafka.zip && \ + unzip ./librdkafka.zip && \ + rm ./librdkafka.zip && \ + cd ./librdkafka-${LIBRDKAFKA_VERSION} && \ + ./configure && \ + make all install && \ + cd /tmp && \ + rm -rf ./librdkafka-${LIBRDKAFKA_VERSION} + +# Setup ODBC config +RUN sed -i 's|Driver=psqlodbca.so|Driver=/usr/lib/x86_64-linux-gnu/odbc/psqlodbca.so|g' /etc/odbcinst.ini && \ + sed -i 's|Driver=psqlodbcw.so|Driver=/usr/lib/x86_64-linux-gnu/odbc/psqlodbcw.so|g' /etc/odbcinst.ini && \ + sed -i 's|Setup=libodbcpsqlS.so|Setup=/usr/lib/x86_64-linux-gnu/odbc/libodbcpsqlS.so|g' /etc/odbcinst.ini + +# Set the locale +RUN locale-gen --no-purge en_US.UTF-8 +ENV LANG=en_US.UTF-8 \ LANGUAGE=en_US:en \ LC_ALL=en_US.UTF-8 +ENV TZ="Etc/UTC" +RUN ln -fs "/usr/share/zoneinfo/${TZ}" /etc/localtime && \ + dpkg-reconfigure -f noninteractive tzdata + +# Use root user +ENV HOME=/root +WORKDIR "${HOME}" + +# Install pyenv +ENV PYENV_ROOT="/usr/local/pyenv" +RUN curl https://pyenv.run/ | /bin/bash +ENV PATH="${PYENV_ROOT}/bin:${PYENV_ROOT}/shims:${PATH}" +RUN echo 'eval "$(pyenv init -)"' >>${HOME}/.bashrc && \ + pyenv update + +# Install Python +ARG PYTHON_VERSIONS="3.12 3.11 3.10 3.9 3.8 3.7 3.13 pypy3.10-7.3.17" +COPY --chown=0:0 --chmod=755 ./install-python.sh /tmp/install-python.sh +RUN /tmp/install-python.sh && \ + rm /tmp/install-python.sh + +# Install dependencies for main python installation +COPY ./requirements.txt /tmp/requirements.txt +RUN pyenv exec pip install --upgrade -r /tmp/requirements.txt && \ + rm /tmp/requirements.txt + +# Install tools with pipx in isolated environments +COPY ./requirements-tools.txt /tmp/requirements-tools.txt +ENV PIPX_HOME="/opt/pipx" +ENV PIPX_BIN_DIR="${PIPX_HOME}/bin" +ENV PATH="${PIPX_BIN_DIR}:${PATH}" +RUN mkdir -p "${PIPX_BIN_DIR}" && \ + while IFS="" read -r line || [ -n "$line" ]; do \ + pyenv exec pipx install --global "${line}"; \ + done +# Defaults to run.local, but can instead be run.latest or any other tag. +.PHONY: run.% +run.%: +# Build image if local was specified, else pull latest + @if [[ "$*" = "local" ]]; then \ + cd ${MAKEFILE_DIR} && $(MAKE) build; \ + else \ + docker pull --platform=${PLATFORM} ${IMAGE_NAME}:$*; \ + fi + @docker run --rm -it \ + --platform=${PLATFORM} \ + --mount type=bind,source="${REPO_ROOT}",target=/home/github/python-agent \ + --workdir=/home/github/python-agent \ + --add-host=host.docker.internal:host-gateway \ + -e NEW_RELIC_HOST="${NEW_RELIC_HOST}" \ + -e NEW_RELIC_LICENSE_KEY="${NEW_RELIC_LICENSE_KEY}" \ + -e NEW_RELIC_DEVELOPER_MODE="${NEW_RELIC_DEVELOPER_MODE}" \ + -e GITHUB_ACTIONS="true" \ + ${IMAGE_NAME}:$* /bin/bash + +# Ensure python versions are usable. Cannot be automatically used with PYTHON_VERSIONS_OVERRIDE. +.PHONY: test +test: build + @docker run --rm \ + --platform=${PLATFORM} \ + ghcr.io/newrelic/python-agent-ci:local \ + /bin/bash -c '\ + python3.10 --version && \ + touch tox.ini && tox --version && \ + echo "Success! Python versions installed."' diff --git a/.github/containers/install-python.sh b/.github/containers/install-python.sh new file mode 100755 index 0000000000..f9da0a0032 --- /dev/null +++ b/.github/containers/install-python.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eo pipefail + +main() { + # Coerce space separated string to array + if [[ ${#PYTHON_VERSIONS[@]} -eq 1 ]]; then + PYTHON_VERSIONS=($PYTHON_VERSIONS) + fi + + if [[ -z "${PYTHON_VERSIONS[@]}" ]]; then + echo "No python versions specified. Make sure PYTHON_VERSIONS is set." 1>&2 + exit 1 + fi + + # Find all latest pyenv supported versions for requested python versions + PYENV_VERSIONS=() + for v in "${PYTHON_VERSIONS[@]}"; do + LATEST=$(pyenv latest -k "$v" || pyenv latest -k "$v-dev") + if [[ -z "$LATEST" ]]; then + echo "Latest version could not be found for ${v}." 1>&2 + exit 1 + fi + PYENV_VERSIONS+=($LATEST) + done + + # Install each specific version + for v in "${PYENV_VERSIONS[@]}"; do + pyenv install "$v" & + done + wait + + # Set all installed versions as globally accessible + pyenv global ${PYENV_VERSIONS[@]} +} + +main diff --git a/.github/containers/requirements-tools.txt b/.github/containers/requirements-tools.txt new file mode 100644 index 0000000000..9227d30ba0 --- /dev/null +++ b/.github/containers/requirements-tools.txt @@ -0,0 +1,6 @@ +bandit +black +flake8 +flynt +isort +tox \ No newline at end of file diff --git a/.github/containers/requirements.txt b/.github/containers/requirements.txt new file mode 100644 index 0000000000..1dbfb5a1dd --- /dev/null +++ b/.github/containers/requirements.txt @@ -0,0 +1,5 @@ +pip +pipx +setuptools +virtualenv +wheel \ No newline at end of file diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000000..4182737fcf --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,21 @@ +--- +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/newrelic/packages/" + schedule: + interval: "daily" + # Disable all pull requests for dependencies + open-pull-requests-limit: 0 + - package-ecosystem: "docker" + directory: "/" + schedule: + interval: "daily" + # Disable all pull requests for dependencies + open-pull-requests-limit: 0 + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + # Disable all pull requests for dependencies + open-pull-requests-limit: 0 diff --git a/.github/scripts/retry.sh b/.github/scripts/retry.sh index 1cb17836eb..079798a72d 100755 --- a/.github/scripts/retry.sh +++ b/.github/scripts/retry.sh @@ -1,4 +1,18 @@ #!/bin/bash +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Time in seconds to backoff after the initial attempt. INITIAL_BACKOFF=10 @@ -25,4 +39,4 @@ for i in $(seq 1 $retries); do done # Exit with status code of wrapped command -exit $? +exit $result diff --git a/.github/stale.yml b/.github/stale.yml index 9d84541dbf..39e9942194 100644 --- a/.github/stale.yml +++ b/.github/stale.yml @@ -13,7 +13,7 @@ # limitations under the License. # # Number of days of inactivity before an issue becomes stale -daysUntilStale: 60 +daysUntilStale: 365 # Number of days of inactivity before a stale issue is closed # Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale. daysUntilClose: false diff --git a/.github/workflows/build-ci-image.yml b/.github/workflows/build-ci-image.yml new file mode 100644 index 0000000000..1404fa768a --- /dev/null +++ b/.github/workflows/build-ci-image.yml @@ -0,0 +1,68 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +name: Build CI Image + +on: + workflow_dispatch: # Allow manual trigger + +concurrency: + group: ${{ github.ref || github.run_id }} + cancel-in-progress: true + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # 3.3.0 + + - name: Generate Docker Metadata (Tags and Labels) + id: meta + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # 5.5.1 + with: + images: ghcr.io/${{ github.repository }}-ci + flavor: | + prefix= + suffix= + latest=false + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=schedule,pattern={{date 'YYYY-MM-DD'}} + type=sha,format=short,prefix=sha- + type=sha,format=long,prefix=sha- + + - name: Login to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@e92390c5fb421da1463c202d546fed0ec5c39f20 # 3.1.0 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and Publish Image + uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # 5.3.0 + with: + push: ${{ github.event_name != 'pull_request' }} + context: .github/containers + platforms: ${{ (format('refs/heads/{0}', github.event.repository.default_branch) == github.ref) && 'linux/amd64,linux/arm64' || 'linux/amd64' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/deploy-python.yml b/.github/workflows/deploy-python.yml index fe16ee4854..44f1c47289 100644 --- a/.github/workflows/deploy-python.yml +++ b/.github/workflows/deploy-python.yml @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +--- name: Deploy on: @@ -20,19 +20,131 @@ on: - published jobs: - deploy-linux: + build-linux-py3-legacy: runs-on: ubuntu-latest + strategy: + matrix: + wheel: + - cp37-manylinux + - cp37-musllinux steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 with: persist-credentials: false fetch-depth: 0 - name: Setup QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # 3.0.0 + + - name: Build Wheels + uses: pypa/cibuildwheel@8d945475ac4b1aac4ae08b2fd27db9917158b6ce # 2.17.0 + env: + CIBW_PLATFORM: linux + CIBW_BUILD: "${{ matrix.wheel }}*" + CIBW_ARCHS_LINUX: x86_64 aarch64 + CIBW_ENVIRONMENT: "LD_LIBRARY_PATH=/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib" + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "PYTHONPATH={project}/tests pytest {project}/tests/agent_unittests -vx" - - uses: actions/setup-python@v2 + - name: Upload Artifacts + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 + with: + name: ${{ github.job }}-${{ matrix.wheel }} + path: ./wheelhouse/*.whl + retention-days: 1 + + build-linux-py3: + runs-on: ubuntu-latest + strategy: + matrix: + wheel: + - cp38-manylinux + - cp38-musllinux + - cp39-manylinux + - cp39-musllinux + - cp310-manylinux + - cp310-musllinux + - cp311-manylinux + - cp311-musllinux + - cp312-manylinux + - cp312-musllinux + - cp313-manylinux + - cp313-musllinux + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Setup QEMU + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # 3.0.0 + + - name: Build Wheels + uses: pypa/cibuildwheel@d4a2945fcc8d13f20a1b99d461b8e844d5fc6e23 # 2.21.1 + env: + CIBW_PLATFORM: linux + CIBW_BUILD: "${{ matrix.wheel }}*" + CIBW_ARCHS_LINUX: x86_64 aarch64 + CIBW_ENVIRONMENT: "LD_LIBRARY_PATH=/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib" + CIBW_TEST_REQUIRES: pytest + CIBW_TEST_COMMAND: "PYTHONPATH={project}/tests pytest {project}/tests/agent_unittests -vx" + + - name: Upload Artifacts + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 + with: + name: ${{ github.job }}-${{ matrix.wheel }} + path: ./wheelhouse/*.whl + retention-days: 1 + + build-sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + with: + persist-credentials: false + fetch-depth: 0 + + - name: Install Dependencies + run: | + pip install -U pip + pip install -U setuptools + + - name: Build Source Package + run: | + python setup.py sdist + + - name: Prepare MD5 Hash File + run: | + tarball="$(python setup.py --fullname).tar.gz" + md5_file=${tarball}.md5 + openssl md5 -binary dist/${tarball} | xxd -p | tr -d '\n' > dist/${md5_file} + + - name: Upload Artifacts + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 + with: + name: ${{ github.job }}-sdist + path: | + ./dist/*.tar.gz + ./dist/*.tar.gz.md5 + retention-days: 1 + + deploy: + runs-on: ubuntu-latest + + needs: + - build-linux-py3-legacy + - build-linux-py3 + - build-sdist + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + with: + persist-credentials: false + fetch-depth: 0 + + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # 5.1.0 with: python-version: "3.x" architecture: x64 @@ -42,32 +154,22 @@ jobs: pip install -U pip pip install -U wheel setuptools twine - - name: Build Source Package - run: python setup.py sdist - - - name: Build Manylinux Wheels (Python 2) - uses: pypa/cibuildwheel@v1.12.0 - env: - CIBW_PLATFORM: linux - CIBW_BUILD: cp27-manylinux_x86_64 - CIBW_ARCHS: x86_64 - CIBW_ENVIRONMENT: "LD_LIBRARY_PATH=/opt/rh/=vtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib" + - name: Download Artifacts + uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # 4.1.4 + with: + path: ./artifacts/ - - name: Build Manylinux Wheels (Python 3) - uses: pypa/cibuildwheel@v2.11.1 - env: - CIBW_PLATFORM: linux - CIBW_BUILD: cp37-manylinux* cp38-manylinux* cp39-manylinux* cp310-manylinux* cp311-manylinux* - CIBW_ARCHS: x86_64 aarch64 - CIBW_ENVIRONMENT: "LD_LIBRARY_PATH=/opt/rh/devtoolset-8/root/usr/lib64:/opt/rh/devtoolset-8/root/usr/lib:/opt/rh/devtoolset-8/root/usr/lib64/dyninst:/opt/rh/devtoolset-8/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib" + - name: Unpack Artifacts + run: | + mkdir -p dist/ + mv artifacts/**/*{.whl,.tar.gz,.tar.gz.md5} dist/ - name: Upload Package to S3 run: | tarball="$(python setup.py --fullname).tar.gz" - md5_file=$(mktemp) - openssl md5 -binary dist/$tarball | xxd -p | tr -d '\n' > $md5_file - aws s3 cp $md5_file $S3_DST/${tarball}.md5 - aws s3 cp dist/$tarball $S3_DST/$tarball + md5_file=${tarball}.md5 + aws s3 cp dist/${md5_file} $S3_DST/${md5_file} + aws s3 cp dist/${tarball} $S3_DST/${tarball} env: S3_DST: s3://nr-downloads-main/python_agent/release AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -76,7 +178,7 @@ jobs: - name: Upload Package to PyPI run: | - twine upload --non-interactive dist/*.tar.gz wheelhouse/*-manylinux*.whl + twine upload --non-interactive dist/*.tar.gz dist/*.whl env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} diff --git a/.github/workflows/get-envs.py b/.github/workflows/get-envs.py index 576cbeb5c8..665387248b 100755 --- a/.github/workflows/get-envs.py +++ b/.github/workflows/get-envs.py @@ -1,4 +1,18 @@ #!/usr/bin/env python3.8 +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import fileinput import os @@ -9,7 +23,10 @@ def main(f): environments = [e.rstrip() for e in f] filtered_envs = environments[GROUP_NUMBER::TOTAL_GROUPS] - print(",".join(filtered_envs)) + joined_envs = ",".join(filtered_envs) + + assert joined_envs, f"No environments found.\nenvironments = {str(environments)}\nGROUP_NUMBER = {GROUP_NUMBER + 1}\nTOTAL_GROUPS = {TOTAL_GROUPS}" + print(joined_envs) if __name__ == "__main__": diff --git a/.github/workflows/mega-linter.yml b/.github/workflows/mega-linter.yml index cd0930507a..6bb938d434 100644 --- a/.github/workflows/mega-linter.yml +++ b/.github/workflows/mega-linter.yml @@ -25,7 +25,7 @@ jobs: steps: # Git Checkout - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 with: token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} fetch-depth: 0 @@ -35,7 +35,7 @@ jobs: id: ml # You can override Mega-Linter flavor used to have faster performances # More info at https://oxsecurity.github.io/megalinter/flavors/ - uses: oxsecurity/megalinter/flavors/python@v6 + uses: oxsecurity/megalinter/flavors/python@a7a0163b6c8ff7474a283d99a706e27483ddd80f # 7.10.0 env: # All available variables are described in documentation # https://oxsecurity.github.io/megalinter/configuration/ @@ -47,7 +47,7 @@ jobs: # Upload Mega-Linter artifacts - name: Archive production artifacts if: ${{ success() }} || ${{ failure() }} - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: Mega-Linter reports path: | @@ -58,7 +58,7 @@ jobs: - name: Create Pull Request with applied fixes id: cpr if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES_EVENT == 'all' || env.APPLY_FIXES_EVENT == github.event_name) && env.APPLY_FIXES_MODE == 'pull_request' && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && !contains(github.event.head_commit.message, 'skip fix') - uses: peter-evans/create-pull-request@v3 + uses: peter-evans/create-pull-request@c55203cfde3e5c11a452d352b4393e68b85b4533 # 6.0.3 with: token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} commit-message: "[Mega-Linter] Apply linters automatic fixes" @@ -76,7 +76,7 @@ jobs: run: sudo chown -Rc $UID .git/ - name: Commit and push applied linter fixes if: steps.ml.outputs.has_updated_sources == 1 && (env.APPLY_FIXES_EVENT == 'all' || env.APPLY_FIXES_EVENT == github.event_name) && env.APPLY_FIXES_MODE == 'commit' && github.ref != 'refs/heads/main' && (github.event_name == 'push' || github.event.pull_request.head.repo.full_name == github.repository) && !contains(github.event.head_commit.message, 'skip fix') - uses: stefanzweifel/git-auto-commit-action@v4 + uses: stefanzweifel/git-auto-commit-action@8621497c8c39c72f3e2a999a26b4ca1b5058a842 # 5.0.1 with: branch: ${{ github.event.pull_request.head.ref || github.head_ref || github.ref }} commit_message: "[Mega-Linter] Apply linters fixes" diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dc73168ebf..4bb929904c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -11,7 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - +--- name: Tests on: @@ -31,43 +31,86 @@ concurrency: jobs: # Aggregate job that provides a single check for all tests passing tests: - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: - python - elasticsearchserver07 - elasticsearchserver08 - - gearman + - firestore - grpc - #- kafka - - libcurl + - kafka - memcached - mongodb + - mssql - mysql - - postgres + - postgres16 + - postgres9 - rabbitmq - redis + - rediscluster - solr steps: - name: Success run: echo "Success!" + # Upload Trivy data + trivy: + if: success() || failure() # Does not run on cancelled workflows + runs-on: ubuntu-20.04 + needs: + - tests + + steps: + # Git Checkout + - name: Checkout Code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + with: + token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }} + fetch-depth: 0 + + - name: Run Trivy vulnerability scanner in repo mode + if: ${{ github.event_name == 'pull_request' }} + uses: aquasecurity/trivy-action@0.20.0 + with: + scan-type: 'fs' + ignore-unfixed: true + format: table + exit-code: 1 + severity: 'CRITICAL,HIGH,MEDIUM,LOW' + + - name: Run Trivy vulnerability scanner in repo mode + if: ${{ github.event_name == 'schedule' }} + uses: aquasecurity/trivy-action@0.20.0 + with: + scan-type: 'fs' + ignore-unfixed: true + format: 'sarif' + output: 'trivy-results.sarif' + severity: 'CRITICAL,HIGH,MEDIUM,LOW' + + - name: Upload Trivy scan results to GitHub Security tab + if: ${{ github.event_name == 'schedule' }} + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: 'trivy-results.sarif' + # Combine and upload coverage data coverage: if: success() || failure() # Does not run on cancelled workflows - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest needs: - tests steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v4 + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # 5.1.0 with: python-version: "3.10" architecture: x64 - name: Download Coverage Artifacts - uses: actions/download-artifact@v3 + uses: actions/download-artifact@c850b930e6ba138125429b7e5c93fc707a7f8427 # 4.1.4 with: path: ./ @@ -79,10 +122,11 @@ jobs: coverage xml - name: Upload Coverage to Codecov - uses: codecov/codecov-action@v3 + uses: codecov/codecov-action@84508663e988701840491b86de86b666e8a86bed # 4.3.0 with: files: coverage.xml fail_ci_if_error: true + token: ${{ secrets.CODECOV_TOKEN }} # Tests python: @@ -116,17 +160,29 @@ jobs: 20, ] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -138,7 +194,7 @@ jobs: PY_COLORS: 0 - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* @@ -153,17 +209,29 @@ jobs: matrix: group-number: [1] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 - steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -175,38 +243,59 @@ jobs: PY_COLORS: 0 - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* retention-days: 1 - libcurl: + postgres16: env: - TOTAL_GROUPS: 1 + TOTAL_GROUPS: 2 strategy: fail-fast: false matrix: - group-number: [1] + group-number: [1, 2] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 + services: + postgres16: + image: postgres:16 + env: + POSTGRES_PASSWORD: postgres + ports: + - 8080:5432 + - 8081:5432 + # Set health checks to wait until postgres has started + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - # Special case packages - - name: Install libcurl-dev + - name: Configure pip cache run: | - sudo apt-get update - sudo apt-get install libcurl4-openssl-dev + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -218,26 +307,29 @@ jobs: PY_COLORS: 0 - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* retention-days: 1 - postgres: + postgres9: env: - TOTAL_GROUPS: 2 + TOTAL_GROUPS: 1 strategy: fail-fast: false matrix: - group-number: [1, 2] + group-number: [1] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 - services: - postgres: + postgres9: image: postgres:9 env: POSTGRES_PASSWORD: postgres @@ -252,21 +344,22 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin - - name: Install odbc driver for postgresql + - name: Configure pip cache run: | - sudo apt-get update - sudo sudo apt-get install odbc-postgresql - sudo sed -i 's/Driver=psqlodbca.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbca.so/g' /etc/odbcinst.ini - sudo sed -i 's/Driver=psqlodbcw.so/Driver=\/usr\/lib\/x86_64-linux-gnu\/odbc\/psqlodbcw.so/g' /etc/odbcinst.ini - sudo sed -i 's/Setup=libodbcpsqlS.so/Setup=\/usr\/lib\/x86_64-linux-gnu\/odbc\/libodbcpsqlS.so/g' /etc/odbcinst.ini + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -278,7 +371,74 @@ jobs: PY_COLORS: 0 - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 + with: + name: coverage-${{ github.job }}-${{ strategy.job-index }} + path: ./**/.coverage.* + retention-days: 1 + + mssql: + env: + TOTAL_GROUPS: 1 + + strategy: + fail-fast: false + matrix: + group-number: [1] + + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway + timeout-minutes: 30 + services: + mssql: + image: mcr.microsoft.com/azure-sql-edge:latest + env: + MSSQL_USER: python_agent + MSSQL_PASSWORD: python_agent + MSSQL_SA_PASSWORD: "python_agent#1234" + ACCEPT_EULA: "Y" + ports: + - 8080:1433 + - 8081:1433 + # Set health checks to wait until mysql has started + options: >- + --health-cmd "/opt/mssql-tools/bin/sqlcmd -U SA -P $MSSQL_SA_PASSWORD -Q 'SELECT 1'" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip + + - name: Get Environments + id: get-envs + run: | + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + env: + GROUP_NUMBER: ${{ matrix.group-number }} + + - name: Test + run: | + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + env: + TOX_PARALLEL_NO_SPINNER: 1 + PY_COLORS: 0 + + - name: Upload Coverage Artifacts + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* @@ -293,9 +453,12 @@ jobs: matrix: group-number: [1, 2] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 - services: mysql: image: mysql:5.6 @@ -315,13 +478,22 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -333,7 +505,110 @@ jobs: PY_COLORS: 0 - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 + with: + name: coverage-${{ github.job }}-${{ strategy.job-index }} + path: ./**/.coverage.* + retention-days: 1 + + rediscluster: + env: + TOTAL_GROUPS: 1 + + strategy: + fail-fast: false + matrix: + group-number: [1] + + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway + timeout-minutes: 30 + services: + redis1: + image: hmstepanek/redis-cluster-node:1.0.0 + ports: + - 6379:6379 + - 16379:16379 + options: >- + --add-host=host.docker.internal:host-gateway + + redis2: + image: hmstepanek/redis-cluster-node:1.0.0 + ports: + - 6380:6379 + - 16380:16379 + options: >- + --add-host=host.docker.internal:host-gateway + + redis3: + image: hmstepanek/redis-cluster-node:1.0.0 + ports: + - 6381:6379 + - 16381:16379 + options: >- + --add-host=host.docker.internal:host-gateway + + redis4: + image: hmstepanek/redis-cluster-node:1.0.0 + ports: + - 6382:6379 + - 16382:16379 + options: >- + --add-host=host.docker.internal:host-gateway + + redis5: + image: hmstepanek/redis-cluster-node:1.0.0 + ports: + - 6383:6379 + - 16383:16379 + options: >- + --add-host=host.docker.internal:host-gateway + + redis6: + image: hmstepanek/redis-cluster-node:1.0.0 + ports: + - 6384:6379 + - 16384:16379 + options: >- + --add-host=host.docker.internal:host-gateway + + cluster-setup: + image: hmstepanek/redis-cluster:1.0.0 + options: >- + --add-host=host.docker.internal:host-gateway + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip + + - name: Get Environments + id: get-envs + run: | + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + env: + GROUP_NUMBER: ${{ matrix.group-number }} + + - name: Test + run: | + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + env: + TOX_PARALLEL_NO_SPINNER: 1 + PY_COLORS: 0 + + - name: Upload Coverage Artifacts + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* @@ -348,9 +623,12 @@ jobs: matrix: group-number: [1, 2] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 - services: redis: image: redis @@ -365,13 +643,22 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -383,7 +670,7 @@ jobs: PY_COLORS: 0 - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* @@ -398,9 +685,12 @@ jobs: matrix: group-number: [1] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 - services: solr: image: bitnami/solr:8.8.2 @@ -417,13 +707,22 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -435,7 +734,7 @@ jobs: PY_COLORS: 0 - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* @@ -450,9 +749,12 @@ jobs: matrix: group-number: [1, 2] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 - services: memcached: image: memcached @@ -467,13 +769,22 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -485,7 +796,7 @@ jobs: PY_COLORS: 0 - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* @@ -500,9 +811,12 @@ jobs: matrix: group-number: [1] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 - services: rabbitmq: image: rabbitmq @@ -518,13 +832,22 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -536,83 +859,85 @@ jobs: PY_COLORS: 0 - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* retention-days: 1 - #kafka: - # env: - # TOTAL_GROUPS: 4 - - # strategy: - # fail-fast: false - # matrix: - # group-number: [1, 2, 3, 4] - - # runs-on: ubuntu-20.04 - # timeout-minutes: 30 - - # services: - # zookeeper: - # image: bitnami/zookeeper:3.7 - # env: - # ALLOW_ANONYMOUS_LOGIN: yes - - # ports: - # - 2181:2181 - - # kafka: - # image: bitnami/kafka:3.2 - # ports: - # - 8080:8080 - # - 8081:8081 - # env: - # ALLOW_PLAINTEXT_LISTENER: yes - # KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - # KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true - # KAFKA_CFG_LISTENERS: L1://:8080,L2://:8081 - # KAFKA_CFG_ADVERTISED_LISTENERS: L1://127.0.0.1:8080,L2://kafka:8081, - # KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT - # KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L2 - - # steps: - # - uses: actions/checkout@v3 - # - uses: ./.github/actions/setup-python-matrix - - # # Special case packages - # - name: Install librdkafka-dev - # run: | - # # Use lsb-release to find the codename of Ubuntu to use to install the correct library name - # sudo apt-get update - # sudo ln -fs /usr/share/zoneinfo/America/Los_Angeles /etc/localtime - # sudo apt-get install -y wget gnupg2 software-properties-common - # sudo wget -qO - https://packages.confluent.io/deb/7.2/archive.key | sudo apt-key add - - # sudo add-apt-repository "deb https://packages.confluent.io/clients/deb $(lsb_release -cs) main" - # sudo apt-get update - # sudo apt-get install -y librdkafka-dev/$(lsb_release -c | cut -f 2) - - # - name: Get Environments - # id: get-envs - # run: | - # echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" - # env: - # GROUP_NUMBER: ${{ matrix.group-number }} - - # - name: Test - # run: | - # tox -vv -e ${{ steps.get-envs.outputs.envs }} - # env: - # TOX_PARALLEL_NO_SPINNER: 1 - # PY_COLORS: 0 - - # - name: Upload Coverage Artifacts - # uses: actions/upload-artifact@v3 - # with: - # name: coverage-${{ github.job }}-${{ strategy.job-index }} - # path: ./**/.coverage.* - # retention-days: 1 + kafka: + env: + TOTAL_GROUPS: 4 + + strategy: + fail-fast: false + matrix: + group-number: [1, 2, 3, 4] + + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway + timeout-minutes: 30 + services: + zookeeper: + image: bitnami/zookeeper:3.9.1 + env: + ALLOW_ANONYMOUS_LOGIN: yes + + ports: + - 2181:2181 + + kafka: + image: bitnami/kafka:3.6.1 + ports: + - 8080:8080 + - 8082:8082 + - 8083:8083 + env: + KAFKA_ENABLE_KRAFT: no + ALLOW_PLAINTEXT_LISTENER: yes + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE: true + KAFKA_CFG_LISTENERS: L1://:8082,L2://:8083,L3://:8080 + KAFKA_CFG_ADVERTISED_LISTENERS: L1://host.docker.internal:8082,L2://host.docker.internal:8083,L3://kafka:8080 + KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: L1:PLAINTEXT,L2:PLAINTEXT,L3:PLAINTEXT + KAFKA_CFG_INTER_BROKER_LISTENER_NAME: L3 + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip + + - name: Get Environments + id: get-envs + run: | + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT + env: + GROUP_NUMBER: ${{ matrix.group-number }} + + - name: Test + run: | + tox -vv -e ${{ steps.get-envs.outputs.envs }} -p auto + env: + TOX_PARALLEL_NO_SPINNER: 1 + PY_COLORS: 0 + + - name: Upload Coverage Artifacts + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 + with: + name: coverage-${{ github.job }}-${{ strategy.job-index }} + path: ./**/.coverage.* + retention-days: 1 mongodb: env: @@ -623,9 +948,12 @@ jobs: matrix: group-number: [1] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 - services: mongodb: image: mongo:3.6.4 @@ -640,13 +968,22 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -658,7 +995,7 @@ jobs: PY_COLORS: 0 - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* @@ -673,11 +1010,14 @@ jobs: matrix: group-number: [1] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 - services: - es07: + elasticsearch: image: elasticsearch:7.17.8 env: "discovery.type": "single-node" @@ -692,13 +1032,22 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -710,7 +1059,7 @@ jobs: PY_COLORS: 0 - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* @@ -725,11 +1074,14 @@ jobs: matrix: group-number: [1] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 - services: - es08: + elasticsearch: image: elasticsearch:8.6.0 env: "xpack.security.enabled": "false" @@ -745,13 +1097,22 @@ jobs: --health-retries 5 steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -763,13 +1124,13 @@ jobs: PY_COLORS: 0 - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* retention-days: 1 - gearman: + firestore: env: TOTAL_GROUPS: 1 @@ -778,29 +1139,49 @@ jobs: matrix: group-number: [1] - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest + container: + image: ghcr.io/newrelic/newrelic-python-agent-ci:latest + options: >- + --add-host=host.docker.internal:host-gateway timeout-minutes: 30 - services: - gearman: - image: artefactual/gearmand + firestore: + # Image set here MUST be repeated down below in options. See comment below. + image: gcr.io/google.com/cloudsdktool/google-cloud-cli:437.0.1-emulators ports: - - 4730:4730 - # Set health checks to wait until gearman has started + - 8080:8080 + # Set health checks to wait 5 seconds in lieu of an actual healthcheck options: >- - --health-cmd "(echo status ; sleep 0.1) | nc 127.0.0.1 4730 -w 1" + --health-cmd "echo success" --health-interval 10s --health-timeout 5s --health-retries 5 + --health-start-period 5s + gcr.io/google.com/cloudsdktool/google-cloud-cli:437.0.1-emulators /bin/bash -c "gcloud emulators firestore start --host-port=0.0.0.0:8080" || + # This is a very hacky solution. GitHub Actions doesn't provide APIs for setting commands on services, but allows adding arbitrary options. + # --entrypoint won't work as it only accepts an executable and not the [] syntax. + # Instead, we specify the image again the command afterwards like a call to docker create. The result is a few environment variables + # and the original command being appended to our hijacked docker create command. We can avoid any issues by adding || to prevent that + # from every being executed as bash commands. steps: - - uses: actions/checkout@v3 - - uses: ./.github/actions/setup-python-matrix + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # 4.1.1 + + - name: Fetch git tags + run: | + git config --global --add safe.directory "$GITHUB_WORKSPACE" + git fetch --tags origin + + - name: Configure pip cache + run: | + mkdir -p /github/home/.cache/pip + chown -R $(whoami) /github/home/.cache/pip - name: Get Environments id: get-envs run: | - echo "::set-output name=envs::$(tox -l | grep "^${{ github.job }}\-" | ./.github/workflows/get-envs.py)" + echo "envs=$(tox -l | grep '^${{ github.job }}\-' | ./.github/workflows/get-envs.py)" >> $GITHUB_OUTPUT env: GROUP_NUMBER: ${{ matrix.group-number }} @@ -812,7 +1193,7 @@ jobs: PY_COLORS: 0 - name: Upload Coverage Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # 4.3.1 with: name: coverage-${{ github.job }}-${{ strategy.job-index }} path: ./**/.coverage.* diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 12081d1ee7..d525b7df4d 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -228,14 +228,3 @@ entering the directory of the tests you want to run. Then, run the following command: ``tox -c tox.ini -e [test environment]`` - -******* - Slack -******* - -We host a public Slack with a dedicated channel for contributors and -maintainers of open source projects hosted by New Relic. If you are -contributing to this project, you're welcome to request access to the -#oss-contributors channel in the newrelicusers.slack.com workspace. To -request access, please use this `link -`__. diff --git a/MANIFEST.in b/MANIFEST.in index 0a75ce7520..0759bce87d 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -8,3 +8,5 @@ include newrelic/common/cacert.pem include newrelic/packages/wrapt/LICENSE include newrelic/packages/wrapt/README include newrelic/packages/urllib3/LICENSE.txt +include newrelic/packages/isort/LICENSE +include newrelic/packages/opentelemetry_proto/LICENSE.txt diff --git a/README.rst b/README.rst index e33809946a..59d5e2c8a8 100644 --- a/README.rst +++ b/README.rst @@ -129,7 +129,7 @@ A note about vulnerabilities As noted in our `security policy `_, New Relic is committed to the privacy and security of our customers and their data. We believe that providing coordinated disclosure by security researchers and engaging with the security community are important means to achieve our security goals. -If you believe you have found a security vulnerability in this project or any of New Relic's products or websites, we welcome and greatly appreciate you reporting it to New Relic through `HackerOne `_. +If you believe you have found a security vulnerability in this project or any of New Relic's products or websites, we welcome and greatly appreciate you reporting it to New Relic through `our bug bounty program `_. License ------- diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 3662484f6b..7aa68f22dd 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -14,16 +14,25 @@ Copyright (c) Django Software Foundation and individual contributors. Distributed under the following license(s): - * [The BSD 3-Clause License](https://opensource.org/licenses/BSD-3-Clause) +* [The BSD 3-Clause License](https://opensource.org/licenses/BSD-3-Clause) -## [six](https://pypi.org/project/six) +## [isort](https://pypi.org/project/isort) -Copyright (c) 2010-2013 Benjamin Peterson +Copyright (c) 2013 Timothy Edmund Crosley Distributed under the following license(s): - * [The MIT License](http://opensource.org/licenses/MIT) +* [The MIT License](http://opensource.org/licenses/MIT) + + +## [opentelemetry-proto](https://pypi.org/project/opentelemetry-proto) + +Copyright (c) The OpenTelemetry Authors + +Distributed under the following license(s): + +* [The Apache License, Version 2.0 License](https://opensource.org/license/apache-2-0/) ## [time.monotonic](newrelic/common/_monotonic.c) @@ -32,7 +41,7 @@ Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, 2011, Distributed under the following license(s): - * [Python Software Foundation](https://docs.python.org/3/license.html) +* [Python Software Foundation](https://docs.python.org/3/license.html) ## [urllib3](https://pypi.org/project/urllib3) @@ -41,7 +50,7 @@ Copyright (c) 2008-2019 Andrey Petrov and contributors (see CONTRIBUTORS.txt) Distributed under the following license(s): - * [The MIT License](http://opensource.org/licenses/MIT) +* [The MIT License](http://opensource.org/licenses/MIT) ## [wrapt](https://pypi.org/project/wrapt) @@ -51,5 +60,5 @@ All rights reserved. Distributed under the following license(s): - * [The BSD 2-Clause License](http://opensource.org/licenses/BSD-2-Clause) +* [The BSD 2-Clause License](http://opensource.org/licenses/BSD-2-Clause) diff --git a/codecov.yml b/codecov.yml index ec600226a2..0f5d4b4bce 100644 --- a/codecov.yml +++ b/codecov.yml @@ -1,21 +1,18 @@ ignore: - - "newrelic/packages/**/*" - - "newrelic/packages/*" - - "newrelic/hooks/adapter_meinheld.py" + - "newreilc/hooks/component_sentry.py" + - "newrelic/admin/*" + - "newrelic/console.py" - "newrelic/hooks/adapter_flup.py" + - "newrelic/hooks/adapter_meinheld.py" + - "newrelic/hooks/adapter_paste.py" - "newrelic/hooks/component_piston.py" + - "newrelic/hooks/database_psycopg2ct.py" + - "newrelic/hooks/datastore_aioredis.py" + - "newrelic/hooks/datastore_aredis.py" + - "newrelic/hooks/datastore_motor.py" - "newrelic/hooks/datastore_pyelasticsearch.py" - - "newrelic/hooks/external_pywapi.py" - "newrelic/hooks/external_dropbox.py" - "newrelic/hooks/external_facepy.py" - "newrelic/hooks/external_xmlrpclib.py" - - "newrelic/hooks/framework_pylons.py" - - "newrelic/hooks/framework_web2py.py" - - "newrelic/hooks/middleware_weberror.py" - - "newrelic/hooks/framework_webpy.py" - - "newrelic/hooks/database_oursql.py" - - "newrelic/hooks/database_psycopg2ct.py" - - "newrelic/hooks/datastore_umemcache.py" - # Temporarily disable kafka - - "newrelic/hooks/messagebroker_kafkapython.py" - - "newrelic/hooks/messagebroker_confluentkafka.py" + - "newrelic/packages/*" + - "newrelic/packages/**/*" diff --git a/newrelic/admin/__init__.py b/newrelic/admin/__init__.py index e41599a318..e07241ba19 100644 --- a/newrelic/admin/__init__.py +++ b/newrelic/admin/__init__.py @@ -12,29 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - -import sys import logging +import sys _builtin_plugins = [ - 'debug_console', - 'generate_config', - 'license_key', - 'local_config', - 'network_config', - 'record_deploy', - 'run_program', - 'run_python', - 'server_config', - 'validate_config' + "debug_console", + "generate_config", + "license_key", + "local_config", + "network_config", + "record_deploy", + "run_program", + "run_python", + "server_config", + "validate_config", ] _commands = {} -def command(name, options='', description='', hidden=False, - log_intercept=True, deprecated=False): +def command(name, options="", description="", hidden=False, log_intercept=True, deprecated=False): def wrapper(callback): callback.name = name callback.options = options @@ -44,6 +41,7 @@ def wrapper(callback): callback.deprecated = deprecated _commands[name] = callback return callback + return wrapper @@ -51,15 +49,15 @@ def usage(name): details = _commands[name] if details.deprecated: print("[WARNING] This command is deprecated and will be removed") - print('Usage: newrelic-admin %s %s' % (name, details.options)) + print(f"Usage: newrelic-admin {name} {details.options}") -@command('help', '[command]', hidden=True) +@command("help", "[command]", hidden=True) def help(args): if not args: - print('Usage: newrelic-admin command [options]') + print("Usage: newrelic-admin command [options]") print() - print("Type 'newrelic-admin help '", end='') + print("Type 'newrelic-admin help '", end="") print("for help on a specific command.") print() print("Available commands are:") @@ -68,24 +66,24 @@ def help(args): for name in commands: details = _commands[name] if not details.hidden: - print(' ', name) + print(" ", name) else: name = args[0] if name not in _commands: - print("Unknown command '%s'." % name, end=' ') + print(f"Unknown command '{name}'.", end=" ") print("Type 'newrelic-admin help' for usage.") else: details = _commands[name] - print('Usage: newrelic-admin %s %s' % (name, details.options)) + print(f"Usage: newrelic-admin {name} {details.options}") if details.description: print() description = details.description if details.deprecated: - description = '[DEPRECATED] ' + description + description = f"[DEPRECATED] {description}" print(description) @@ -99,7 +97,7 @@ def emit(self, record): if len(logging.root.handlers) != 0: return - if record.name.startswith('newrelic.packages'): + if record.name.startswith("newrelic.packages"): return if record.levelno < logging.WARNING: @@ -107,9 +105,9 @@ def emit(self, record): return logging.StreamHandler.emit(self, record) - _stdout_logger = logging.getLogger('newrelic') + _stdout_logger = logging.getLogger("newrelic") _stdout_handler = FilteredStreamHandler(sys.stdout) - _stdout_format = '%(levelname)s - %(message)s\n' + _stdout_format = "%(levelname)s - %(message)s\n" _stdout_formatter = logging.Formatter(_stdout_format) _stdout_handler.setFormatter(_stdout_formatter) _stdout_logger.addHandler(_stdout_handler) @@ -117,19 +115,27 @@ def emit(self, record): def load_internal_plugins(): for name in _builtin_plugins: - module_name = '%s.%s' % (__name__, name) + module_name = f"{__name__}.{name}" __import__(module_name) def load_external_plugins(): try: - import pkg_resources + # Preferred after Python 3.10 + if sys.version_info >= (3, 10): + from importlib.metadata import entry_points + # Introduced in Python 3.8 + elif sys.version_info >= (3, 8) and sys.version_info <= (3, 9): + from importlib_metadata import entry_points + # Removed in Python 3.12 + else: + from pkg_resources import iter_entry_points as entry_points except ImportError: return - group = 'newrelic.admin' + group = "newrelic.admin" - for entrypoint in pkg_resources.iter_entry_points(group=group): + for entrypoint in entry_points(group=group): __import__(entrypoint.module_name) @@ -138,12 +144,12 @@ def main(): if len(sys.argv) > 1: command = sys.argv[1] else: - command = 'help' + command = "help" callback = _commands[command] except Exception: - print("Unknown command '%s'." % command, end='') + print(f"Unknown command '{command}'.", end="") print("Type 'newrelic-admin help' for usage.") sys.exit(1) @@ -156,5 +162,5 @@ def main(): load_internal_plugins() load_external_plugins() -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/newrelic/admin/debug_console.py b/newrelic/admin/debug_console.py index 1a61629946..65fff008d0 100644 --- a/newrelic/admin/debug_console.py +++ b/newrelic/admin/debug_console.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - from newrelic.admin import command, usage @command('debug-console', 'config_file [session_log]', diff --git a/newrelic/admin/generate_config.py b/newrelic/admin/generate_config.py index c48dff6a28..1613143c0d 100644 --- a/newrelic/admin/generate_config.py +++ b/newrelic/admin/generate_config.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - from newrelic.admin import command, usage @command('generate-config', 'license_key [output_file]', diff --git a/newrelic/admin/license_key.py b/newrelic/admin/license_key.py index 35aaed1f41..4effdbd820 100644 --- a/newrelic/admin/license_key.py +++ b/newrelic/admin/license_key.py @@ -12,21 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - from newrelic.admin import command, usage +from newrelic.common.encoding_utils import obfuscate_license_key -@command('license-key', 'config_file [log_file]', -"""Prints out the account license key after having loaded the settings -from .""") +@command( + "license-key", + "config_file [log_file]", + """Prints out an obfuscated account license key after having loaded the settings +from .""", +) def license_key(args): + import logging import os import sys - import logging if len(args) == 0: - usage('license-key') + usage("license-key") sys.exit(1) from newrelic.config import initialize @@ -35,7 +37,7 @@ def license_key(args): if len(args) >= 2: log_file = args[1] else: - log_file = '/tmp/python-agent-test.log' + log_file = "/tmp/python-agent-test.log" log_level = logging.DEBUG @@ -45,14 +47,13 @@ def license_key(args): pass config_file = args[0] - environment = os.environ.get('NEW_RELIC_ENVIRONMENT') + environment = os.environ.get("NEW_RELIC_ENVIRONMENT") - if config_file == '-': - config_file = os.environ.get('NEW_RELIC_CONFIG_FILE') + if config_file == "-": + config_file = os.environ.get("NEW_RELIC_CONFIG_FILE") - initialize(config_file, environment, ignore_errors=False, - log_file=log_file, log_level=log_level) + initialize(config_file, environment, ignore_errors=False, log_file=log_file, log_level=log_level) _settings = global_settings() - print('license_key = %r' % _settings.license_key) + print(f"license_key = {obfuscate_license_key(_settings.license_key)!r}") diff --git a/newrelic/admin/local_config.py b/newrelic/admin/local_config.py index 6585bfcf70..5d6d1daaab 100644 --- a/newrelic/admin/local_config.py +++ b/newrelic/admin/local_config.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - from newrelic.admin import command, usage @@ -54,4 +52,4 @@ def local_config(args): log_file=log_file, log_level=log_level) for key, value in sorted(global_settings()): - print('%s = %r' % (key, value)) + print(f'{key} = {value!r}') diff --git a/newrelic/admin/network_config.py b/newrelic/admin/network_config.py index d2ce41aaf4..590e5a400b 100644 --- a/newrelic/admin/network_config.py +++ b/newrelic/admin/network_config.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - from newrelic.admin import command, usage @@ -55,10 +53,10 @@ def network_config(args): _settings = global_settings() - print('host = %r' % _settings.host) - print('port = %r' % _settings.port) - print('proxy_scheme = %r' % _settings.proxy_scheme) - print('proxy_host = %r' % _settings.proxy_host) - print('proxy_port = %r' % _settings.proxy_port) - print('proxy_user = %r' % _settings.proxy_user) - print('proxy_pass = %r' % _settings.proxy_pass) + print(f'host = {_settings.host!r}') + print(f'port = {_settings.port!r}') + print(f'proxy_scheme = {_settings.proxy_scheme!r}') + print(f'proxy_host = {_settings.proxy_host!r}') + print(f'proxy_port = {_settings.proxy_port!r}') + print(f'proxy_user = {_settings.proxy_user!r}') + print(f'proxy_pass = {_settings.proxy_pass!r}') diff --git a/newrelic/admin/record_deploy.py b/newrelic/admin/record_deploy.py index 65748cc2a3..d08ef1dd71 100644 --- a/newrelic/admin/record_deploy.py +++ b/newrelic/admin/record_deploy.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import pwd @@ -78,12 +76,10 @@ def record_deploy( app_id = fetch_app_id(app_name, client, headers) if app_id is None: raise RuntimeError( - "The application named %r was not found in your account. Please " - "try running the newrelic-admin server-config command to force " - "the application to register with New Relic." % app_name + f"The application named {app_name!r} was not found in your account. Please try running the newrelic-admin server-config command to force the application to register with New Relic." ) - path = "/v2/applications/{}/deployments.json".format(app_id) + path = f"/v2/applications/{app_id}/deployments.json" if user is None: user = pwd.getpwuid(os.getuid()).pw_gecos @@ -107,12 +103,7 @@ def record_deploy( if status_code != 201: raise RuntimeError( - "An unexpected HTTP response of %r was received " - "for request made to https://%s:%d%s. The payload for the " - "request was %r. The response payload for the request was %r. " - "If this issue persists then please report this problem to New " - "Relic support for further investigation." - % (status_code, host, port, path, data, response) + f"An unexpected HTTP response of {status_code!r} was received for request made to https://{host}:{int(port)}{path}. The payload for the request was {data!r}. The response payload for the request was {response!r}. If this issue persists then please report this problem to New Relic support for further investigation." ) diff --git a/newrelic/admin/run_program.py b/newrelic/admin/run_program.py index 0a0f8e7787..747cf95d24 100644 --- a/newrelic/admin/run_program.py +++ b/newrelic/admin/run_program.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - from newrelic.admin import command, usage @@ -42,7 +40,7 @@ def log_message(text, *args): if startup_debug: text = text % args timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) - print('NEWRELIC: %s (%d) - %s' % (timestamp, os.getpid(), text)) + print(f'NEWRELIC: {timestamp} ({os.getpid()}) - {text}') log_message('New Relic Admin Script (%s)', __file__) @@ -80,8 +78,7 @@ def log_message(text, *args): if 'PYTHONPATH' in os.environ: path = os.environ['PYTHONPATH'].split(os.path.pathsep) if boot_directory not in path: - python_path = "%s%s%s" % (boot_directory, os.path.pathsep, - os.environ['PYTHONPATH']) + python_path = f"{boot_directory}{os.path.pathsep}{os.environ['PYTHONPATH']}" os.environ['PYTHONPATH'] = python_path diff --git a/newrelic/admin/run_python.py b/newrelic/admin/run_python.py index ccdb5be55e..6aad03b5c5 100644 --- a/newrelic/admin/run_python.py +++ b/newrelic/admin/run_python.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - from newrelic.admin import command, usage @command('run-python', '...', @@ -37,7 +35,7 @@ def log_message(text, *args): if startup_debug: text = text % args timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime()) - print('NEWRELIC: %s (%d) - %s' % (timestamp, os.getpid(), text)) + print(f'NEWRELIC: {timestamp} ({os.getpid()}) - {text}') log_message('New Relic Admin Script (%s)', __file__) @@ -73,8 +71,7 @@ def log_message(text, *args): if 'PYTHONPATH' in os.environ: path = os.environ['PYTHONPATH'].split(os.path.pathsep) if not boot_directory in path: - python_path = "%s%s%s" % (boot_directory, os.path.pathsep, - os.environ['PYTHONPATH']) + python_path = f"{boot_directory}{os.path.pathsep}{os.environ['PYTHONPATH']}" os.environ['PYTHONPATH'] = python_path diff --git a/newrelic/admin/server_config.py b/newrelic/admin/server_config.py index cd463226d5..b2a4b5074e 100644 --- a/newrelic/admin/server_config.py +++ b/newrelic/admin/server_config.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - from newrelic.admin import command, usage @@ -75,4 +73,4 @@ def server_config(args): _logger.debug('Registration took %s seconds.', _duration) for key, value in sorted(_application.settings): - print('%s = %r' % (key, value)) + print(f'{key} = {value!r}') diff --git a/newrelic/admin/validate_config.py b/newrelic/admin/validate_config.py index ac25b715e1..c31bd1c5ef 100644 --- a/newrelic/admin/validate_config.py +++ b/newrelic/admin/validate_config.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - from newrelic.admin import command, usage @@ -149,6 +147,7 @@ def validate_config(args): sys.exit(1) from newrelic.api.application import register_application + from newrelic.common.encoding_utils import obfuscate_license_key from newrelic.config import initialize from newrelic.core.config import global_settings @@ -200,7 +199,7 @@ def validate_config(args): _logger.debug("Proxy port is %r.", _settings.proxy_port) _logger.debug("Proxy user is %r.", _settings.proxy_user) - _logger.debug("License key is %r.", _settings.license_key) + _logger.debug("License key is %r.", obfuscate_license_key(_settings.license_key)) _timeout = 30.0 @@ -224,7 +223,7 @@ def validate_config(args): url = parts[1].strip() print("Registration successful. Reporting to:") print() - print(" %s" % url) + print(f" {url}") print() break diff --git a/newrelic/agent.py b/newrelic/agent.py index 95a540780e..76f02b8e0d 100644 --- a/newrelic/agent.py +++ b/newrelic/agent.py @@ -15,7 +15,7 @@ from newrelic.api.application import application_instance as __application from newrelic.api.application import application_settings as __application_settings from newrelic.api.application import register_application as __register_application -from newrelic.api.log import NewRelicContextFormatter # noqa +from newrelic.api.log import NewRelicContextFormatter as __NewRelicContextFormatter from newrelic.api.time_trace import ( add_custom_span_attribute as __add_custom_span_attribute, ) @@ -59,6 +59,7 @@ from newrelic.api.transaction import record_custom_metric as __record_custom_metric from newrelic.api.transaction import record_custom_metrics as __record_custom_metrics from newrelic.api.transaction import record_log_event as __record_log_event +from newrelic.api.transaction import record_ml_event as __record_ml_event from newrelic.api.transaction import set_background_task as __set_background_task from newrelic.api.transaction import set_transaction_name as __set_transaction_name from newrelic.api.transaction import suppress_apdex_metric as __suppress_apdex_metric @@ -138,6 +139,9 @@ def __asgi_application(*args, **kwargs): from newrelic.api.html_insertion import verify_body_exists as __verify_body_exists from newrelic.api.lambda_handler import LambdaHandlerWrapper as __LambdaHandlerWrapper from newrelic.api.lambda_handler import lambda_handler as __lambda_handler +from newrelic.api.llm_custom_attributes import ( + WithLlmCustomAttributes as __WithLlmCustomAttributes, +) from newrelic.api.message_trace import MessageTrace as __MessageTrace from newrelic.api.message_trace import MessageTraceWrapper as __MessageTraceWrapper from newrelic.api.message_trace import message_trace as __message_trace @@ -152,6 +156,13 @@ def __asgi_application(*args, **kwargs): from newrelic.api.message_transaction import ( wrap_message_transaction as __wrap_message_transaction, ) +from newrelic.api.ml_model import ( + record_llm_feedback_event as __record_llm_feedback_event, +) +from newrelic.api.ml_model import ( + set_llm_token_count_callback as __set_llm_token_count_callback, +) +from newrelic.api.ml_model import wrap_mlmodel as __wrap_mlmodel from newrelic.api.profile_trace import ProfileTraceWrapper as __ProfileTraceWrapper from newrelic.api.profile_trace import profile_trace as __profile_trace from newrelic.api.profile_trace import wrap_profile_trace as __wrap_profile_trace @@ -172,6 +183,7 @@ def __asgi_application(*args, **kwargs): from newrelic.api.web_transaction import web_transaction as __web_transaction from newrelic.api.web_transaction import wrap_web_transaction as __wrap_web_transaction from newrelic.common.object_names import callable_name as __callable_name +from newrelic.common.object_wrapper import CallableObjectProxy as __CallableObjectProxy from newrelic.common.object_wrapper import FunctionWrapper as __FunctionWrapper from newrelic.common.object_wrapper import InFunctionWrapper as __InFunctionWrapper from newrelic.common.object_wrapper import ObjectProxy as __ObjectProxy @@ -206,11 +218,6 @@ def __asgi_application(*args, **kwargs): # EXPERIMENTAL - Generator traces are currently experimental and may not # exist in this form in future versions of the agent. - -# EXPERIMENTAL - Profile traces are currently experimental and may not -# exist in this form in future versions of the agent. - - initialize = __initialize extra_settings = __wrap_api_call(__extra_settings, "extra_settings") global_settings = __wrap_api_call(__global_settings, "global_settings") @@ -248,6 +255,8 @@ def __asgi_application(*args, **kwargs): record_custom_metrics = __wrap_api_call(__record_custom_metrics, "record_custom_metrics") record_custom_event = __wrap_api_call(__record_custom_event, "record_custom_event") record_log_event = __wrap_api_call(__record_log_event, "record_log_event") +record_ml_event = __wrap_api_call(__record_ml_event, "record_ml_event") +WithLlmCustomAttributes = __wrap_api_call(__WithLlmCustomAttributes, "WithLlmCustomAttributes") accept_distributed_trace_payload = __wrap_api_call( __accept_distributed_trace_payload, "accept_distributed_trace_payload" ) @@ -278,6 +287,7 @@ def __asgi_application(*args, **kwargs): wrap_background_task = __wrap_api_call(__wrap_background_task, "wrap_background_task") LambdaHandlerWrapper = __wrap_api_call(__LambdaHandlerWrapper, "LambdaHandlerWrapper") lambda_handler = __wrap_api_call(__lambda_handler, "lambda_handler") +NewRelicContextFormatter = __wrap_api_call(__NewRelicContextFormatter, "NewRelicContextFormatter") transaction_name = __wrap_api_call(__transaction_name, "transaction_name") TransactionNameWrapper = __wrap_api_call(__TransactionNameWrapper, "TransactionNameWrapper") wrap_transaction_name = __wrap_api_call(__wrap_transaction_name, "wrap_transaction_name") @@ -318,6 +328,7 @@ def __asgi_application(*args, **kwargs): wrap_message_transaction = __wrap_api_call(__wrap_message_transaction, "wrap_message_transaction") callable_name = __wrap_api_call(__callable_name, "callable_name") ObjectProxy = __wrap_api_call(__ObjectProxy, "ObjectProxy") +CallableObjectProxy = __wrap_api_call(__CallableObjectProxy, "CallableObjectProxy") wrap_object = __wrap_api_call(__wrap_object, "wrap_object") wrap_object_attribute = __wrap_api_call(__wrap_object_attribute, "wrap_object_attribute") resolve_path = __wrap_api_call(__resolve_path, "resolve_path") @@ -341,3 +352,6 @@ def __asgi_application(*args, **kwargs): wrap_out_function = __wrap_api_call(__wrap_out_function, "wrap_out_function") insert_html_snippet = __wrap_api_call(__insert_html_snippet, "insert_html_snippet") verify_body_exists = __wrap_api_call(__verify_body_exists, "verify_body_exists") +wrap_mlmodel = __wrap_api_call(__wrap_mlmodel, "wrap_mlmodel") +record_llm_feedback_event = __wrap_api_call(__record_llm_feedback_event, "record_llm_feedback_event") +set_llm_token_count_callback = __wrap_api_call(__set_llm_token_count_callback, "set_llm_token_count_callback") diff --git a/newrelic/api/application.py b/newrelic/api/application.py index ea57829f28..e6f2832e5d 100644 --- a/newrelic/api/application.py +++ b/newrelic/api/application.py @@ -18,11 +18,9 @@ import newrelic.api.import_hook import newrelic.core.agent import newrelic.core.config -import newrelic.packages.six as six -class Application(object): - +class Application(): _lock = threading.Lock() _instances = {} @@ -107,7 +105,7 @@ def shutdown(self): @property def linked_applications(self): - return list(six.iterkeys(self._linked)) + return list(self._linked.keys()) def link_to_application(self, name): self._linked[name] = True @@ -142,17 +140,31 @@ def record_custom_metrics(self, metrics): if self.active and metrics: self._agent.record_custom_metrics(self._name, metrics) + def record_dimensional_metric(self, name, value, tags=None): + if self.active: + self._agent.record_dimensional_metric(self._name, name, value, tags) + + def record_dimensional_metrics(self, metrics): + if self.active and metrics: + self._agent.record_dimensional_metrics(self._name, metrics) + def record_custom_event(self, event_type, params): if self.active: self._agent.record_custom_event(self._name, event_type, params) + def record_ml_event(self, event_type, params): + if self.active: + self._agent.record_ml_event(self._name, event_type, params) + def record_transaction(self, data): if self.active: self._agent.record_transaction(self._name, data) - def record_log_event(self, message, level=None, timestamp=None, priority=None): + def record_log_event(self, message, level=None, timestamp=None, attributes=None, priority=None): if self.active: - self._agent.record_log_event(self._name, message, level, timestamp, priority=priority) + self._agent.record_log_event( + self._name, message, level, timestamp, attributes=attributes, priority=priority + ) def normalize_name(self, name, rule_type="url"): if self.active: diff --git a/newrelic/api/asgi_application.py b/newrelic/api/asgi_application.py index 2e4e4979b3..6650923cce 100644 --- a/newrelic/api/asgi_application.py +++ b/newrelic/api/asgi_application.py @@ -26,7 +26,7 @@ function_wrapper, wrap_object, ) -from newrelic.packages import asgiref_compatibility, six +from newrelic.packages import asgiref_compatibility def _bind_scope(scope, *args, **kwargs): @@ -49,7 +49,7 @@ def double_to_single_callable(wrapped, instance, args, kwargs): return coro_function_wrapper(coro_function, receive, send) -class ASGIBrowserMiddleware(object): +class ASGIBrowserMiddleware(): def __init__(self, app, transaction=None, search_maximum=64 * 1024): self.app = app self.send = None @@ -97,7 +97,9 @@ def should_insert_html(self, headers): content_type = None for header_name, header_value in headers: - # assume header names are lower cased in accordance with ASGI spec + # ASGI spec (https://asgi.readthedocs.io/en/latest/specs/www.html#http) states + # header names should be lower cased, but not required + header_name = header_name.lower() if header_name == b"content-type": content_type = header_value elif header_name == b"content-encoding": @@ -155,16 +157,9 @@ async def send_inject_browser_agent(self, message): # if there's a valid body string, attempt to insert the HTML if verify_body_exists(self.body): - header = self.transaction.browser_timing_header() - if not header: - # If there's no header, abort browser monitoring injection - await self.send_buffered() - return - - footer = self.transaction.browser_timing_footer() - browser_agent_data = six.b(header) + six.b(footer) - - body = insert_html_snippet(self.body, lambda: browser_agent_data, self.search_maximum) + body = insert_html_snippet( + self.body, lambda: self.transaction.browser_timing_header().encode("latin-1"), self.search_maximum + ) # If we have inserted the browser agent if len(body) != len(self.body): @@ -180,7 +175,7 @@ async def send_inject_browser_agent(self, message): try: content_length = int(header_value) - except ValueError: + except (TypeError, ValueError): # Invalid content length results in an abort await self.send_buffered() return @@ -318,7 +313,6 @@ async def nr_async_asgi(receive, send): send=send, source=wrapped, ) as transaction: - # Record details of framework against the transaction for later # reporting as supportability metrics. if framework: diff --git a/newrelic/api/background_task.py b/newrelic/api/background_task.py index a4a9e8e6a6..4cdcd8a0d4 100644 --- a/newrelic/api/background_task.py +++ b/newrelic/api/background_task.py @@ -13,19 +13,16 @@ # limitations under the License. import functools -import sys from newrelic.api.application import Application, application_instance from newrelic.api.transaction import Transaction, current_transaction -from newrelic.common.async_proxy import async_proxy, TransactionContext +from newrelic.common.async_proxy import TransactionContext, async_proxy from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import FunctionWrapper, wrap_object class BackgroundTask(Transaction): - def __init__(self, application, name, group=None, source=None): - # Initialise the common transaction base class. super(BackgroundTask, self).__init__(application, source=source) @@ -53,7 +50,6 @@ def __init__(self, application, name, group=None, source=None): def BackgroundTaskWrapper(wrapped, application=None, name=None, group=None): - def wrapper(wrapped, instance, args, kwargs): if callable(name): if instance is not None: @@ -107,39 +103,19 @@ def create_transaction(transaction): manager = create_transaction(current_transaction(active_only=False)) + # This means that a transaction already exists, so we want to return if not manager: return wrapped(*args, **kwargs) - success = True - - try: - manager.__enter__() - try: - return wrapped(*args, **kwargs) - except: - success = False - if not manager.__exit__(*sys.exc_info()): - raise - finally: - if success and manager._ref_count == 0: - manager._is_finalized = True - manager.__exit__(None, None, None) - else: - manager._request_handler_finalize = True - manager._server_adapter_finalize = True - old_transaction = current_transaction() - if old_transaction is not None: - old_transaction.drop_transaction() + with manager: + return wrapped(*args, **kwargs) return FunctionWrapper(wrapped, wrapper) def background_task(application=None, name=None, group=None): - return functools.partial(BackgroundTaskWrapper, - application=application, name=name, group=group) + return functools.partial(BackgroundTaskWrapper, application=application, name=name, group=group) -def wrap_background_task(module, object_path, application=None, - name=None, group=None): - wrap_object(module, object_path, BackgroundTaskWrapper, - (application, name, group)) +def wrap_background_task(module, object_path, application=None, name=None, group=None): + wrap_object(module, object_path, BackgroundTaskWrapper, (application, name, group)) diff --git a/newrelic/api/cat_header_mixin.py b/newrelic/api/cat_header_mixin.py index fe5c0a71ff..1455cba288 100644 --- a/newrelic/api/cat_header_mixin.py +++ b/newrelic/api/cat_header_mixin.py @@ -17,11 +17,12 @@ # CatHeaderMixin assumes the mixin class also inherits from TimeTrace -class CatHeaderMixin(object): +class CatHeaderMixin(): cat_id_key = 'X-NewRelic-ID' cat_transaction_key = 'X-NewRelic-Transaction' cat_appdata_key = 'X-NewRelic-App-Data' cat_synthetics_key = 'X-NewRelic-Synthetics' + cat_synthetics_info_key = 'X-NewRelic-Synthetics-Info' cat_metadata_key = 'x-newrelic-trace' cat_distributed_trace_key = 'newrelic' settings = None @@ -105,8 +106,9 @@ def generate_request_headers(cls, transaction): (cls.cat_transaction_key, encoded_transaction)) if transaction.synthetics_header: - nr_headers.append( - (cls.cat_synthetics_key, transaction.synthetics_header)) + nr_headers.append((cls.cat_synthetics_key, transaction.synthetics_header)) + if transaction.synthetics_info_header: + nr_headers.append((cls.cat_synthetics_info_key, transaction.synthetics_info_header)) return nr_headers diff --git a/newrelic/api/database_trace.py b/newrelic/api/database_trace.py index 2bc4976887..c09d1345bf 100644 --- a/newrelic/api/database_trace.py +++ b/newrelic/api/database_trace.py @@ -16,7 +16,7 @@ import logging from newrelic.api.time_trace import TimeTrace, current_trace -from newrelic.common.async_wrapper import async_wrapper +from newrelic.common.async_wrapper import async_wrapper as get_async_wrapper from newrelic.common.object_wrapper import FunctionWrapper, wrap_object from newrelic.core.database_node import DatabaseNode from newrelic.core.stack_trace import current_stack @@ -44,11 +44,6 @@ def register_database_client( dbapi2_module._nr_explain_query = explain_query dbapi2_module._nr_explain_stmts = explain_stmts dbapi2_module._nr_instance_info = instance_info - dbapi2_module._nr_datastore_instance_feature_flag = False - - -def enable_datastore_instance_feature(dbapi2_module): - dbapi2_module._nr_datastore_instance_feature_flag = True class DatabaseTrace(TimeTrace): @@ -94,11 +89,7 @@ def __enter__(self): return result def __repr__(self): - return "<%s object at 0x%x %s>" % ( - self.__class__.__name__, - id(self), - dict(sql=self.sql, dbapi2_module=self.dbapi2_module), - ) + return f"<{self.__class__.__name__} object at 0x{id(self):x} {dict(sql=self.sql, dbapi2_module=self.dbapi2_module)}>" @property def is_async_mode(self): @@ -153,12 +144,7 @@ def finalize_data(self, transaction, exc=None, value=None, tb=None): if instance_enabled or db_name_enabled: - if ( - self.dbapi2_module - and self.connect_params - and self.dbapi2_module._nr_datastore_instance_feature_flag - and self.dbapi2_module._nr_instance_info is not None - ): + if self.dbapi2_module and self.connect_params and self.dbapi2_module._nr_instance_info is not None: instance_info = self.dbapi2_module._nr_instance_info(*self.connect_params) @@ -244,9 +230,9 @@ def create_node(self): ) -def DatabaseTraceWrapper(wrapped, sql, dbapi2_module=None): +def DatabaseTraceWrapper(wrapped, sql, dbapi2_module=None, async_wrapper=None): def _nr_database_trace_wrapper_(wrapped, instance, args, kwargs): - wrapper = async_wrapper(wrapped) + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) if not wrapper: parent = current_trace() if not parent: @@ -273,9 +259,9 @@ def _nr_database_trace_wrapper_(wrapped, instance, args, kwargs): return FunctionWrapper(wrapped, _nr_database_trace_wrapper_) -def database_trace(sql, dbapi2_module=None): - return functools.partial(DatabaseTraceWrapper, sql=sql, dbapi2_module=dbapi2_module) +def database_trace(sql, dbapi2_module=None, async_wrapper=None): + return functools.partial(DatabaseTraceWrapper, sql=sql, dbapi2_module=dbapi2_module, async_wrapper=async_wrapper) -def wrap_database_trace(module, object_path, sql, dbapi2_module=None): - wrap_object(module, object_path, DatabaseTraceWrapper, (sql, dbapi2_module)) +def wrap_database_trace(module, object_path, sql, dbapi2_module=None, async_wrapper=None): + wrap_object(module, object_path, DatabaseTraceWrapper, (sql, dbapi2_module, async_wrapper)) diff --git a/newrelic/api/datastore_trace.py b/newrelic/api/datastore_trace.py index fb40abcab3..df198f094f 100644 --- a/newrelic/api/datastore_trace.py +++ b/newrelic/api/datastore_trace.py @@ -15,7 +15,7 @@ import functools from newrelic.api.time_trace import TimeTrace, current_trace -from newrelic.common.async_wrapper import async_wrapper +from newrelic.common.async_wrapper import async_wrapper as get_async_wrapper from newrelic.common.object_wrapper import FunctionWrapper, wrap_object from newrelic.core.datastore_node import DatastoreNode @@ -82,6 +82,9 @@ def __enter__(self): self.product = transaction._intern_string(self.product) self.target = transaction._intern_string(self.target) self.operation = transaction._intern_string(self.operation) + self.host = transaction._intern_string(self.host) + self.port_path_or_id = transaction._intern_string(self.port_path_or_id) + self.database_name = transaction._intern_string(self.database_name) datastore_tracer_settings = transaction.settings.datastore_tracer self.instance_reporting_enabled = datastore_tracer_settings.instance_reporting.enabled @@ -89,11 +92,7 @@ def __enter__(self): return result def __repr__(self): - return "<%s object at 0x%x %s>" % ( - self.__class__.__name__, - id(self), - dict(product=self.product, target=self.target, operation=self.operation), - ) + return f"<{self.__class__.__name__} object at 0x{id(self):x} {dict(product=self.product, target=self.target, operation=self.operation, host=self.host, port_path_or_id=self.port_path_or_id, database_name=self.database_name)}>" def finalize_data(self, transaction, exc=None, value=None, tb=None): if not self.instance_reporting_enabled: @@ -125,7 +124,7 @@ def create_node(self): ) -def DatastoreTraceWrapper(wrapped, product, target, operation): +def DatastoreTraceWrapper(wrapped, product, target, operation, host=None, port_path_or_id=None, database_name=None, async_wrapper=None): """Wraps a method to time datastore queries. :param wrapped: The function to apply the trace to. @@ -140,6 +139,16 @@ def DatastoreTraceWrapper(wrapped, product, target, operation): or the name of any API function/method in the client library. :type operation: str or callable + :param host: The name of the server hosting the actual datastore. + :type host: str + :param port_path_or_id: The value passed in can represent either the port, + path, or id of the datastore being connected to. + :type port_path_or_id: str + :param database_name: The name of database where the current query is being + executed. + :type database_name: str + :param async_wrapper: An async trace wrapper from newrelic.common.async_wrapper. + :type async_wrapper: callable or None :rtype: :class:`newrelic.common.object_wrapper.FunctionWrapper` This is typically used to wrap datastore queries such as calls to Redis or @@ -155,7 +164,7 @@ def DatastoreTraceWrapper(wrapped, product, target, operation): """ def _nr_datastore_trace_wrapper_(wrapped, instance, args, kwargs): - wrapper = async_wrapper(wrapped) + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) if not wrapper: parent = current_trace() if not parent: @@ -187,7 +196,33 @@ def _nr_datastore_trace_wrapper_(wrapped, instance, args, kwargs): else: _operation = operation - trace = DatastoreTrace(_product, _target, _operation, parent=parent, source=wrapped) + if callable(host): + if instance is not None: + _host = host(instance, *args, **kwargs) + else: + _host = host(*args, **kwargs) + else: + _host = host + + if callable(port_path_or_id): + if instance is not None: + _port_path_or_id = port_path_or_id(instance, *args, **kwargs) + else: + _port_path_or_id = port_path_or_id(*args, **kwargs) + else: + _port_path_or_id = port_path_or_id + + if callable(database_name): + if instance is not None: + _database_name = database_name(instance, *args, **kwargs) + else: + _database_name = database_name(*args, **kwargs) + else: + _database_name = database_name + + trace = DatastoreTrace( + _product, _target, _operation, _host, _port_path_or_id, _database_name, parent=parent, source=wrapped + ) if wrapper: # pylint: disable=W0125,W0126 return wrapper(wrapped, trace)(*args, **kwargs) @@ -198,7 +233,7 @@ def _nr_datastore_trace_wrapper_(wrapped, instance, args, kwargs): return FunctionWrapper(wrapped, _nr_datastore_trace_wrapper_) -def datastore_trace(product, target, operation): +def datastore_trace(product, target, operation, host=None, port_path_or_id=None, database_name=None, async_wrapper=None): """Decorator allows datastore query to be timed. :param product: The name of the vendor. @@ -211,6 +246,16 @@ def datastore_trace(product, target, operation): or the name of any API function/method in the client library. :type operation: str + :param host: The name of the server hosting the actual datastore. + :type host: str + :param port_path_or_id: The value passed in can represent either the port, + path, or id of the datastore being connected to. + :type port_path_or_id: str + :param database_name: The name of database where the current query is being + executed. + :type database_name: str + :param async_wrapper: An async trace wrapper from newrelic.common.async_wrapper. + :type async_wrapper: callable or None This is typically used to decorate datastore queries such as calls to Redis or ElasticSearch. @@ -224,10 +269,21 @@ def datastore_trace(product, target, operation): ... time.sleep(*args, **kwargs) """ - return functools.partial(DatastoreTraceWrapper, product=product, target=target, operation=operation) - - -def wrap_datastore_trace(module, object_path, product, target, operation): + return functools.partial( + DatastoreTraceWrapper, + product=product, + target=target, + operation=operation, + host=host, + port_path_or_id=port_path_or_id, + database_name=database_name, + async_wrapper=async_wrapper, + ) + + +def wrap_datastore_trace( + module, object_path, product, target, operation, host=None, port_path_or_id=None, database_name=None, async_wrapper=None +): """Method applies custom timing to datastore query. :param module: Module containing the method to be instrumented. @@ -244,6 +300,16 @@ def wrap_datastore_trace(module, object_path, product, target, operation): or the name of any API function/method in the client library. :type operation: str + :param host: The name of the server hosting the actual datastore. + :type host: str + :param port_path_or_id: The value passed in can represent either the port, + path, or id of the datastore being connected to. + :type port_path_or_id: str + :param database_name: The name of database where the current query is being + executed. + :type database_name: str + :param async_wrapper: An async trace wrapper from newrelic.common.async_wrapper. + :type async_wrapper: callable or None This is typically used to time database query method calls such as Redis GET. @@ -256,4 +322,6 @@ def wrap_datastore_trace(module, object_path, product, target, operation): ... 'sleep') """ - wrap_object(module, object_path, DatastoreTraceWrapper, (product, target, operation)) + wrap_object( + module, object_path, DatastoreTraceWrapper, (product, target, operation, host, port_path_or_id, database_name, async_wrapper) + ) diff --git a/newrelic/api/error_trace.py b/newrelic/api/error_trace.py index a6fedeced5..8b2c8d4ef7 100644 --- a/newrelic/api/error_trace.py +++ b/newrelic/api/error_trace.py @@ -19,7 +19,7 @@ from newrelic.common.object_wrapper import FunctionWrapper, wrap_object -class ErrorTrace(object): +class ErrorTrace(): def __init__( self, ignore_errors=[], diff --git a/newrelic/api/external_trace.py b/newrelic/api/external_trace.py index c43c560c6c..26762e687c 100644 --- a/newrelic/api/external_trace.py +++ b/newrelic/api/external_trace.py @@ -16,7 +16,7 @@ from newrelic.api.cat_header_mixin import CatHeaderMixin from newrelic.api.time_trace import TimeTrace, current_trace -from newrelic.common.async_wrapper import async_wrapper +from newrelic.common.async_wrapper import async_wrapper as get_async_wrapper from newrelic.common.object_wrapper import FunctionWrapper, wrap_object from newrelic.core.external_node import ExternalNode @@ -36,11 +36,7 @@ def __init__(self, library, url, method=None, **kwargs): self.params = {} def __repr__(self): - return "<%s object at 0x%x %s>" % ( - self.__class__.__name__, - id(self), - dict(library=self.library, url=self.url, method=self.method), - ) + return f"<{self.__class__.__name__} object at 0x{id(self):x} {dict(library=self.library, url=self.url, method=self.method)}>" def process_response(self, status_code, headers): self._add_agent_attribute("http.statusCode", status_code) @@ -66,9 +62,9 @@ def create_node(self): ) -def ExternalTraceWrapper(wrapped, library, url, method=None): +def ExternalTraceWrapper(wrapped, library, url, method=None, async_wrapper=None): def dynamic_wrapper(wrapped, instance, args, kwargs): - wrapper = async_wrapper(wrapped) + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) if not wrapper: parent = current_trace() if not parent: @@ -103,7 +99,7 @@ def dynamic_wrapper(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) def literal_wrapper(wrapped, instance, args, kwargs): - wrapper = async_wrapper(wrapped) + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) if not wrapper: parent = current_trace() if not parent: @@ -125,9 +121,9 @@ def literal_wrapper(wrapped, instance, args, kwargs): return FunctionWrapper(wrapped, literal_wrapper) -def external_trace(library, url, method=None): - return functools.partial(ExternalTraceWrapper, library=library, url=url, method=method) +def external_trace(library, url, method=None, async_wrapper=None): + return functools.partial(ExternalTraceWrapper, library=library, url=url, method=method, async_wrapper=async_wrapper) -def wrap_external_trace(module, object_path, library, url, method=None): - wrap_object(module, object_path, ExternalTraceWrapper, (library, url, method)) +def wrap_external_trace(module, object_path, library, url, method=None, async_wrapper=None): + wrap_object(module, object_path, ExternalTraceWrapper, (library, url, method, async_wrapper)) diff --git a/newrelic/api/function_profile.py b/newrelic/api/function_profile.py index ae691f0c99..127b25072d 100644 --- a/newrelic/api/function_profile.py +++ b/newrelic/api/function_profile.py @@ -20,7 +20,7 @@ from newrelic.common.object_wrapper import FunctionWrapper, wrap_object -class FunctionProfile(object): +class FunctionProfile(): def __init__(self, profile): self.profile = profile @@ -33,7 +33,7 @@ def __exit__(self, exc, value, tb): self.profile.disable() pass -class FunctionProfileSession(object): +class FunctionProfileSession(): def __init__(self, filename, delay=1.0, checkpoint=30): self.filename = filename % { 'pid': os.getpid() } diff --git a/newrelic/api/function_trace.py b/newrelic/api/function_trace.py index 474c1b2266..56bcad8607 100644 --- a/newrelic/api/function_trace.py +++ b/newrelic/api/function_trace.py @@ -15,7 +15,7 @@ import functools from newrelic.api.time_trace import TimeTrace, current_trace -from newrelic.common.async_wrapper import async_wrapper +from newrelic.common.async_wrapper import async_wrapper as get_async_wrapper from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import FunctionWrapper, wrap_object from newrelic.core.function_node import FunctionNode @@ -37,7 +37,7 @@ def __init__(self, name, group=None, label=None, params=None, terminal=False, ro group = group or "Function" if group.startswith("/"): - group = "Function" + group + group = f"Function{group}" self.name = name self.group = group @@ -55,18 +55,7 @@ def __enter__(self): return result def __repr__(self): - return "<%s object at 0x%x %s>" % ( - self.__class__.__name__, - id(self), - dict( - name=self.name, - group=self.group, - label=self.label, - params=self.params, - terminal=self.terminal, - rollup=self.rollup, - ), - ) + return f"<{self.__class__.__name__} object at 0x{id(self):x} {dict(name=self.name, group=self.group, label=self.label, params=self.params, terminal=self.terminal, rollup=self.rollup)}>" def terminal_node(self): return self.terminal @@ -89,9 +78,9 @@ def create_node(self): ) -def FunctionTraceWrapper(wrapped, name=None, group=None, label=None, params=None, terminal=False, rollup=None): +def FunctionTraceWrapper(wrapped, name=None, group=None, label=None, params=None, terminal=False, rollup=None, async_wrapper=None): def dynamic_wrapper(wrapped, instance, args, kwargs): - wrapper = async_wrapper(wrapped) + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) if not wrapper: parent = current_trace() if not parent: @@ -147,7 +136,7 @@ def dynamic_wrapper(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) def literal_wrapper(wrapped, instance, args, kwargs): - wrapper = async_wrapper(wrapped) + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) if not wrapper: parent = current_trace() if not parent: @@ -171,13 +160,13 @@ def literal_wrapper(wrapped, instance, args, kwargs): return FunctionWrapper(wrapped, literal_wrapper) -def function_trace(name=None, group=None, label=None, params=None, terminal=False, rollup=None): +def function_trace(name=None, group=None, label=None, params=None, terminal=False, rollup=None, async_wrapper=None): return functools.partial( - FunctionTraceWrapper, name=name, group=group, label=label, params=params, terminal=terminal, rollup=rollup + FunctionTraceWrapper, name=name, group=group, label=label, params=params, terminal=terminal, rollup=rollup, async_wrapper=async_wrapper ) def wrap_function_trace( - module, object_path, name=None, group=None, label=None, params=None, terminal=False, rollup=None + module, object_path, name=None, group=None, label=None, params=None, terminal=False, rollup=None, async_wrapper=None ): - return wrap_object(module, object_path, FunctionTraceWrapper, (name, group, label, params, terminal, rollup)) + return wrap_object(module, object_path, FunctionTraceWrapper, (name, group, label, params, terminal, rollup, async_wrapper)) diff --git a/newrelic/api/generator_trace.py b/newrelic/api/generator_trace.py index 4196597093..f6ac3f9b45 100644 --- a/newrelic/api/generator_trace.py +++ b/newrelic/api/generator_trace.py @@ -71,7 +71,7 @@ def wrapper(wrapped, instance, args, kwargs): _params = params def _generator(generator): - _gname = '%s (generator)' % _name + _gname = f'{_name} (generator)' try: value = None diff --git a/newrelic/api/graphql_trace.py b/newrelic/api/graphql_trace.py index 7a2c9ec02f..3887744f48 100644 --- a/newrelic/api/graphql_trace.py +++ b/newrelic/api/graphql_trace.py @@ -16,7 +16,7 @@ from newrelic.api.time_trace import TimeTrace, current_trace from newrelic.api.transaction import current_transaction -from newrelic.common.async_wrapper import async_wrapper +from newrelic.common.async_wrapper import async_wrapper as get_async_wrapper from newrelic.common.object_wrapper import FunctionWrapper, wrap_object from newrelic.core.graphql_node import GraphQLOperationNode, GraphQLResolverNode @@ -39,16 +39,7 @@ def __init__(self, **kwargs): self.product = "GraphQL" def __repr__(self): - return "<%s object at 0x%x %s>" % ( - self.__class__.__name__, - id(self), - dict( - operation_name=self.operation_name, - operation_type=self.operation_type, - deepest_path=self.deepest_path, - graphql=self.graphql, - ), - ) + return f"<{self.__class__.__name__} object at 0x{id(self):x} {dict(operation_name=self.operation_name, operation_type=self.operation_type, deepest_path=self.deepest_path, graphql=self.graphql)}>" @property def formatted(self): @@ -102,16 +93,16 @@ def set_transaction_name(self, priority=None): transaction = current_transaction() if transaction: name = ( - "%s/%s/%s" % (self.operation_type, self.operation_name, self.deepest_path) + f"{self.operation_type}/{self.operation_name}/{self.deepest_path}" if self.deepest_path - else "%s/%s" % (self.operation_type, self.operation_name) + else f"{self.operation_type}/{self.operation_name}" ) transaction.set_transaction_name(name, "GraphQL", priority=priority) -def GraphQLOperationTraceWrapper(wrapped): +def GraphQLOperationTraceWrapper(wrapped, async_wrapper=None): def _nr_graphql_trace_wrapper_(wrapped, instance, args, kwargs): - wrapper = async_wrapper(wrapped) + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) if not wrapper: parent = current_trace() if not parent: @@ -130,16 +121,16 @@ def _nr_graphql_trace_wrapper_(wrapped, instance, args, kwargs): return FunctionWrapper(wrapped, _nr_graphql_trace_wrapper_) -def graphql_operation_trace(): - return functools.partial(GraphQLOperationTraceWrapper) +def graphql_operation_trace(async_wrapper=None): + return functools.partial(GraphQLOperationTraceWrapper, async_wrapper=async_wrapper) -def wrap_graphql_operation_trace(module, object_path): - wrap_object(module, object_path, GraphQLOperationTraceWrapper) +def wrap_graphql_operation_trace(module, object_path, async_wrapper=None): + wrap_object(module, object_path, GraphQLOperationTraceWrapper, (async_wrapper,)) class GraphQLResolverTrace(TimeTrace): - def __init__(self, field_name=None, **kwargs): + def __init__(self, field_name=None, field_parent_type=None, field_return_type=None, field_path=None, **kwargs): parent = kwargs.pop("parent", None) source = kwargs.pop("source", None) if kwargs: @@ -148,10 +139,13 @@ def __init__(self, field_name=None, **kwargs): super(GraphQLResolverTrace, self).__init__(parent=parent, source=source) self.field_name = field_name + self.field_parent_type = field_parent_type + self.field_return_type = field_return_type + self.field_path = field_path self._product = None def __repr__(self): - return "<%s object at 0x%x %s>" % (self.__class__.__name__, id(self), dict(field_name=self.field_name)) + return f"<{self.__class__.__name__} object at 0x{id(self):x} {dict(field_name=self.field_name)}>" def __enter__(self): super(GraphQLResolverTrace, self).__enter__() @@ -175,6 +169,9 @@ def product(self): def finalize_data(self, *args, **kwargs): self._add_agent_attribute("graphql.field.name", self.field_name) + self._add_agent_attribute("graphql.field.parentType", self.field_parent_type) + self._add_agent_attribute("graphql.field.returnType", self.field_return_type) + self._add_agent_attribute("graphql.field.path", self.field_path) return super(GraphQLResolverTrace, self).finalize_data(*args, **kwargs) @@ -193,9 +190,9 @@ def create_node(self): ) -def GraphQLResolverTraceWrapper(wrapped): +def GraphQLResolverTraceWrapper(wrapped, async_wrapper=None): def _nr_graphql_trace_wrapper_(wrapped, instance, args, kwargs): - wrapper = async_wrapper(wrapped) + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) if not wrapper: parent = current_trace() if not parent: @@ -214,9 +211,9 @@ def _nr_graphql_trace_wrapper_(wrapped, instance, args, kwargs): return FunctionWrapper(wrapped, _nr_graphql_trace_wrapper_) -def graphql_resolver_trace(): - return functools.partial(GraphQLResolverTraceWrapper) +def graphql_resolver_trace(async_wrapper=None): + return functools.partial(GraphQLResolverTraceWrapper, async_wrapper=async_wrapper) -def wrap_graphql_resolver_trace(module, object_path): - wrap_object(module, object_path, GraphQLResolverTraceWrapper) +def wrap_graphql_resolver_trace(module, object_path, async_wrapper=None): + wrap_object(module, object_path, GraphQLResolverTraceWrapper, (async_wrapper,)) diff --git a/newrelic/api/import_hook.py b/newrelic/api/import_hook.py index b36262afc3..2b6359c6d3 100644 --- a/newrelic/api/import_hook.py +++ b/newrelic/api/import_hook.py @@ -15,14 +15,11 @@ import logging import sys -from newrelic.packages import six +from importlib.util import find_spec + _logger = logging.getLogger(__name__) -try: - from importlib.util import find_spec -except ImportError: - find_spec = None _import_hooks = {} @@ -30,7 +27,6 @@ # These modules are imported by the newrelic package and/or do not do # nested imports, so they're ok to import before newrelic. "urllib", - "urllib2", "httplib", "http.client", "urllib.request", @@ -62,71 +58,61 @@ def register_import_hook(name, callable): # pylint: disable=redefined-builtin - if six.PY2: - import imp - - imp.acquire_lock() - - try: - hooks = _import_hooks.get(name, None) + hooks = _import_hooks.get(name, None) - if name not in _import_hooks or hooks is None: + if name not in _import_hooks or hooks is None: - # If no entry in registry or entry already flagged with - # None then module may have been loaded, in which case - # need to check and fire hook immediately. + # If no entry in registry or entry already flagged with + # None then module may have been loaded, in which case + # need to check and fire hook immediately. - hooks = _import_hooks.get(name) + hooks = _import_hooks.get(name) - module = sys.modules.get(name, None) + module = sys.modules.get(name, None) - if module is not None: + if module is not None: - # The module has already been loaded so fire hook - # immediately. + # The module has already been loaded so fire hook + # immediately. - if module.__name__ not in _ok_modules: - _logger.debug( - "Module %s has been imported before the " - "newrelic.agent.initialize call. Import and " - "initialize the New Relic agent before all " - "other modules for best monitoring " - "results.", - module, - ) + if module.__name__ not in _ok_modules: + _logger.debug( + "Module %s has been imported before the " + "newrelic.agent.initialize call. Import and " + "initialize the New Relic agent before all " + "other modules for best monitoring " + "results.", + module, + ) - # Add the module name to the set of uninstrumented modules. - # During harvest, this set will be used to produce metrics. - # The adding of names here and the reading of them during - # harvest should be thread safe. This is because the code - # here is only run during `initialize` which will no-op if - # run multiple times (even if in a thread). The set is read - # from the harvest thread which will run one minute after - # `initialize` is called. + # Add the module name to the set of uninstrumented modules. + # During harvest, this set will be used to produce metrics. + # The adding of names here and the reading of them during + # harvest should be thread safe. This is because the code + # here is only run during `initialize` which will no-op if + # run multiple times (even if in a thread). The set is read + # from the harvest thread which will run one minute after + # `initialize` is called. - _uninstrumented_modules.add(module.__name__) + _uninstrumented_modules.add(module.__name__) - _import_hooks[name] = None + _import_hooks[name] = None - callable(module) + callable(module) - else: + else: - # No hook has been registered so far so create list - # and add current hook. + # No hook has been registered so far so create list + # and add current hook. - _import_hooks[name] = [callable] + _import_hooks[name] = [callable] - else: + else: - # Hook has already been registered, so append current - # hook. + # Hook has already been registered, so append current + # hook. - _import_hooks[name].append(callable) - - finally: - if six.PY2: - imp.release_lock() + _import_hooks[name].append(callable) def _notify_import_hooks(name, module): @@ -181,63 +167,10 @@ class ImportHookFinder: def __init__(self): self._skip = {} - def find_module(self, fullname, path=None): - """ - Find spec and patch import hooks into loader before returning. - - Required for Python 2. - - https://docs.python.org/3/library/importlib.html#importlib.abc.MetaPathFinder.find_module - """ - - # If not something we are interested in we can return. - - if fullname not in _import_hooks: - return None - - # Check whether this is being called on the second time - # through and return. - - if fullname in self._skip: - return None - - # We are now going to call back into import. We set a - # flag to see we are handling the module so that check - # above drops out on subsequent pass and we don't go - # into an infinite loop. - - self._skip[fullname] = True - - try: - # For Python 3 we need to use find_spec() from the importlib - # module. - - if find_spec: - spec = find_spec(fullname) - loader = getattr(spec, "loader", None) - - if loader and not isinstance(loader, (_ImportHookChainedLoader, _ImportHookLoader)): - return _ImportHookChainedLoader(loader) - - else: - __import__(fullname) - - # If we get this far then the module we are - # interested in does actually exist and so return - # our loader to trigger import hooks and then return - # the module. - - return _ImportHookLoader() - - finally: - del self._skip[fullname] - def find_spec(self, fullname, path=None, target=None): """ Find spec and patch import hooks into loader before returning. - Required for Python 3.10+ to avoid warnings. - https://docs.python.org/3/library/importlib.html#importlib.abc.MetaPathFinder.find_spec """ @@ -260,23 +193,18 @@ def find_spec(self, fullname, path=None, target=None): self._skip[fullname] = True try: - # For Python 3 we need to use find_spec() from the importlib - # module. - - if find_spec: - spec = find_spec(fullname) - loader = getattr(spec, "loader", None) + # We call find_spec() from the importlib module. - if loader and not isinstance(loader, (_ImportHookChainedLoader, _ImportHookLoader)): - spec.loader = _ImportHookChainedLoader(loader) + spec = find_spec(fullname) + loader = getattr(spec, "loader", None) - return spec + if loader and not isinstance(loader, (_ImportHookChainedLoader, _ImportHookLoader)): + spec.loader = _ImportHookChainedLoader(loader) - else: - # Not possible, Python 3 defines find_spec and Python 2 does not have find_spec on Finders - return None + return spec finally: + # Delete flag now that it's not needed del self._skip[fullname] diff --git a/newrelic/api/llm_custom_attributes.py b/newrelic/api/llm_custom_attributes.py new file mode 100644 index 0000000000..37745ba062 --- /dev/null +++ b/newrelic/api/llm_custom_attributes.py @@ -0,0 +1,47 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from newrelic.api.transaction import current_transaction + +_logger = logging.getLogger(__name__) + + +class WithLlmCustomAttributes(object): + def __init__(self, custom_attr_dict): + transaction = current_transaction() + if not custom_attr_dict or not isinstance(custom_attr_dict, dict): + raise TypeError( + "custom_attr_dict must be a non-empty dictionary. Received type: %s" % type(custom_attr_dict) + ) + + # Add "llm." prefix to all keys in attribute dictionary + context_attrs = {k if k.startswith("llm.") else f"llm.{k}": v for k, v in custom_attr_dict.items()} + + self.attr_dict = context_attrs + self.transaction = transaction + + def __enter__(self): + if not self.transaction: + _logger.warning("WithLlmCustomAttributes must be called within the scope of a transaction.") + return self + + self.transaction._llm_context_attrs = self.attr_dict + return self + + def __exit__(self, exc, value, tb): + # Clear out context attributes once we leave the current context + if self.transaction: + del self.transaction._llm_context_attrs diff --git a/newrelic/api/log.py b/newrelic/api/log.py index 846ef275ab..4cb1c84a37 100644 --- a/newrelic/api/log.py +++ b/newrelic/api/log.py @@ -16,40 +16,86 @@ import logging import re import warnings -from logging import Formatter, LogRecord +from traceback import format_tb +from newrelic.api.application import application_instance from newrelic.api.time_trace import get_linking_metadata from newrelic.api.transaction import current_transaction, record_log_event from newrelic.common import agent_http +from newrelic.common.encoding_utils import json_encode from newrelic.common.object_names import parse_exc_info from newrelic.core.attribute import truncate from newrelic.core.config import global_settings, is_expected_error -def format_exc_info(exc_info): - _, _, fullnames, message = parse_exc_info(exc_info) - fullname = fullnames[0] +def safe_json_encode(obj, ignore_string_types=False, **kwargs): + # Performs the same operation as json_encode but replaces unserializable objects with a string containing their class name. + # If ignore_string_types is True, do not encode string types further. + # Currently used for safely encoding logging attributes. - formatted = { - "error.class": fullname, - "error.message": message, - } + if ignore_string_types and isinstance(obj, (str, bytes)): + return obj - expected = is_expected_error(exc_info) - if expected is not None: - formatted["error.expected"] = expected + # Attempt to run through JSON serialization + try: + return json_encode(obj, **kwargs) + except Exception: + pass - return formatted + # If JSON serialization fails then return a repr + try: + return repr(obj) + except Exception: + # If repr fails then default to an unprinatable object name + return f"" -class NewRelicContextFormatter(Formatter): - DEFAULT_LOG_RECORD_KEYS = frozenset(vars(LogRecord("", 0, "", 0, "", (), None))) +class NewRelicContextFormatter(logging.Formatter): + DEFAULT_LOG_RECORD_KEYS = frozenset(set(vars(logging.LogRecord("", 0, "", 0, "", (), None))) | {"message"}) def __init__(self, *args, **kwargs): - super(NewRelicContextFormatter, self).__init__() + """ + :param Optional[int] stack_trace_limit: + Specifies the maximum number of frames to include for stack traces. + Defaults to `0` to suppress stack traces. + Setting this to `None` will make it so all available frames are included. + """ + stack_trace_limit = kwargs.pop("stack_trace_limit", 0) + + if stack_trace_limit is not None: + if not isinstance(stack_trace_limit, int): + raise TypeError("stack_trace_limit must be None or a non-negative integer") + if stack_trace_limit < 0: + raise ValueError("stack_trace_limit must be None or a non-negative integer") + self._stack_trace_limit = stack_trace_limit + + super(NewRelicContextFormatter, self).__init__(*args, **kwargs) + + @classmethod + def format_exc_info(cls, exc_info, stack_trace_limit=0): + _, _, fullnames, message = parse_exc_info(exc_info) + fullname = fullnames[0] + + formatted = { + "error.class": fullname, + "error.message": message, + } + + expected = is_expected_error(exc_info) + if expected is not None: + formatted["error.expected"] = expected + + if stack_trace_limit is None or stack_trace_limit > 0: + if exc_info[2] is not None: + stack_trace = "".join(format_tb(exc_info[2], limit=stack_trace_limit)) or None + else: + stack_trace = None + formatted["error.stack_trace"] = stack_trace + + return formatted @classmethod - def log_record_to_dict(cls, record): + def log_record_to_dict(cls, record, stack_trace_limit=0): output = { "timestamp": int(record.created * 1000), "message": record.getMessage(), @@ -65,40 +111,83 @@ def log_record_to_dict(cls, record): output.update(get_linking_metadata()) DEFAULT_LOG_RECORD_KEYS = cls.DEFAULT_LOG_RECORD_KEYS - if len(record.__dict__) > len(DEFAULT_LOG_RECORD_KEYS): - for key in record.__dict__: - if key not in DEFAULT_LOG_RECORD_KEYS: - output["extra." + key] = getattr(record, key) + # If any keys are present in record that aren't in the default, + # add them to the output record. + keys_to_add = set(record.__dict__.keys()) - DEFAULT_LOG_RECORD_KEYS + for key in keys_to_add: + output[f"extra.{key}"] = getattr(record, key) if record.exc_info: - output.update(format_exc_info(record.exc_info)) + output.update(cls.format_exc_info(record.exc_info, stack_trace_limit)) return output def format(self, record): - def safe_str(object, *args, **kwargs): - """Convert object to str, catching any errors raised.""" - try: - return str(object, *args, **kwargs) - except: - return "" % type(object).__name__ + return json.dumps(self.log_record_to_dict(record, self._stack_trace_limit), default=safe_json_encode, separators=(",", ":")) + - return json.dumps(self.log_record_to_dict(record), default=safe_str, separators=(",", ":")) +# Export class methods as top level functions for compatibility +log_record_to_dict = NewRelicContextFormatter.log_record_to_dict +format_exc_info = NewRelicContextFormatter.format_exc_info class NewRelicLogForwardingHandler(logging.Handler): + IGNORED_LOG_RECORD_KEYS = set(["message", "msg"]) + def emit(self, record): try: - # Avoid getting local log decorated message - if hasattr(record, "_nr_original_message"): - message = record._nr_original_message() + nr = None + transaction = current_transaction() + # Retrieve settings + if transaction: + settings = transaction.settings + nr = transaction else: - message = record.getMessage() + application = application_instance(activate=False) + if application and application.enabled: + nr = application + settings = application.settings + else: + # If no settings have been found, fallback to global settings + settings = global_settings() + + # If logging is enabled and the application or transaction is not None. + if settings and settings.application_logging.enabled and nr: + level_name = str(getattr(record, "levelname", "UNKNOWN")) + if settings.application_logging.metrics.enabled: + nr.record_custom_metric("Logging/lines", {"count": 1}) + nr.record_custom_metric(f"Logging/lines/{level_name}", {"count": 1}) + + if settings.application_logging.forwarding.enabled: + if self.formatter: + # Formatter supplied, allow log records to be formatted into a string + message = self.format(record) + else: + # No formatter supplied, attempt to handle dict log records + message = record.msg + if not isinstance(message, dict): + # Allow python to convert the message to a string and template it with args. + message = record.getMessage() - record_log_event(message, record.levelname, int(record.created * 1000)) + # Grab and filter context attributes from log record + context_attrs = self.filter_record_attributes(record) + + record_log_event( + message=message, + level=level_name, + timestamp=int(record.created * 1000), + attributes=context_attrs, + ) + except RecursionError: # Emulates behavior of CPython. + raise except Exception: self.handleError(record) + @classmethod + def filter_record_attributes(cls, record): + record_attrs = vars(record) + return {k: record_attrs[k] for k in record_attrs if k not in cls.IGNORED_LOG_RECORD_KEYS} + class NewRelicLogHandler(logging.Handler): """ @@ -126,8 +215,8 @@ def __init__( "The contributed NewRelicLogHandler has been superseded by automatic instrumentation for " "logging in the standard lib. If for some reason you need to manually configure a handler, " "please use newrelic.api.log.NewRelicLogForwardingHandler to take advantage of all the " - "features included in application log forwarding such as proper batching.", - DeprecationWarning + "features included in application log forwarding such as proper batching.", + DeprecationWarning, ) super(NewRelicLogHandler, self).__init__(level=level) self.license_key = license_key or self.settings.license_key @@ -162,16 +251,7 @@ def emit(self, record): status_code, response = self.client.send_request(path=self.PATH, headers=headers, payload=payload) if status_code < 200 or status_code >= 300: raise RuntimeError( - "An unexpected HTTP response of %r was received for request made to https://%s:%d%s." - "The response payload for the request was %r. If this issue persists then please " - "report this problem to New Relic support for further investigation." - % ( - status_code, - self.client._host, - self.client._port, - self.PATH, - truncate(response.decode("utf-8"), 1024), - ) + f"An unexpected HTTP response of {status_code!r} was received for request made to https://{self.client._host}:{int(self.client._port)}{self.PATH}.The response payload for the request was {truncate(response.decode('utf-8'), 1024)!r}. If this issue persists then please report this problem to New Relic support for further investigation." ) except Exception: @@ -186,5 +266,5 @@ def default_host(self, license_key): return "log-api.newrelic.com" region = region_aware_match.group(1) - host = "log-api." + region + ".newrelic.com" + host = f"log-api.{region}.newrelic.com" return host diff --git a/newrelic/api/memcache_trace.py b/newrelic/api/memcache_trace.py index 6657a9ce27..1dd5886b09 100644 --- a/newrelic/api/memcache_trace.py +++ b/newrelic/api/memcache_trace.py @@ -15,7 +15,7 @@ import functools from newrelic.api.time_trace import TimeTrace, current_trace -from newrelic.common.async_wrapper import async_wrapper +from newrelic.common.async_wrapper import async_wrapper as get_async_wrapper from newrelic.common.object_wrapper import FunctionWrapper, wrap_object from newrelic.core.memcache_node import MemcacheNode @@ -32,7 +32,7 @@ def __init__(self, command, **kwargs): self.command = command def __repr__(self): - return "<%s object at 0x%x %s>" % (self.__class__.__name__, id(self), dict(command=self.command)) + return f"<{self.__class__.__name__} object at 0x{id(self):x} {dict(command=self.command)}>" def terminal_node(self): return True @@ -51,9 +51,9 @@ def create_node(self): ) -def MemcacheTraceWrapper(wrapped, command): +def MemcacheTraceWrapper(wrapped, command, async_wrapper=None): def _nr_wrapper_memcache_trace_(wrapped, instance, args, kwargs): - wrapper = async_wrapper(wrapped) + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) if not wrapper: parent = current_trace() if not parent: @@ -80,9 +80,9 @@ def _nr_wrapper_memcache_trace_(wrapped, instance, args, kwargs): return FunctionWrapper(wrapped, _nr_wrapper_memcache_trace_) -def memcache_trace(command): - return functools.partial(MemcacheTraceWrapper, command=command) +def memcache_trace(command, async_wrapper=None): + return functools.partial(MemcacheTraceWrapper, command=command, async_wrapper=async_wrapper) -def wrap_memcache_trace(module, object_path, command): - wrap_object(module, object_path, MemcacheTraceWrapper, (command,)) +def wrap_memcache_trace(module, object_path, command, async_wrapper=None): + wrap_object(module, object_path, MemcacheTraceWrapper, (command, async_wrapper)) diff --git a/newrelic/api/message_trace.py b/newrelic/api/message_trace.py index be819d7044..7279b2a192 100644 --- a/newrelic/api/message_trace.py +++ b/newrelic/api/message_trace.py @@ -16,17 +16,17 @@ from newrelic.api.cat_header_mixin import CatHeaderMixin from newrelic.api.time_trace import TimeTrace, current_trace -from newrelic.common.async_wrapper import async_wrapper +from newrelic.common.async_wrapper import async_wrapper as get_async_wrapper from newrelic.common.object_wrapper import FunctionWrapper, wrap_object from newrelic.core.message_node import MessageNode class MessageTrace(CatHeaderMixin, TimeTrace): - cat_id_key = "NewRelicID" cat_transaction_key = "NewRelicTransaction" cat_appdata_key = "NewRelicAppData" cat_synthetics_key = "NewRelicSynthetics" + cat_synthetics_info_key = "NewRelicSyntheticsInfo" def __init__(self, library, operation, destination_type, destination_name, params=None, terminal=True, **kwargs): parent = kwargs.pop("parent", None) @@ -64,11 +64,7 @@ def __enter__(self): return result def __repr__(self): - return "<%s object at 0x%x %s>" % ( - self.__class__.__name__, - id(self), - dict(library=self.library, operation=self.operation), - ) + return f"<{self.__class__.__name__} object at 0x{id(self):x} {dict(library=self.library, operation=self.operation)}>" def terminal_node(self): return self.terminal @@ -91,9 +87,11 @@ def create_node(self): ) -def MessageTraceWrapper(wrapped, library, operation, destination_type, destination_name, params={}, terminal=True): +def MessageTraceWrapper( + wrapped, library, operation, destination_type, destination_name, params={}, terminal=True, async_wrapper=None +): def _nr_message_trace_wrapper_(wrapped, instance, args, kwargs): - wrapper = async_wrapper(wrapped) + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) if not wrapper: parent = current_trace() if not parent: @@ -133,7 +131,16 @@ def _nr_message_trace_wrapper_(wrapped, instance, args, kwargs): else: _destination_name = destination_name - trace = MessageTrace(_library, _operation, _destination_type, _destination_name, params={}, terminal=terminal, parent=parent, source=wrapped) + trace = MessageTrace( + _library, + _operation, + _destination_type, + _destination_name, + params=params, + terminal=terminal, + parent=parent, + source=wrapped, + ) if wrapper: # pylint: disable=W0125,W0126 return wrapper(wrapped, trace)(*args, **kwargs) @@ -144,7 +151,7 @@ def _nr_message_trace_wrapper_(wrapped, instance, args, kwargs): return FunctionWrapper(wrapped, _nr_message_trace_wrapper_) -def message_trace(library, operation, destination_type, destination_name, params={}, terminal=True): +def message_trace(library, operation, destination_type, destination_name, params={}, terminal=True, async_wrapper=None): return functools.partial( MessageTraceWrapper, library=library, @@ -153,10 +160,24 @@ def message_trace(library, operation, destination_type, destination_name, params destination_name=destination_name, params=params, terminal=terminal, + async_wrapper=async_wrapper, ) -def wrap_message_trace(module, object_path, library, operation, destination_type, destination_name, params={}, terminal=True): +def wrap_message_trace( + module, + object_path, + library, + operation, + destination_type, + destination_name, + params={}, + terminal=True, + async_wrapper=None, +): wrap_object( - module, object_path, MessageTraceWrapper, (library, operation, destination_type, destination_name, params, terminal) + module, + object_path, + MessageTraceWrapper, + (library, operation, destination_type, destination_name, params, terminal, async_wrapper), ) diff --git a/newrelic/api/message_transaction.py b/newrelic/api/message_transaction.py index 291a3897e6..c1e46bdd79 100644 --- a/newrelic/api/message_transaction.py +++ b/newrelic/api/message_transaction.py @@ -13,7 +13,6 @@ # limitations under the License. import functools -import sys from newrelic.api.application import Application, application_instance from newrelic.api.background_task import BackgroundTask @@ -39,7 +38,6 @@ def __init__( transport_type="AMQP", source=None, ): - name, group = self.get_transaction_name(library, destination_type, destination_name) super(MessageTransaction, self).__init__(application, name, group=group, source=source) @@ -62,8 +60,8 @@ def __init__( @staticmethod def get_transaction_name(library, destination_type, destination_name): - group = "Message/%s/%s" % (library, destination_type) - name = "Named/%s" % destination_name + group = f"Message/{library}/{destination_type}" + name = f"Named/{destination_name}" return name, group def _update_agent_attributes(self): @@ -79,7 +77,7 @@ def _update_agent_attributes(self): ms_attrs["message.correlationId"] = self.correlation_id if self.headers: for k, v in self.headers.items(): - new_key = "message.headers.%s" % k + new_key = f"message.headers.{k}" new_val = str(v) ms_attrs[new_key] = new_val if self.routing_key is not None: @@ -218,30 +216,12 @@ def create_transaction(transaction): manager = create_transaction(current_transaction(active_only=False)) + # This means that transaction already exists and we want to return if not manager: return wrapped(*args, **kwargs) - success = True - - try: - manager.__enter__() - try: - return wrapped(*args, **kwargs) - except: # Catch all - success = False - if not manager.__exit__(*sys.exc_info()): - raise - finally: - if success and manager._ref_count == 0: - manager._is_finalized = True - manager.__exit__(None, None, None) - else: - manager._request_handler_finalize = True - manager._server_adapter_finalize = True - - old_transaction = current_transaction() - if old_transaction is not None: - old_transaction.drop_transaction() + with manager: + return wrapped(*args, **kwargs) return FunctionWrapper(wrapped, wrapper) diff --git a/newrelic/api/ml_model.py b/newrelic/api/ml_model.py new file mode 100644 index 0000000000..c3ab678d3e --- /dev/null +++ b/newrelic/api/ml_model.py @@ -0,0 +1,125 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +import uuid +import warnings + +from newrelic.api.transaction import current_transaction +from newrelic.common.object_names import callable_name +from newrelic.core.config import global_settings +from newrelic.hooks.mlmodel_sklearn import _nr_instrument_model + +_logger = logging.getLogger(__name__) + + +def wrap_mlmodel(model, name=None, version=None, feature_names=None, label_names=None, metadata=None): + model_callable_name = callable_name(model) + _class = model.__class__.__name__ + module = sys.modules[model_callable_name.split(":")[0]] + _nr_instrument_model(module, _class) + if name: + model._nr_wrapped_name = name + if version: + model._nr_wrapped_version = version + if feature_names: + model._nr_wrapped_feature_names = feature_names + if label_names: + model._nr_wrapped_label_names = label_names + if metadata: + model._nr_wrapped_metadata = metadata + + +def record_llm_feedback_event(trace_id, rating, category=None, message=None, metadata=None): + transaction = current_transaction() + if not transaction: + warnings.warn( + "No message feedback events will be recorded. record_llm_feedback_event must be called within the " + "scope of a transaction." + ) + return + + feedback_event_id = str(uuid.uuid4()) + feedback_event = metadata.copy() if metadata else {} + feedback_event.update( + { + "id": feedback_event_id, + "trace_id": trace_id, + "rating": rating, + "category": category, + "message": message, + "ingest_source": "Python", + } + ) + + transaction.record_custom_event("LlmFeedbackMessage", feedback_event) + + +def set_llm_token_count_callback(callback, application=None): + """ + Set the current callback to be used to calculate LLM token counts. + + Arguments: + callback -- the user-defined callback that will calculate and return the total token count as an integer or None if it does not know + application -- optional application object to associate call with + """ + if callback and not callable(callback): + _logger.error( + "callback passed to set_llm_token_count_callback must be a Callable type or None to unset the callback." + ) + return + + from newrelic.api.application import application_instance + + # Check for activated application if it exists and was not given. + application = application or application_instance(activate=False) + + # Get application settings if it exists, or fallback to global settings object. + settings = application.settings if application else global_settings() + + if not settings: + _logger.error( + "Failed to set llm_token_count_callback. Settings not found on application or in global_settings." + ) + return + + if not callback: + settings.ai_monitoring._llm_token_count_callback = None + return + + def _wrap_callback(model, content): + if model is None: + _logger.debug( + "The model argument passed to the user-defined token calculation callback is None. The callback will not be run." + ) + return None + + if content is None: + _logger.debug( + "The content argument passed to the user-defined token calculation callback is None. The callback will not be run." + ) + return None + + token_count_val = callback(model, content) + + if not isinstance(token_count_val, int) or token_count_val < 0: + _logger.warning( + f"llm_token_count_callback returned an invalid value of {token_count_val}. This value must be a positive integer and will not be recorded for the token_count." + ) + return None + + return token_count_val + + settings.ai_monitoring._llm_token_count_callback = _wrap_callback diff --git a/newrelic/api/profile_trace.py b/newrelic/api/profile_trace.py index 28113b1d81..fe65d02ac2 100644 --- a/newrelic/api/profile_trace.py +++ b/newrelic/api/profile_trace.py @@ -13,31 +13,26 @@ # limitations under the License. import functools -import sys import os +import sys -from newrelic.packages import six - -from newrelic.api.time_trace import current_trace +from newrelic import __file__ as AGENT_PACKAGE_FILE from newrelic.api.function_trace import FunctionTrace -from newrelic.common.object_wrapper import FunctionWrapper, wrap_object +from newrelic.api.time_trace import current_trace from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import FunctionWrapper, wrap_object -from newrelic import __file__ as AGENT_PACKAGE_FILE -AGENT_PACKAGE_DIRECTORY = os.path.dirname(AGENT_PACKAGE_FILE) + '/' - +AGENT_PACKAGE_DIRECTORY = f"{os.path.dirname(AGENT_PACKAGE_FILE)}/" -class ProfileTrace(object): +class ProfileTrace(): def __init__(self, depth): self.function_traces = [] self.maximum_depth = depth self.current_depth = 0 - def __call__(self, frame, event, arg): - - if event not in ['call', 'c_call', 'return', 'c_return', - 'exception', 'c_exception']: + def __call__(self, frame, event, arg): # pragma: no cover + if event not in ["call", "c_call", "return", "c_return", "exception", "c_exception"]: return parent = current_trace() @@ -49,8 +44,7 @@ def __call__(self, frame, event, arg): # coroutine systems based on greenlets so don't run # if we detect may be using greenlets. - if (hasattr(sys, '_current_frames') and - parent.thread_id not in sys._current_frames()): + if hasattr(sys, "_current_frames") and parent.thread_id not in sys._current_frames(): return co = frame.f_code @@ -76,7 +70,7 @@ def _callable(): except Exception: pass - for name, obj in six.iteritems(frame.f_globals): + for name, obj in frame.f_globals.items(): try: if obj.__dict__[func_name].func_code is co: return obj.__dict__[func_name] @@ -84,7 +78,7 @@ def _callable(): except Exception: pass - if event in ['call', 'c_call']: + if event in ["call", "c_call"]: # Skip the outermost as we catch that with the root # function traces for the profile trace. @@ -100,19 +94,17 @@ def _callable(): self.function_traces.append(None) return - if event == 'call': + if event == "call": func = _callable() if func: name = callable_name(func) else: - name = '%s:%s#%s' % (func_filename, func_name, - func_line_no) + name = f"{func_filename}:{func_name}#{func_line_no}" else: func = arg name = callable_name(arg) if not name: - name = '%s:@%s#%s' % (func_filename, func_name, - func_line_no) + name = f"{func_filename}:@{func_name}#{func_line_no}" function_trace = FunctionTrace(name=name, parent=parent) function_trace.__enter__() @@ -127,7 +119,7 @@ def _callable(): self.function_traces.append(function_trace) self.current_depth += 1 - elif event in ['return', 'c_return', 'c_exception']: + elif event in ["return", "c_return", "c_exception"]: if not self.function_traces: return @@ -143,9 +135,7 @@ def _callable(): self.current_depth -= 1 -def ProfileTraceWrapper(wrapped, name=None, group=None, label=None, - params=None, depth=3): - +def ProfileTraceWrapper(wrapped, name=None, group=None, label=None, params=None, depth=3): def wrapper(wrapped, instance, args, kwargs): parent = current_trace() @@ -192,7 +182,7 @@ def wrapper(wrapped, instance, args, kwargs): _params = params with FunctionTrace(_name, _group, _label, _params, parent=parent, source=wrapped): - if not hasattr(sys, 'getprofile'): + if not hasattr(sys, "getprofile"): return wrapped(*args, **kwargs) profiler = sys.getprofile() @@ -212,11 +202,8 @@ def wrapper(wrapped, instance, args, kwargs): def profile_trace(name=None, group=None, label=None, params=None, depth=3): - return functools.partial(ProfileTraceWrapper, name=name, - group=group, label=label, params=params, depth=depth) + return functools.partial(ProfileTraceWrapper, name=name, group=group, label=label, params=params, depth=depth) -def wrap_profile_trace(module, object_path, name=None, - group=None, label=None, params=None, depth=3): - return wrap_object(module, object_path, ProfileTraceWrapper, - (name, group, label, params, depth)) +def wrap_profile_trace(module, object_path, name=None, group=None, label=None, params=None, depth=3): + return wrap_object(module, object_path, ProfileTraceWrapper, (name, group, label, params, depth)) diff --git a/newrelic/api/solr_trace.py b/newrelic/api/solr_trace.py index e482158ee9..5b03c33289 100644 --- a/newrelic/api/solr_trace.py +++ b/newrelic/api/solr_trace.py @@ -14,6 +14,7 @@ import newrelic.api.object_wrapper import newrelic.api.time_trace +import newrelic.common.object_wrapper import newrelic.core.solr_node @@ -30,11 +31,7 @@ def __init__(self, library, command, **kwargs): self.command = command def __repr__(self): - return "<%s object at 0x%x %s>" % ( - self.__class__.__name__, - id(self), - dict(library=self.library, command=self.command), - ) + return f"<{self.__class__.__name__} object at 0x{id(self):x} {dict(library=self.library, command=self.command)}>" def terminal_node(self): return True @@ -54,7 +51,7 @@ def create_node(self): ) -class SolrTraceWrapper(object): +class SolrTraceWrapper(): def __init__(self, wrapped, library, command): if isinstance(wrapped, tuple): (instance, wrapped) = wrapped @@ -111,4 +108,4 @@ def decorator(wrapped): def wrap_solr_trace(module, object_path, library, command): - newrelic.api.object_wrapper.wrap_object(module, object_path, SolrTraceWrapper, (library, command)) + newrelic.common.object_wrapper.wrap_object(module, object_path, SolrTraceWrapper, (library, command)) diff --git a/newrelic/api/supportability.py b/newrelic/api/supportability.py index 1fd727d466..0451ba5e6d 100644 --- a/newrelic/api/supportability.py +++ b/newrelic/api/supportability.py @@ -18,7 +18,7 @@ def wrap_api_call(method, method_name): - metric_name = 'Supportability/api/%s' % method_name + metric_name = f'Supportability/api/{method_name}' @function_wrapper def _nr_wrap_api_call_(wrapped, instance, args, kwargs): diff --git a/newrelic/api/time_trace.py b/newrelic/api/time_trace.py index 24be0e00f6..38546ea778 100644 --- a/newrelic/api/time_trace.py +++ b/newrelic/api/time_trace.py @@ -30,12 +30,10 @@ from newrelic.core.config import is_expected_error, should_ignore_error from newrelic.core.trace_cache import trace_cache -from newrelic.packages import six - _logger = logging.getLogger(__name__) -class TimeTrace(object): +class TimeTrace(): def __init__(self, parent=None, source=None): self.parent = parent self.root = None @@ -54,7 +52,7 @@ def __init__(self, parent=None, source=None): self.exc_data = (None, None, None) self.should_record_segment_params = False # 16-digit random hex. Padded with zeros in the front. - self.guid = "%016x" % random.getrandbits(64) + self.guid = f"{random.getrandbits(64):016x}" self.agent_attributes = {} self.user_attributes = {} @@ -73,7 +71,7 @@ def _is_leaf(self): return self.child_count == len(self.children) def __repr__(self): - return "<%s object at 0x%x %s>" % (self.__class__.__name__, id(self), dict(name=getattr(self, "name", None))) + return f"<{self.__class__.__name__} object at 0x{id(self):x} {dict(name=getattr(self, 'name', None))}>" def __enter__(self): self.parent = parent = self.parent or current_trace() @@ -216,8 +214,7 @@ def add_code_level_metrics(self, source): node.add_attrs(self._add_agent_attribute) except Exception as exc: _logger.debug( - "Failed to extract source code context from callable %s. Report this issue to newrelic support. Exception: %s" - % (source, exc) + f"Failed to extract source code context from callable {source}. Report this issue to newrelic support. Exception: {exc}" ) def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_code=None): @@ -260,6 +257,11 @@ def _observe_exception(self, exc_info=None, ignore=None, expected=None, status_c module, name, fullnames, message_raw = parse_exc_info((exc, value, tb)) fullname = fullnames[0] + # In case message is in JSON format for OpenAI models + # this will result in a "cleaner" message format + if getattr(value, "_nr_message", None): + message_raw = value._nr_message + # Check to see if we need to strip the message before recording it. if settings.strip_exception_messages.enabled and fullname not in settings.strip_exception_messages.allowlist: @@ -422,23 +424,31 @@ def notice_error(self, error=None, attributes=None, expected=None, ignore=None, input_attributes = {} input_attributes.update(transaction._custom_params) input_attributes.update(attributes) - error_group_name_raw = settings.error_collector.error_group_callback(value, { - "traceback": tb, - "error.class": exc, - "error.message": message_raw, - "error.expected": is_expected, - "custom_params": input_attributes, - "transactionName": getattr(transaction, "name", None), - "response.status": getattr(transaction, "_response_code", None), - "request.method": getattr(transaction, "_request_method", None), - "request.uri": getattr(transaction, "_request_uri", None), - }) + error_group_name_raw = settings.error_collector.error_group_callback( + value, + { + "traceback": tb, + "error.class": exc, + "error.message": message_raw, + "error.expected": is_expected, + "custom_params": input_attributes, + "transactionName": getattr(transaction, "name", None), + "response.status": getattr(transaction, "_response_code", None), + "request.method": getattr(transaction, "_request_method", None), + "request.uri": getattr(transaction, "_request_uri", None), + }, + ) if error_group_name_raw: _, error_group_name = process_user_attribute("error.group.name", error_group_name_raw) - if error_group_name is None or not isinstance(error_group_name, six.string_types): - raise ValueError("Invalid attribute value for error.group.name. Expected string, got: %s" % repr(error_group_name_raw)) + if error_group_name is None or not isinstance(error_group_name, str): + raise ValueError( + f"Invalid attribute value for error.group.name. Expected string, got: {repr(error_group_name_raw)}" + ) except Exception: - _logger.error("Encountered error when calling error group callback:\n%s", "".join(traceback.format_exception(*sys.exc_info()))) + _logger.error( + "Encountered error when calling error group callback:\n%s", + "".join(traceback.format_exception(*sys.exc_info())), + ) error_group_name = None transaction._create_error_node( @@ -595,13 +605,11 @@ def update_async_exclusive_time(self, min_child_start_time, exclusive_duration): def process_child(self, node, is_async): self.children.append(node) if is_async: - # record the lowest start time self.min_child_start_time = min(self.min_child_start_time, node.start_time) # if there are no children running, finalize exclusive time if self.child_count == len(self.children): - exclusive_duration = node.end_time - self.min_child_start_time self.update_async_exclusive_time(self.min_child_start_time, exclusive_duration) diff --git a/newrelic/api/transaction.py b/newrelic/api/transaction.py index f04bcba849..5c990926d3 100644 --- a/newrelic/api/transaction.py +++ b/newrelic/api/transaction.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import logging import os import random @@ -44,6 +42,7 @@ json_decode, json_encode, obfuscate, + snake_case, ) from newrelic.core.attribute import ( MAX_ATTRIBUTE_LENGTH, @@ -51,27 +50,31 @@ MAX_NUM_USER_ATTRIBUTES, create_agent_attributes, create_attributes, - create_user_attributes, process_user_attribute, + resolve_logging_context_attributes, truncate, ) from newrelic.core.attribute_filter import ( + DST_ALL, DST_ERROR_COLLECTOR, DST_NONE, DST_TRANSACTION_TRACER, ) -from newrelic.core.config import CUSTOM_EVENT_RESERVOIR_SIZE, LOG_EVENT_RESERVOIR_SIZE +from newrelic.core.config import ( + CUSTOM_EVENT_RESERVOIR_SIZE, + LOG_EVENT_RESERVOIR_SIZE, + ML_EVENT_RESERVOIR_SIZE, +) from newrelic.core.custom_event import create_custom_event from newrelic.core.log_event_node import LogEventNode from newrelic.core.stack_trace import exception_stack -from newrelic.core.stats_engine import CustomMetrics, SampledDataSet +from newrelic.core.stats_engine import CustomMetrics, DimensionalMetrics, SampledDataSet from newrelic.core.thread_utilization import utilization_tracker from newrelic.core.trace_cache import ( TraceCacheActiveTraceError, TraceCacheNoActiveTraceError, trace_cache, ) -from newrelic.packages import six _logger = logging.getLogger(__name__) @@ -142,7 +145,7 @@ def root(self, value): pass -class CachedPath(object): +class CachedPath(): def __init__(self, transaction): self._name = None self.transaction = weakref.ref(transaction) @@ -158,21 +161,19 @@ def path(self): return "Unknown" -class Transaction(object): - +class Transaction(): STATE_PENDING = 0 STATE_RUNNING = 1 STATE_STOPPED = 2 def __init__(self, application, enabled=None, source=None): - self._application = application self._source = source self.thread_id = None - self._transaction_id = id(self) + self._identity = id(self) self._transaction_lock = threading.Lock() self._dead = False @@ -189,6 +190,7 @@ def __init__(self, application, enabled=None, source=None): self._frameworks = set() self._message_brokers = set() self._dispatchers = set() + self._ml_models = set() self._frozen_path = None @@ -267,9 +269,10 @@ def __init__(self, application, enabled=None, source=None): self.rum_token = None - trace_id = "%032x" % random.getrandbits(128) + trace_id = f"{random.getrandbits(128):032x}" # 16-digit random hex. Padded with zeros in the front. + # This is the official transactionId in the UI. self.guid = trace_id[:16] # 32-digit random hex. Padded with zeros in the front. @@ -301,12 +304,20 @@ def __init__(self, application, enabled=None, source=None): self._alternate_path_hashes = {} self.is_part_of_cat = False + # Synthetics Header self.synthetics_resource_id = None self.synthetics_job_id = None self.synthetics_monitor_id = None self.synthetics_header = None + # Synthetics Info Header + self.synthetics_type = None + self.synthetics_initiator = None + self.synthetics_attributes = None + self.synthetics_info_header = None + self._custom_metrics = CustomMetrics() + self._dimensional_metrics = DimensionalMetrics() global_settings = application.global_settings @@ -330,12 +341,14 @@ def __init__(self, application, enabled=None, source=None): self._custom_events = SampledDataSet( capacity=self._settings.event_harvest_config.harvest_limits.custom_event_data ) + self._ml_events = SampledDataSet(capacity=self._settings.event_harvest_config.harvest_limits.ml_event_data) self._log_events = SampledDataSet( capacity=self._settings.event_harvest_config.harvest_limits.log_event_data ) else: self._custom_events = SampledDataSet(capacity=CUSTOM_EVENT_RESERVOIR_SIZE) self._log_events = SampledDataSet(capacity=LOG_EVENT_RESERVOIR_SIZE) + self._ml_events = SampledDataSet(capacity=ML_EVENT_RESERVOIR_SIZE) def __del__(self): self._dead = True @@ -343,7 +356,6 @@ def __del__(self): self.__exit__(None, None, None) def __enter__(self): - assert self._state == self.STATE_PENDING # Bail out if the transaction is not enabled. @@ -403,13 +415,12 @@ def __enter__(self): return self def __exit__(self, exc, value, tb): - # Bail out if the transaction is not enabled. if not self.enabled: return - if self._transaction_id != id(self): + if self._identity != id(self): return if not self._settings: @@ -536,25 +547,29 @@ def __exit__(self, exc, value, tb): self.total_time += exclusive if self.client_cross_process_id is not None: - metric_name = "ClientApplication/%s/all" % (self.client_cross_process_id) + metric_name = f"ClientApplication/{self.client_cross_process_id}/all" self.record_custom_metric(metric_name, duration) # Record supportability metrics for api calls - for key, value in six.iteritems(self._transaction_metrics): + for key, value in self._transaction_metrics.items(): self.record_custom_metric(key, {"count": value}) if self._frameworks: for framework, version in self._frameworks: - self.record_custom_metric("Python/Framework/%s/%s" % (framework, version), 1) + self.record_custom_metric(f"Python/Framework/{framework}/{version}", 1) if self._message_brokers: for message_broker, version in self._message_brokers: - self.record_custom_metric("Python/MessageBroker/%s/%s" % (message_broker, version), 1) + self.record_custom_metric(f"Python/MessageBroker/{message_broker}/{version}", 1) if self._dispatchers: for dispatcher, version in self._dispatchers: - self.record_custom_metric("Python/Dispatcher/%s/%s" % (dispatcher, version), 1) + self.record_custom_metric(f"Python/Dispatcher/{dispatcher}/{version}", 1) + + if self._ml_models: + for ml_model, version in self._ml_models: + self.record_custom_metric(f"Supportability/Python/ML/{ml_model}/{version}", 1) if self._settings.distributed_tracing.enabled: # Sampled and priority need to be computed at the end of the @@ -584,10 +599,12 @@ def __exit__(self, exc, value, tb): errors=tuple(self._errors), slow_sql=tuple(self._slow_sql), custom_events=self._custom_events, + ml_events=self._ml_events, log_events=self._log_events, apdex_t=self.apdex, suppress_apdex=self.suppress_apdex, custom_metrics=self._custom_metrics, + dimensional_metrics=self._dimensional_metrics, guid=self.guid, cpu_time=self._cpu_user_time_value, suppress_transaction_trace=self.suppress_transaction_trace, @@ -598,6 +615,10 @@ def __exit__(self, exc, value, tb): synthetics_job_id=self.synthetics_job_id, synthetics_monitor_id=self.synthetics_monitor_id, synthetics_header=self.synthetics_header, + synthetics_type=self.synthetics_type, + synthetics_initiator=self.synthetics_initiator, + synthetics_attributes=self.synthetics_attributes, + synthetics_info_header=self.synthetics_info_header, is_part_of_cat=self.is_part_of_cat, trip_id=self.trip_id, path_hash=self.path_hash, @@ -636,7 +657,6 @@ def __exit__(self, exc, value, tb): # new samples can cause an error. if not self.ignore_transaction: - self._application.record_transaction(node) @property @@ -696,9 +716,9 @@ def name_for_metric(self): # leading slash may be significant in that situation. if group in ("Uri", "NormalizedUri") and transaction_name.startswith("/"): - name = "%s%s" % (group, transaction_name) + name = f"{group}{transaction_name}" else: - name = "%s/%s" % (group, transaction_name) + name = f"{group}/{transaction_name}" return name @@ -719,7 +739,7 @@ def path(self): if self._frozen_path: return self._frozen_path - return "%s/%s" % (self.type, self.name_for_metric) + return f"{self.type}/{self.name_for_metric}" @property def trip_id(self): @@ -751,7 +771,7 @@ def path_hash(self): if not self.is_part_of_cat: return None - identifier = "%s;%s" % (self.application.name, self.path) + identifier = f"{self.application.name};{self.path}" # Check if identifier is already part of the _alternate_path_hashes and # return the value if available. @@ -836,11 +856,26 @@ def trace_intrinsics(self): i_attrs["synthetics_job_id"] = self.synthetics_job_id if self.synthetics_monitor_id: i_attrs["synthetics_monitor_id"] = self.synthetics_monitor_id + if self.synthetics_type: + i_attrs["synthetics_type"] = self.synthetics_type + if self.synthetics_initiator: + i_attrs["synthetics_initiator"] = self.synthetics_initiator + if self.synthetics_attributes: + # Add all synthetics attributes + for k, v in self.synthetics_attributes.items(): + if k: + i_attrs[f"synthetics_{snake_case(k)}"] = v + if self.total_time: i_attrs["totalTime"] = self.total_time if self._loop_time: i_attrs["eventLoopTime"] = self._loop_time + # `guid` is added here to make it an intrinsic + # that is agnostic to distributed tracing. + if self.guid: + i_attrs["guid"] = self.guid + # Add in special CPU time value for UI to display CPU burn. # TODO: Disable cpu time value for CPU burn as was @@ -859,10 +894,18 @@ def trace_intrinsics(self): def distributed_trace_intrinsics(self): i_attrs = {} + # Include this here since guid is now an intrinsic attribute, + # whether or not DT is enabled. In most cases, trace_intrinsics + # is called and, within that, this function is called. However, + # there are cases, such as slow SQL calls in database_node that + # call this function directly, so we want to make sure this is + # included here as well. (as of now, the guid is thought to be + # a distributed tracing intrinsic that should be included elsewhere) + i_attrs["guid"] = self.guid + if not self._settings.distributed_tracing.enabled: return i_attrs - i_attrs["guid"] = self.guid i_attrs["sampled"] = self.sampled i_attrs["priority"] = self.priority i_attrs["traceId"] = self.trace_id @@ -929,13 +972,11 @@ def filter_request_parameters(self, params): @property def request_parameters(self): if (self.capture_params is None) or self.capture_params: - if self._request_params: - r_attrs = {} for k, v in self._request_params.items(): - new_key = "request.parameters.%s" % k + new_key = f"request.parameters.{k}" new_val = ",".join(v) final_key, final_val = process_user_attribute(new_key, new_val) @@ -966,12 +1007,12 @@ def _update_agent_attributes(self): @property def user_attributes(self): - return create_user_attributes(self._custom_params, self.attribute_filter) + return create_attributes(self._custom_params, DST_ALL, self.attribute_filter) def _compute_sampled_and_priority(self): if self._priority is None: # truncate priority field to 6 digits past the decimal - self._priority = float("%.6f" % random.random()) # nosec + self._priority = float(f"{random.random():.6f}") # nosec if self._sampled is None: self._sampled = self._application.compute_sampled() @@ -1037,7 +1078,9 @@ def _create_distributed_trace_data(self): settings = self._settings account_id = settings.account_id - trusted_account_key = settings.trusted_account_key + trusted_account_key = settings.trusted_account_key or ( + self._settings.serverless_mode.enabled and self._settings.account_id + ) application_id = settings.primary_application_id if not (account_id and application_id and trusted_account_key and settings.distributed_tracing.enabled): @@ -1095,13 +1138,12 @@ def _generate_distributed_trace_headers(self, data=None): try: data = data or self._create_distributed_trace_data() if data: - traceparent = W3CTraceParent(data).text() yield ("traceparent", traceparent) tracestate = NrTraceState(data).text() if self.tracestate: - tracestate += "," + self.tracestate + tracestate += f",{self.tracestate}" yield ("tracestate", tracestate) self._record_supportability("Supportability/TraceContext/Create/Success") @@ -1129,7 +1171,10 @@ def _can_accept_distributed_trace_headers(self): return False settings = self._settings - if not (settings.distributed_tracing.enabled and settings.trusted_account_key): + trusted_account_key = settings.trusted_account_key or ( + self._settings.serverless_mode.enabled and self._settings.account_id + ) + if not (settings.distributed_tracing.enabled and trusted_account_key): return False if self._distributed_trace_state: @@ -1175,10 +1220,13 @@ def _accept_distributed_trace_payload(self, payload, transport_type="HTTP"): settings = self._settings account_id = data.get("ac") + trusted_account_key = settings.trusted_account_key or ( + self._settings.serverless_mode.enabled and self._settings.account_id + ) # If trust key doesn't exist in the payload, use account_id received_trust_key = data.get("tk", account_id) - if settings.trusted_account_key != received_trust_key: + if trusted_account_key != received_trust_key: self._record_supportability("Supportability/DistributedTrace/AcceptPayload/Ignored/UntrustedAccount") if settings.debug.log_untrusted_distributed_trace_keys: _logger.debug( @@ -1192,11 +1240,10 @@ def _accept_distributed_trace_payload(self, payload, transport_type="HTTP"): except: return False - if "pr" in data: - try: - data["pr"] = float(data["pr"]) - except: - data["pr"] = None + try: + data["pr"] = float(data["pr"]) + except Exception: + data["pr"] = None self._accept_distributed_trace_data(data, transport_type) self._record_supportability("Supportability/DistributedTrace/AcceptPayload/Success") @@ -1288,8 +1335,10 @@ def accept_distributed_trace_headers(self, headers, transport_type="HTTP"): tracestate = ensure_str(tracestate) try: vendors = W3CTraceState.decode(tracestate) - tk = self._settings.trusted_account_key - payload = vendors.pop(tk + "@nr", "") + trusted_account_key = self._settings.trusted_account_key or ( + self._settings.serverless_mode.enabled and self._settings.account_id + ) + payload = vendors.pop(f"{trusted_account_key}@nr", "") self.tracing_vendors = ",".join(vendors.keys()) self.tracestate = vendors.text(limit=31) except: @@ -1298,7 +1347,7 @@ def accept_distributed_trace_headers(self, headers, transport_type="HTTP"): # Remove trusted new relic header if available and parse if payload: try: - tracestate_data = NrTraceState.decode(payload, tk) + tracestate_data = NrTraceState.decode(payload, trusted_account_key) except: tracestate_data = None if tracestate_data: @@ -1368,9 +1417,9 @@ def _process_incoming_cat_headers(self, encoded_cross_process_id, encoded_txn_he self.record_tt = self.record_tt or txn_header[1] - if isinstance(txn_header[2], six.string_types): + if isinstance(txn_header[2], str): self._trip_id = txn_header[2] - if isinstance(txn_header[3], six.string_types): + if isinstance(txn_header[3], str): self._referring_path_hash = txn_header[3] except Exception: pass @@ -1382,7 +1431,6 @@ def _generate_response_headers(self, read_length=None): # process web external calls. if self.client_cross_process_id is not None: - # Need to work out queueing time and duration up to this # point for inclusion in metrics and response header. If the # recording of the transaction had been prematurely stopped @@ -1426,11 +1474,17 @@ def _generate_response_headers(self, read_length=None): return nr_headers - def get_response_metadata(self): + # This function is CAT related and has been deprecated. + # Eventually, this will be removed. Until then, coverage + # does not need to factor this function into its analysis. + def get_response_metadata(self): # pragma: no cover nr_headers = dict(self._generate_response_headers()) return convert_to_cat_metadata_value(nr_headers) - def process_request_metadata(self, cat_linking_value): + # This function is CAT related and has been deprecated. + # Eventually, this will be removed. Until then, coverage + # does not need to factor this function into its analysis. + def process_request_metadata(self, cat_linking_value): # pragma: no cover try: payload = base64_decode(cat_linking_value) except: @@ -1447,7 +1501,6 @@ def process_request_metadata(self, cat_linking_value): return self._process_incoming_cat_headers(encoded_cross_process_id, encoded_txn_header) def set_transaction_name(self, name, group=None, priority=None): - # Always perform this operation even if the transaction # is not active at the time as will be called from # constructor. If path has been frozen do not allow @@ -1483,12 +1536,12 @@ def set_transaction_name(self, name, group=None, priority=None): group = group or "Function" if group.startswith("/"): - group = "Function" + group + group = f"Function{group}" self._group = group self._name = name - def record_log_event(self, message, level=None, timestamp=None, priority=None): + def record_log_event(self, message, level=None, timestamp=None, attributes=None, priority=None): settings = self.settings if not ( settings @@ -1501,23 +1554,69 @@ def record_log_event(self, message, level=None, timestamp=None, priority=None): timestamp = timestamp if timestamp is not None else time.time() level = str(level) if level is not None else "UNKNOWN" + context_attributes = attributes # Name reassigned for clarity - if not message or message.isspace(): - _logger.debug("record_log_event called where message was missing. No log event will be sent.") - return + # Unpack message and attributes from dict inputs + if isinstance(message, dict): + message_attributes = {k: v for k, v in message.items() if k != "message"} + message = message.get("message", "") + else: + message_attributes = None + + if message is not None: + # Coerce message into a string type + if not isinstance(message, str): + try: + message = str(message) + except Exception: + # Exit early for invalid message type after unpacking + _logger.debug( + "record_log_event called where message could not be converted to a string type. No log event will be sent." + ) + return + + # Truncate the now unpacked and string converted message + message = truncate(message, MAX_LOG_MESSAGE_LENGTH) + + # Collect attributes from linking metadata, context data, and message attributes + collected_attributes = {} + if settings and settings.application_logging.forwarding.context_data.enabled: + if context_attributes: + context_attributes = resolve_logging_context_attributes( + context_attributes, settings.attribute_filter, "context." + ) + if context_attributes: + collected_attributes.update(context_attributes) + + if message_attributes: + message_attributes = resolve_logging_context_attributes( + message_attributes, settings.attribute_filter, "message." + ) + if message_attributes: + collected_attributes.update(message_attributes) + + # Exit early if no message or attributes found after filtering + if (not message or message.isspace()) and not context_attributes and not message_attributes: + _logger.debug( + "record_log_event called where no message and no attributes were found. No log event will be sent." + ) + return - message = truncate(message, MAX_LOG_MESSAGE_LENGTH) + # Finally, add in linking attributes after checking that there is a valid message or at least 1 attribute + collected_attributes.update(get_linking_metadata()) event = LogEventNode( timestamp=timestamp, level=level, message=message, - attributes=get_linking_metadata(), + attributes=collected_attributes, ) self._log_events.add(event, priority=priority) - def record_exception(self, exc=None, value=None, tb=None, params=None, ignore_errors=None): + # This function has been deprecated (and will be removed eventually) + # and therefore does not need to be included in coverage analysis + def record_exception(self, exc=None, value=None, tb=None, params=None, ignore_errors=None): # pragma: no cover # Deprecation Warning warnings.warn( ("The record_exception function is deprecated. Please use the new api named notice_error instead."), @@ -1600,6 +1699,16 @@ def record_custom_metrics(self, metrics): for name, value in metrics: self._custom_metrics.record_custom_metric(name, value) + def record_dimensional_metric(self, name, value, tags=None): + self._dimensional_metrics.record_dimensional_metric(name, value, tags) + + def record_dimensional_metrics(self, metrics): + for metric in metrics: + name, value = metric[:2] + tags = metric[2] if len(metric) >= 3 else None + + self._dimensional_metrics.record_dimensional_metric(name, value, tags) + def record_custom_event(self, event_type, params): settings = self._settings @@ -1609,10 +1718,23 @@ def record_custom_event(self, event_type, params): if not settings.custom_insights_events.enabled: return - event = create_custom_event(event_type, params) + event = create_custom_event(event_type, params, settings=settings) if event: self._custom_events.add(event, priority=self.priority) + def record_ml_event(self, event_type, params): + settings = self._settings + + if not settings: + return + + if not settings.ml_insights_events.enabled: + return + + event = create_custom_event(event_type, params, settings=settings, is_ml_event=True) + if event: + self._ml_events.add(event, priority=self.priority) + def _intern_string(self, value): return self._string_cache.setdefault(value, value) @@ -1684,7 +1806,9 @@ def add_custom_attributes(self, items): return result - def add_custom_parameter(self, name, value): + # This function has been deprecated (and will be removed eventually) + # and therefore does not need to be included in coverage analysis + def add_custom_parameter(self, name, value): # pragma: no cover # Deprecation warning warnings.warn( ("The add_custom_parameter API has been deprecated. " "Please use the add_custom_attribute API."), @@ -1692,7 +1816,9 @@ def add_custom_parameter(self, name, value): ) return self.add_custom_attribute(name, value) - def add_custom_parameters(self, items): + # This function has been deprecated (and will be removed eventually) + # and therefore does not need to be included in coverage analysis + def add_custom_parameters(self, items): # pragma: no cover # Deprecation warning warnings.warn( ("The add_custom_parameters API has been deprecated. " "Please use the add_custom_attributes API."), @@ -1712,25 +1838,29 @@ def add_dispatcher_info(self, name, version=None): if name: self._dispatchers.add((name, version)) + def add_ml_model_info(self, name, version=None): + if name: + self._ml_models.add((name, version)) + def dump(self, file): """Dumps details about the transaction to the file object.""" - print("Application: %s" % (self.application.name), file=file) - print("Time Started: %s" % (time.asctime(time.localtime(self.start_time))), file=file) - print("Thread Id: %r" % (self.thread_id), file=file) - print("Current Status: %d" % (self._state), file=file) - print("Recording Enabled: %s" % (self.enabled), file=file) - print("Ignore Transaction: %s" % (self.ignore_transaction), file=file) - print("Transaction Dead: %s" % (self._dead), file=file) - print("Transaction Stopped: %s" % (self.stopped), file=file) - print("Background Task: %s" % (self.background_task), file=file) - print("Request URI: %s" % (self._request_uri), file=file) - print("Transaction Group: %s" % (self._group), file=file) - print("Transaction Name: %s" % (self._name), file=file) - print("Name Priority: %r" % (self._name_priority), file=file) - print("Frozen Path: %s" % (self._frozen_path), file=file) - print("AutoRUM Disabled: %s" % (self.autorum_disabled), file=file) - print("Supress Apdex: %s" % (self.suppress_apdex), file=file) + print(f"Application: {self.application.name}", file=file) + print(f"Time Started: {time.asctime(time.localtime(self.start_time))}", file=file) + print(f"Thread Id: {self.thread_id!r}", file=file) + print(f"Current Status: {self._state}", file=file) + print(f"Recording Enabled: {self.enabled}", file=file) + print(f"Ignore Transaction: {self.ignore_transaction}", file=file) + print(f"Transaction Dead: {self._dead}", file=file) + print(f"Transaction Stopped: {self.stopped}", file=file) + print(f"Background Task: {self.background_task}", file=file) + print(f"Request URI: {self._request_uri}", file=file) + print(f"Transaction Group: {self._group}", file=file) + print(f"Transaction Name: {self._name}", file=file) + print(f"Name Priority: {self._name_priority!r}", file=file) + print(f"Frozen Path: {self._frozen_path}", file=file) + print(f"AutoRUM Disabled: {self.autorum_disabled}", file=file) + print(f"Supress Apdex: {self.suppress_apdex}", file=file) def current_transaction(active_only=True): @@ -1796,19 +1926,23 @@ def add_custom_attributes(items): return False -def add_custom_parameter(key, value): +# This function has been deprecated (and will be removed eventually) +# and therefore does not need to be included in coverage analysis +def add_custom_parameter(key, value): # pragma: no cover # Deprecation warning warnings.warn( - ("The add_custom_parameter API has been deprecated. " "Please use the add_custom_attribute API."), + ("The add_custom_parameter API has been deprecated. Please use the add_custom_attribute API."), DeprecationWarning, ) return add_custom_attribute(key, value) -def add_custom_parameters(items): +# This function has been deprecated (and will be removed eventually) +# and therefore does not need to be included in coverage analysis +def add_custom_parameters(items): # pragma: no cover # Deprecation warning warnings.warn( - ("The add_custom_parameters API has been deprecated. " "Please use the add_custom_attributes API."), + ("The add_custom_parameters API has been deprecated. Please use the add_custom_attributes API."), DeprecationWarning, ) return add_custom_attributes(items) @@ -1820,7 +1954,7 @@ def set_user_id(user_id): if not user_id or not transaction: return - if not isinstance(user_id, six.string_types): + if not isinstance(user_id, str): _logger.warning("The set_user_id API requires a string-based user ID.") return @@ -1835,17 +1969,18 @@ def add_framework_info(name, version=None): transaction.add_framework_info(name, version) -def get_browser_timing_header(): +def get_browser_timing_header(nonce=None): transaction = current_transaction() if transaction and hasattr(transaction, "browser_timing_header"): - return transaction.browser_timing_header() + return transaction.browser_timing_header(nonce) return "" -def get_browser_timing_footer(): - transaction = current_transaction() - if transaction and hasattr(transaction, "browser_timing_footer"): - return transaction.browser_timing_footer() +def get_browser_timing_footer(nonce=None): + warnings.warn( + "The get_browser_timing_footer function is deprecated. Please migrate to only using the get_browser_timing_header API instead.", + DeprecationWarning, + ) return "" @@ -1898,6 +2033,44 @@ def record_custom_metrics(metrics, application=None): application.record_custom_metrics(metrics) +def record_dimensional_metric(name, value, tags=None, application=None): + if application is None: + transaction = current_transaction() + if transaction: + transaction.record_dimensional_metric(name, value, tags) + else: + _logger.debug( + "record_dimensional_metric has been called but no " + "transaction was running. As a result, the following metric " + "has not been recorded. Name: %r Value: %r Tags: %r. To correct this " + "problem, supply an application object as a parameter to this " + "record_dimensional_metrics call.", + name, + value, + tags, + ) + elif application.enabled: + application.record_dimensional_metric(name, value, tags) + + +def record_dimensional_metrics(metrics, application=None): + if application is None: + transaction = current_transaction() + if transaction: + transaction.record_dimensional_metrics(metrics) + else: + _logger.debug( + "record_dimensional_metrics has been called but no " + "transaction was running. As a result, the following metrics " + "have not been recorded: %r. To correct this problem, " + "supply an application object as a parameter to this " + "record_dimensional_metric call.", + list(metrics), + ) + elif application.enabled: + application.record_dimensional_metrics(metrics) + + def record_custom_event(event_type, params, application=None): """Record a custom event. @@ -1926,7 +2099,35 @@ def record_custom_event(event_type, params, application=None): application.record_custom_event(event_type, params) -def record_log_event(message, level=None, timestamp=None, application=None, priority=None): +def record_ml_event(event_type, params, application=None): + """Record a machine learning custom event. + + Args: + event_type (str): The type (name) of the ml event. + params (dict): Attributes to add to the event. + application (newrelic.api.Application): Application instance. + + """ + + if application is None: + transaction = current_transaction() + if transaction: + transaction.record_ml_event(event_type, params) + else: + _logger.debug( + "record_ml_event has been called but no " + "transaction was running. As a result, the following event " + "has not been recorded. event_type: %r params: %r. To correct " + "this problem, supply an application object as a parameter to " + "this record_ml_event call.", + event_type, + params, + ) + elif application.enabled: + application.record_ml_event(event_type, params) + + +def record_log_event(message, level=None, timestamp=None, attributes=None, application=None, priority=None): """Record a log event. Args: @@ -1937,12 +2138,12 @@ def record_log_event(message, level=None, timestamp=None, application=None, prio if application is None: transaction = current_transaction() if transaction: - transaction.record_log_event(message, level, timestamp) + transaction.record_log_event(message, level, timestamp, attributes=attributes) else: application = application_instance(activate=False) if application and application.enabled: - application.record_log_event(message, level, timestamp, priority=priority) + application.record_log_event(message, level, timestamp, attributes=attributes, priority=priority) else: _logger.debug( "record_log_event has been called but no transaction or application was running. As a result, " @@ -1953,7 +2154,7 @@ def record_log_event(message, level=None, timestamp=None, application=None, prio timestamp, ) elif application.enabled: - application.record_log_event(message, level, timestamp, priority=priority) + application.record_log_event(message, level, timestamp, attributes=attributes, priority=priority) def accept_distributed_trace_payload(payload, transport_type="HTTP"): diff --git a/newrelic/api/web_transaction.py b/newrelic/api/web_transaction.py index 9749e26194..b2dd27cece 100644 --- a/newrelic/api/web_transaction.py +++ b/newrelic/api/web_transaction.py @@ -13,35 +13,28 @@ # limitations under the License. import functools -import time import logging +import time import warnings -try: - import urlparse -except ImportError: - import urllib.parse as urlparse +import urllib.parse as urlparse from newrelic.api.application import Application, application_instance from newrelic.api.transaction import Transaction, current_transaction - -from newrelic.common.async_proxy import async_proxy, TransactionContext -from newrelic.common.encoding_utils import (obfuscate, json_encode, - decode_newrelic_header, ensure_str) - -from newrelic.core.attribute import create_attributes, process_user_attribute -from newrelic.core.attribute_filter import DST_BROWSER_MONITORING, DST_NONE - -from newrelic.packages import six - +from newrelic.common.async_proxy import TransactionContext, async_proxy +from newrelic.common.encoding_utils import ( + decode_newrelic_header, + ensure_str, + json_encode, + obfuscate, +) from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import FunctionWrapper, wrap_object +from newrelic.core.attribute_filter import DST_BROWSER_MONITORING _logger = logging.getLogger(__name__) -_js_agent_header_fragment = '' -_js_agent_footer_fragment = '' +_js_agent_header_fragment = '' # Seconds since epoch for Jan 1 2000 JAN_1_2000 = time.mktime((2000, 1, 1, 0, 0, 0, 0, 0, 0)) @@ -81,8 +74,8 @@ def _parse_time_stamp(time_stamp): return converted_time -TRUE_VALUES = {'on', 'true', '1'} -FALSE_VALUES = {'off', 'false', '0'} +TRUE_VALUES = {"on", "true", "1"} +FALSE_VALUES = {"off", "false", "0"} def _lookup_environ_setting(environ, name, default=False): @@ -91,7 +84,7 @@ def _lookup_environ_setting(environ, name, default=False): flag = environ[name] - if isinstance(flag, six.string_types): + if isinstance(flag, str): flag = flag.lower() if flag in TRUE_VALUES: @@ -114,43 +107,78 @@ def _parse_synthetics_header(header): version = int(header[0]) if version == 1: - synthetics['version'] = version - synthetics['account_id'] = int(header[1]) - synthetics['resource_id'] = header[2] - synthetics['job_id'] = header[3] - synthetics['monitor_id'] = header[4] + synthetics["version"] = version + synthetics["account_id"] = int(header[1]) + synthetics["resource_id"] = header[2] + synthetics["job_id"] = header[3] + synthetics["monitor_id"] = header[4] except Exception: return return synthetics +def _parse_synthetics_info_header(header): + # Return a dictionary of values from SyntheticsInfo header + # Returns empty dict, if version is not supported. + + synthetics_info = {} + version = None + + try: + version = int(header.get("version")) + + if version == 1: + synthetics_info["version"] = version + synthetics_info["type"] = header.get("type") + synthetics_info["initiator"] = header.get("initiator") + synthetics_info["attributes"] = header.get("attributes") + except Exception: + return + + return synthetics_info + + def _remove_query_string(url): url = ensure_str(url) out = urlparse.urlsplit(url) - return urlparse.urlunsplit((out.scheme, out.netloc, out.path, '', '')) + return urlparse.urlunsplit((out.scheme, out.netloc, out.path, "", "")) def _is_websocket(environ): - return environ.get('HTTP_UPGRADE', '').lower() == 'websocket' + return environ.get("HTTP_UPGRADE", "").lower() == "websocket" -class WebTransaction(Transaction): - unicode_error_reported = False - QUEUE_TIME_HEADERS = ('x-request-start', 'x-queue-start') +def _encode_nonce(nonce): + if not nonce: + return "" + else: + return f' nonce="{ensure_str(nonce)}"' # Extra space intentional - def __init__(self, application, name, group=None, - scheme=None, host=None, port=None, request_method=None, - request_path=None, query_string=None, headers=None, - enabled=None, source=None): +class WebTransaction(Transaction): + unicode_error_reported = False + QUEUE_TIME_HEADERS = ("x-request-start", "x-queue-start") + + def __init__( + self, + application, + name, + group=None, + scheme=None, + host=None, + port=None, + request_method=None, + request_path=None, + query_string=None, + headers=None, + enabled=None, + source=None, + ): super(WebTransaction, self).__init__(application, enabled, source=source) - # Flags for tracking whether RUM header and footer have been - # generated. - + # Flag for tracking whether RUM header has been generated. self.rum_header_generated = False - self.rum_footer_generated = False if not self.enabled: return @@ -188,9 +216,7 @@ def __init__(self, application, name, group=None, if query_string and not self._settings.high_security: query_string = ensure_str(query_string) try: - params = urlparse.parse_qs( - query_string, - keep_blank_values=True) + params = urlparse.parse_qs(query_string, keep_blank_values=True) self._request_params.update(params) except Exception: pass @@ -202,7 +228,7 @@ def __init__(self, application, name, group=None, if name is not None: self.set_transaction_name(name, group, priority=1) elif request_path is not None: - self.set_transaction_name(request_path, 'Uri', priority=1) + self.set_transaction_name(request_path, "Uri", priority=1) def _process_queue_time(self): for queue_time_header in self.QUEUE_TIME_HEADERS: @@ -212,7 +238,7 @@ def _process_queue_time(self): value = ensure_str(value) try: - if value.startswith('t='): + if value.startswith("t="): self.queue_start = _parse_time_stamp(float(value[2:])) else: self.queue_start = _parse_time_stamp(float(value)) @@ -227,31 +253,37 @@ def _process_synthetics_header(self): settings = self._settings - if settings.synthetics.enabled and \ - settings.trusted_account_ids and \ - settings.encoding_key: - - encoded_header = self._request_headers.get('x-newrelic-synthetics') + if settings.synthetics.enabled and settings.trusted_account_ids and settings.encoding_key: + # Synthetics Header + encoded_header = self._request_headers.get("x-newrelic-synthetics") encoded_header = encoded_header and ensure_str(encoded_header) if not encoded_header: return - decoded_header = decode_newrelic_header( - encoded_header, - settings.encoding_key) + decoded_header = decode_newrelic_header(encoded_header, settings.encoding_key) synthetics = _parse_synthetics_header(decoded_header) - if synthetics and \ - synthetics['account_id'] in \ - settings.trusted_account_ids: + # Synthetics Info Header + encoded_info_header = self._request_headers.get("x-newrelic-synthetics-info") + encoded_info_header = encoded_info_header and ensure_str(encoded_info_header) + + decoded_info_header = decode_newrelic_header(encoded_info_header, settings.encoding_key) + synthetics_info = _parse_synthetics_info_header(decoded_info_header) - # Save obfuscated header, because we will pass it along + if synthetics and synthetics["account_id"] in settings.trusted_account_ids: + # Save obfuscated headers, because we will pass them along # unchanged in all external requests. self.synthetics_header = encoded_header - self.synthetics_resource_id = synthetics['resource_id'] - self.synthetics_job_id = synthetics['job_id'] - self.synthetics_monitor_id = synthetics['monitor_id'] + self.synthetics_resource_id = synthetics["resource_id"] + self.synthetics_job_id = synthetics["job_id"] + self.synthetics_monitor_id = synthetics["monitor_id"] + + if synthetics_info: + self.synthetics_info_header = encoded_info_header + self.synthetics_type = synthetics_info["type"] + self.synthetics_initiator = synthetics_info["initiator"] + self.synthetics_attributes = synthetics_info["attributes"] def _process_context_headers(self): # Process the New Relic cross process ID header and extract @@ -259,11 +291,9 @@ def _process_context_headers(self): if self._settings.distributed_tracing.enabled: self.accept_distributed_trace_headers(self._request_headers) else: - client_cross_process_id = \ - self._request_headers.get('x-newrelic-id') - txn_header = self._request_headers.get('x-newrelic-transaction') - self._process_incoming_cat_headers(client_cross_process_id, - txn_header) + client_cross_process_id = self._request_headers.get("x-newrelic-id") + txn_header = self._request_headers.get("x-newrelic-transaction") + self._process_incoming_cat_headers(client_cross_process_id, txn_header) def process_response(self, status_code, response_headers): """Processes response status and headers, extracting any @@ -302,96 +332,84 @@ def process_response(self, status_code, response_headers): # Generate CAT response headers try: - read_length = int(self._request_headers.get('content-length')) + read_length = int(self._request_headers.get("content-length")) except Exception: read_length = -1 return self._generate_response_headers(read_length) def _update_agent_attributes(self): - if 'accept' in self._request_headers: - self._add_agent_attribute('request.headers.accept', - self._request_headers['accept']) + if "accept" in self._request_headers: + self._add_agent_attribute("request.headers.accept", self._request_headers["accept"]) try: - content_length = int(self._request_headers['content-length']) - self._add_agent_attribute('request.headers.contentLength', - content_length) + content_length = int(self._request_headers["content-length"]) + self._add_agent_attribute("request.headers.contentLength", content_length) except: pass - if 'content-type' in self._request_headers: - self._add_agent_attribute('request.headers.contentType', - self._request_headers['content-type']) - if 'host' in self._request_headers: - self._add_agent_attribute('request.headers.host', - self._request_headers['host']) - if 'referer' in self._request_headers: - self._add_agent_attribute('request.headers.referer', - _remove_query_string(self._request_headers['referer'])) - if 'user-agent' in self._request_headers: - self._add_agent_attribute('request.headers.userAgent', - self._request_headers['user-agent']) + if "content-type" in self._request_headers: + self._add_agent_attribute("request.headers.contentType", self._request_headers["content-type"]) + if "host" in self._request_headers: + self._add_agent_attribute("request.headers.host", self._request_headers["host"]) + if "referer" in self._request_headers: + self._add_agent_attribute("request.headers.referer", _remove_query_string(self._request_headers["referer"])) + if "user-agent" in self._request_headers: + self._add_agent_attribute("request.headers.userAgent", self._request_headers["user-agent"]) if self._request_method: - self._add_agent_attribute('request.method', self._request_method) + self._add_agent_attribute("request.method", self._request_method) if self._request_uri: - self._add_agent_attribute('request.uri', self._request_uri) + self._add_agent_attribute("request.uri", self._request_uri) try: - content_length = int(self._response_headers['content-length']) - self._add_agent_attribute('response.headers.contentLength', - content_length) + content_length = int(self._response_headers["content-length"]) + self._add_agent_attribute("response.headers.contentLength", content_length) except: pass - if 'content-type' in self._response_headers: - self._add_agent_attribute('response.headers.contentType', - self._response_headers['content-type']) + if "content-type" in self._response_headers: + self._add_agent_attribute("response.headers.contentType", self._response_headers["content-type"]) if self._response_code is not None: - self._add_agent_attribute('response.status', - str(self._response_code)) + self._add_agent_attribute("response.status", str(self._response_code)) return super(WebTransaction, self)._update_agent_attributes() - def browser_timing_header(self): + def browser_timing_header(self, nonce=None): """Returns the JavaScript header to be included in any HTML response to perform real user monitoring. This function returns - the header as a native Python string. In Python 2 native strings - are stored as bytes. In Python 3 native strings are stored as - unicode. - + the header as a native Python string. """ if not self.enabled: - return '' + return "" if self._state != self.STATE_RUNNING: - return '' + return "" if self.background_task: - return '' + return "" if self.ignore_transaction: - return '' + return "" if not self._settings: - return '' + return "" if not self._settings.browser_monitoring.enabled: - return '' + return "" if not self._settings.license_key: - return '' + return "" # Don't return the header a second time if it has already # been generated. if self.rum_header_generated: - return '' + return "" # Requirement is that the first 13 characters of the account # license key is used as the key when obfuscating values for - # the RUM footer. Will not be able to perform the obfuscation + # the RUM configuration. Will not be able to perform the obfuscation # if license key isn't that long for some reason. if len(self._settings.license_key) < 13: - return '' + return "" # Return the RUM header only if the agent received a valid value # for js_agent_loader from the data collector. The data @@ -400,142 +418,85 @@ def browser_timing_header(self): # 'none'. if self._settings.js_agent_loader: - header = _js_agent_header_fragment % self._settings.js_agent_loader + # Make sure we freeze the path. + + self._freeze_path() + + # When obfuscating values for the browser agent configuration, we only use the + # first 13 characters of the account license key. + + obfuscation_key = self._settings.license_key[:13] + + attributes = {} + + user_attributes = {} + for attr in self.user_attributes: + if attr.destinations & DST_BROWSER_MONITORING: + user_attributes[attr.name] = attr.value + + if user_attributes: + attributes["u"] = user_attributes + + request_parameters = self.request_parameters + request_parameter_attributes = self.filter_request_parameters(request_parameters) + agent_attributes = {} + for attr in request_parameter_attributes: + if attr.destinations & DST_BROWSER_MONITORING: + agent_attributes[attr.name] = attr.value + + if agent_attributes: + attributes["a"] = agent_attributes + + # create the data structure that pull all our data in + + browser_agent_configuration = self.browser_monitoring_intrinsics(obfuscation_key) + + if attributes: + attributes = obfuscate(json_encode(attributes), obfuscation_key) + browser_agent_configuration["atts"] = attributes + + header = _js_agent_header_fragment % ( + _encode_nonce(nonce), + json_encode(browser_agent_configuration), + self._settings.js_agent_loader, + ) # To avoid any issues with browser encodings, we will make sure # that the javascript we inject for the browser agent is ASCII # encodable. Since we obfuscate all agent and user attributes, and # the transaction name with base 64 encoding, this will preserve # those strings, if they have values outside of the ASCII character - # set. In the case of Python 2, we actually then use the encoded - # value as we need a native string, which for Python 2 is a byte - # string. If encoding as ASCII fails we will return an empty + # set. If encoding as ASCII fails we will return an empty # string. try: - if six.PY2: - header = header.encode('ascii') - else: - header.encode('ascii') + header.encode("ascii") except UnicodeError: if not WebTransaction.unicode_error_reported: - _logger.error('ASCII encoding of js-agent-header failed.', - header) + _logger.error("ASCII encoding of js-agent-header failed.", header) WebTransaction.unicode_error_reported = True - header = '' + header = "" else: - header = '' + header = "" # We remember if we have returned a non empty string value and - # if called a second time we will not return it again. The flag - # will also be used to check whether the footer should be - # generated. + # if called a second time we will not return it again. if header: self.rum_header_generated = True return header - def browser_timing_footer(self): - """Returns the JavaScript footer to be included in any HTML - response to perform real user monitoring. This function returns - the footer as a native Python string. In Python 2 native strings - are stored as bytes. In Python 3 native strings are stored as - unicode. - - """ - - if not self.enabled: - return '' - - if self._state != self.STATE_RUNNING: - return '' - - if self.ignore_transaction: - return '' - - # Only generate a footer if the header had already been - # generated and we haven't already generated the footer. - - if not self.rum_header_generated: - return '' - - if self.rum_footer_generated: - return '' - - # Make sure we freeze the path. - - self._freeze_path() - - # When obfuscating values for the footer, we only use the - # first 13 characters of the account license key. - - obfuscation_key = self._settings.license_key[:13] - - attributes = {} - - user_attributes = {} - for attr in self.user_attributes: - if attr.destinations & DST_BROWSER_MONITORING: - user_attributes[attr.name] = attr.value - - if user_attributes: - attributes['u'] = user_attributes - - request_parameters = self.request_parameters - request_parameter_attributes = self.filter_request_parameters( - request_parameters) - agent_attributes = {} - for attr in request_parameter_attributes: - if attr.destinations & DST_BROWSER_MONITORING: - agent_attributes[attr.name] = attr.value - - if agent_attributes: - attributes['a'] = agent_attributes - - # create the data structure that pull all our data in - - footer_data = self.browser_monitoring_intrinsics(obfuscation_key) - - if attributes: - attributes = obfuscate(json_encode(attributes), obfuscation_key) - footer_data['atts'] = attributes - - footer = _js_agent_footer_fragment % json_encode(footer_data) - - # To avoid any issues with browser encodings, we will make sure that - # the javascript we inject for the browser agent is ASCII encodable. - # Since we obfuscate all agent and user attributes, and the transaction - # name with base 64 encoding, this will preserve those strings, if - # they have values outside of the ASCII character set. - # In the case of Python 2, we actually then use the encoded value - # as we need a native string, which for Python 2 is a byte string. - # If encoding as ASCII fails we will return an empty string. - - try: - if six.PY2: - footer = footer.encode('ascii') - else: - footer.encode('ascii') - - except UnicodeError: - if not WebTransaction.unicode_error_reported: - _logger.error('ASCII encoding of js-agent-footer failed.', - footer) - WebTransaction.unicode_error_reported = True - - footer = '' - - # We remember if we have returned a non empty string value and - # if called a second time we will not return it again. - - if footer: - self.rum_footer_generated = True - - return footer + def browser_timing_footer(self, nonce=None): + """Deprecated API that has been replaced entirely by browser_timing_header().""" + warnings.warn( + "The browser_timing_footer function is deprecated. Please migrate to only using the browser_timing_header api instead.", + DeprecationWarning, + ) + return "" def browser_monitoring_intrinsics(self, obfuscation_key): txn_name = obfuscate(self.path, obfuscation_key) @@ -560,12 +521,12 @@ def browser_monitoring_intrinsics(self, obfuscation_key): if self._settings.browser_monitoring.ssl_for_http is not None: ssl_for_http = self._settings.browser_monitoring.ssl_for_http - intrinsics['sslForHttp'] = ssl_for_http + intrinsics["sslForHttp"] = ssl_for_http return intrinsics -class WSGIHeaderProxy(object): +class WSGIHeaderProxy(): def __init__(self, environ): self.environ = environ self.length = None @@ -573,16 +534,16 @@ def __init__(self, environ): @staticmethod def _to_wsgi(key): key = key.upper() - if key == 'CONTENT-LENGTH': - return 'CONTENT_LENGTH' - elif key == 'CONTENT-TYPE': - return 'CONTENT_TYPE' - return 'HTTP_' + key.replace('-', '_') + if key == "CONTENT-LENGTH": + return "CONTENT_LENGTH" + elif key == "CONTENT-TYPE": + return "CONTENT_TYPE" + return f"HTTP_{key.replace('-', '_')}" @staticmethod def _from_wsgi(key): key = key.lower() - return key[5:].replace('_', '-') + return key[5:].replace("_", "-") def __getitem__(self, key): wsgi_key = self._to_wsgi(key) @@ -590,14 +551,14 @@ def __getitem__(self, key): def __iter__(self): for key in self.environ: - if key == 'CONTENT_LENGTH': - yield 'content-length', self.environ['CONTENT_LENGTH'] - elif key == 'CONTENT_TYPE': - yield 'content-type', self.environ['CONTENT_TYPE'] - elif key == 'HTTP_CONTENT_LENGTH' or key == 'HTTP_CONTENT_TYPE': + if key == "CONTENT_LENGTH": + yield "content-length", self.environ["CONTENT_LENGTH"] + elif key == "CONTENT_TYPE": + yield "content-type", self.environ["CONTENT_TYPE"] + elif key in ("HTTP_CONTENT_LENGTH", "HTTP_CONTENT_TYPE"): # These keys are illegal and should be ignored continue - elif key.startswith('HTTP_'): + elif key.startswith("HTTP_"): yield self._from_wsgi(key), self.environ[key] def __len__(self): @@ -607,11 +568,9 @@ def __len__(self): class WSGIWebTransaction(WebTransaction): - - MOD_WSGI_HEADERS = ('mod_wsgi.request_start', 'mod_wsgi.queue_start') + MOD_WSGI_HEADERS = ("mod_wsgi.request_start", "mod_wsgi.queue_start") def __init__(self, application, environ, source=None): - # The web transaction can be enabled/disabled by # the value of the variable "newrelic.enabled" # in the WSGI environ dictionary. We need to check @@ -621,17 +580,20 @@ def __init__(self, application, environ, source=None): # base class making the decision based on whether # application or agent as a whole are enabled. - enabled = _lookup_environ_setting(environ, - 'newrelic.enabled', None) + enabled = _lookup_environ_setting(environ, "newrelic.enabled", None) # Initialise the common transaction base class. super(WSGIWebTransaction, self).__init__( - application, name=None, port=environ.get('SERVER_PORT'), - request_method=environ.get('REQUEST_METHOD'), - query_string=environ.get('QUERY_STRING'), + application, + name=None, + port=environ.get("SERVER_PORT"), + request_method=environ.get("REQUEST_METHOD"), + query_string=environ.get("QUERY_STRING"), headers=iter(WSGIHeaderProxy(environ)), - enabled=enabled, source=source) + enabled=enabled, + source=source, + ) # Disable transactions for websocket connections. # Also disable autorum if this is a websocket. This is a good idea for @@ -656,21 +618,17 @@ def __init__(self, application, environ, source=None): # Check for override settings from WSGI environ. - self.background_task = _lookup_environ_setting(environ, - 'newrelic.set_background_task', False) - - self.ignore_transaction = _lookup_environ_setting(environ, - 'newrelic.ignore_transaction', False) - self.suppress_apdex = _lookup_environ_setting(environ, - 'newrelic.suppress_apdex_metric', False) - self.suppress_transaction_trace = _lookup_environ_setting(environ, - 'newrelic.suppress_transaction_trace', False) - self.capture_params = _lookup_environ_setting(environ, - 'newrelic.capture_request_params', - settings.capture_params) - self.autorum_disabled = _lookup_environ_setting(environ, - 'newrelic.disable_browser_autorum', - not settings.browser_monitoring.auto_instrument) + self.background_task = _lookup_environ_setting(environ, "newrelic.set_background_task", False) + + self.ignore_transaction = _lookup_environ_setting(environ, "newrelic.ignore_transaction", False) + self.suppress_apdex = _lookup_environ_setting(environ, "newrelic.suppress_apdex_metric", False) + self.suppress_transaction_trace = _lookup_environ_setting(environ, "newrelic.suppress_transaction_trace", False) + self.capture_params = _lookup_environ_setting( + environ, "newrelic.capture_request_params", settings.capture_params + ) + self.autorum_disabled = _lookup_environ_setting( + environ, "newrelic.disable_browser_autorum", not settings.browser_monitoring.auto_instrument + ) # Make sure that if high security mode is enabled that # capture of request params is still being disabled. @@ -697,17 +655,17 @@ def __init__(self, application, environ, source=None): # due to use of REST style URL concepts or # otherwise. - request_uri = environ.get('REQUEST_URI', None) + request_uri = environ.get("REQUEST_URI", None) if request_uri is None: # The gunicorn WSGI server uses RAW_URI instead # of the more typical REQUEST_URI used by Apache # and other web servers. - request_uri = environ.get('RAW_URI', None) + request_uri = environ.get("RAW_URI", None) - script_name = environ.get('SCRIPT_NAME', None) - path_info = environ.get('PATH_INFO', None) + script_name = environ.get("SCRIPT_NAME", None) + path_info = environ.get("PATH_INFO", None) self._request_uri = request_uri @@ -728,13 +686,13 @@ def __init__(self, application, environ, source=None): else: path = script_name + path_info - self.set_transaction_name(path, 'Uri', priority=1) + self.set_transaction_name(path, "Uri", priority=1) if self._request_uri is None: self._request_uri = path else: if self._request_uri is not None: - self.set_transaction_name(self._request_uri, 'Uri', priority=1) + self.set_transaction_name(self._request_uri, "Uri", priority=1) # mod_wsgi sets its own distinct variables for queue time # automatically. Initially it set mod_wsgi.queue_start, @@ -758,7 +716,7 @@ def __init__(self, application, environ, source=None): continue try: - if value.startswith('t='): + if value.startswith("t="): try: self.queue_start = _parse_time_stamp(float(value[2:])) except Exception: @@ -773,58 +731,40 @@ def __init__(self, application, environ, source=None): pass def __exit__(self, exc, value, tb): - self.record_custom_metric('Python/WSGI/Input/Bytes', - self._bytes_read) - self.record_custom_metric('Python/WSGI/Input/Time', - self.read_duration) - self.record_custom_metric('Python/WSGI/Input/Calls/read', - self._calls_read) - self.record_custom_metric('Python/WSGI/Input/Calls/readline', - self._calls_readline) - self.record_custom_metric('Python/WSGI/Input/Calls/readlines', - self._calls_readlines) - - self.record_custom_metric('Python/WSGI/Output/Bytes', - self._bytes_sent) - self.record_custom_metric('Python/WSGI/Output/Time', - self.sent_duration) - self.record_custom_metric('Python/WSGI/Output/Calls/yield', - self._calls_yield) - self.record_custom_metric('Python/WSGI/Output/Calls/write', - self._calls_write) + self.record_custom_metric("Python/WSGI/Input/Bytes", self._bytes_read) + self.record_custom_metric("Python/WSGI/Input/Time", self.read_duration) + self.record_custom_metric("Python/WSGI/Input/Calls/read", self._calls_read) + self.record_custom_metric("Python/WSGI/Input/Calls/readline", self._calls_readline) + self.record_custom_metric("Python/WSGI/Input/Calls/readlines", self._calls_readlines) + + self.record_custom_metric("Python/WSGI/Output/Bytes", self._bytes_sent) + self.record_custom_metric("Python/WSGI/Output/Time", self.sent_duration) + self.record_custom_metric("Python/WSGI/Output/Calls/yield", self._calls_yield) + self.record_custom_metric("Python/WSGI/Output/Calls/write", self._calls_write) return super(WSGIWebTransaction, self).__exit__(exc, value, tb) def _update_agent_attributes(self): # Add WSGI agent attributes if self.read_duration != 0: - self._add_agent_attribute('wsgi.input.seconds', - self.read_duration) + self._add_agent_attribute("wsgi.input.seconds", self.read_duration) if self._bytes_read != 0: - self._add_agent_attribute('wsgi.input.bytes', - self._bytes_read) + self._add_agent_attribute("wsgi.input.bytes", self._bytes_read) if self._calls_read != 0: - self._add_agent_attribute('wsgi.input.calls.read', - self._calls_read) + self._add_agent_attribute("wsgi.input.calls.read", self._calls_read) if self._calls_readline != 0: - self._add_agent_attribute('wsgi.input.calls.readline', - self._calls_readline) + self._add_agent_attribute("wsgi.input.calls.readline", self._calls_readline) if self._calls_readlines != 0: - self._add_agent_attribute('wsgi.input.calls.readlines', - self._calls_readlines) + self._add_agent_attribute("wsgi.input.calls.readlines", self._calls_readlines) if self.sent_duration != 0: - self._add_agent_attribute('wsgi.output.seconds', - self.sent_duration) + self._add_agent_attribute("wsgi.output.seconds", self.sent_duration) if self._bytes_sent != 0: - self._add_agent_attribute('wsgi.output.bytes', - self._bytes_sent) + self._add_agent_attribute("wsgi.output.bytes", self._bytes_sent) if self._calls_write != 0: - self._add_agent_attribute('wsgi.output.calls.write', - self._calls_write) + self._add_agent_attribute("wsgi.output.calls.write", self._calls_write) if self._calls_yield != 0: - self._add_agent_attribute('wsgi.output.calls.yield', - self._calls_yield) + self._add_agent_attribute("wsgi.output.calls.yield", self._calls_yield) return super(WSGIWebTransaction, self)._update_agent_attributes() @@ -842,20 +782,28 @@ def process_response(self, status, response_headers, *args): # would raise as a 500 for WSGI applications). try: - status = status.split(' ', 1)[0] + status = status.split(" ", 1)[0] except Exception: status = None - return super(WSGIWebTransaction, self).process_response( - status, response_headers) - - -def WebTransactionWrapper(wrapped, application=None, name=None, group=None, - scheme=None, host=None, port=None, request_method=None, - request_path=None, query_string=None, headers=None, source=None): - + return super(WSGIWebTransaction, self).process_response(status, response_headers) + + +def WebTransactionWrapper( + wrapped, + application=None, + name=None, + group=None, + scheme=None, + host=None, + port=None, + request_method=None, + request_path=None, + query_string=None, + headers=None, + source=None, +): def wrapper(wrapped, instance, args, kwargs): - if type(application) != Application: _application = application_instance(application) else: @@ -935,7 +883,6 @@ def wrapper(wrapped, instance, args, kwargs): else: _headers = headers - proxy = async_proxy(wrapped) source_arg = source or wrapped @@ -943,17 +890,37 @@ def wrapper(wrapped, instance, args, kwargs): def create_transaction(transaction): if transaction: return None - return WebTransaction( _application, _name, _group, - _scheme, _host, _port, _request_method, - _request_path, _query_string, _headers, source=source_arg) + return WebTransaction( + _application, + _name, + _group, + _scheme, + _host, + _port, + _request_method, + _request_path, + _query_string, + _headers, + source=source_arg, + ) if proxy: context_manager = TransactionContext(create_transaction) return proxy(wrapped(*args, **kwargs), context_manager) transaction = WebTransaction( - _application, _name, _group, _scheme, _host, _port, - _request_method, _request_path, _query_string, _headers, source=source_arg) + _application, + _name, + _group, + _scheme, + _host, + _port, + _request_method, + _request_path, + _query_string, + _headers, + source=source_arg, + ) transaction = create_transaction(current_transaction(active_only=False)) @@ -966,22 +933,50 @@ def create_transaction(transaction): return FunctionWrapper(wrapped, wrapper) -def web_transaction(application=None, name=None, group=None, - scheme=None, host=None, port=None, request_method=None, - request_path=None, query_string=None, headers=None): - - return functools.partial(WebTransactionWrapper, - application=application, name=name, group=group, - scheme=scheme, host=host, port=port, request_method=request_method, - request_path=request_path, query_string=query_string, - headers=headers) - - -def wrap_web_transaction(module, object_path, - application=None, name=None, group=None, - scheme=None, host=None, port=None, request_method=None, - request_path=None, query_string=None, headers=None): - - return wrap_object(module, object_path, WebTransactionWrapper, - (application, name, group, scheme, host, port, request_method, - request_path, query_string, headers)) +def web_transaction( + application=None, + name=None, + group=None, + scheme=None, + host=None, + port=None, + request_method=None, + request_path=None, + query_string=None, + headers=None, +): + return functools.partial( + WebTransactionWrapper, + application=application, + name=name, + group=group, + scheme=scheme, + host=host, + port=port, + request_method=request_method, + request_path=request_path, + query_string=query_string, + headers=headers, + ) + + +def wrap_web_transaction( + module, + object_path, + application=None, + name=None, + group=None, + scheme=None, + host=None, + port=None, + request_method=None, + request_path=None, + query_string=None, + headers=None, +): + return wrap_object( + module, + object_path, + WebTransactionWrapper, + (application, name, group, scheme, host, port, request_method, request_path, query_string, headers), + ) diff --git a/newrelic/api/wsgi_application.py b/newrelic/api/wsgi_application.py index 67338cbddd..874665e047 100644 --- a/newrelic/api/wsgi_application.py +++ b/newrelic/api/wsgi_application.py @@ -25,12 +25,11 @@ from newrelic.api.web_transaction import WSGIWebTransaction from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import FunctionWrapper, wrap_object -from newrelic.packages import six _logger = logging.getLogger(__name__) -class _WSGIApplicationIterable(object): +class _WSGIApplicationIterable(): def __init__(self, transaction, generator): self.transaction = transaction self.generator = generator @@ -78,7 +77,6 @@ def close(self): try: with FunctionTrace(name="Finalize", group="Python/WSGI"): - if isinstance(self.generator, _WSGIApplicationMiddleware): self.generator.close() @@ -97,7 +95,7 @@ def close(self): self.closed = True -class _WSGIInputWrapper(object): +class _WSGIInputWrapper(): def __init__(self, transaction, input): self.__transaction = transaction self.__input = input @@ -152,8 +150,7 @@ def readlines(self, *args, **kwargs): return lines -class _WSGIApplicationMiddleware(object): - +class _WSGIApplicationMiddleware(): # This is a WSGI middleware for automatically inserting RUM into # HTML responses. It only works for where a WSGI application is # returning response content via a iterable/generator. It does not @@ -204,16 +201,7 @@ def process_data(self, data): # works then we are done, else we move to next phase of # buffering up content until we find the body element. - def html_to_be_inserted(): - header = self.transaction.browser_timing_header() - - if not header: - return b"" - - footer = self.transaction.browser_timing_footer() - - return six.b(header) + six.b(footer) - + html_to_be_inserted = lambda: self.transaction.browser_timing_header().encode("latin-1") if not self.response_data: modified = insert_html_snippet(data, html_to_be_inserted) @@ -340,7 +328,6 @@ def start_response(self, status, response_headers, *args): # Also check whether RUM insertion has already occurred. if self.transaction.autorum_disabled or self.transaction.rum_header_generated: - self.flush_headers() self.pass_through = True @@ -360,7 +347,7 @@ def start_response(self, status, response_headers, *args): content_encoding = None content_disposition = None - for (name, value) in response_headers: + for name, value in response_headers: _name = name.lower() if _name == "content-length": @@ -508,24 +495,20 @@ def __iter__(self): def WSGIApplicationWrapper(wrapped, application=None, name=None, group=None, framework=None, dispatcher=None): - - # Python 2 does not allow rebinding nonlocal variables, so to fix this - # framework must be stored in list so it can be edited by closure. - _framework = [framework] - def get_framework(): """Used to delay imports by passing framework as a callable.""" - framework = _framework[0] + + # Use same framework variable as closure + nonlocal framework + if isinstance(framework, tuple) or framework is None: return framework if callable(framework): framework = framework() - _framework[0] = framework if framework is not None and not isinstance(framework, tuple): framework = (framework, None) - _framework[0] = framework return framework @@ -649,7 +632,6 @@ def _args(environ, start_response, *args, **kwargs): transaction.set_transaction_name(name, group, priority=1) def _start_response(status, response_headers, *args): - additional_headers = transaction.process_response(status, response_headers, *args) _write = start_response(status, response_headers + additional_headers, *args) diff --git a/newrelic/bootstrap/sitecustomize.py b/newrelic/bootstrap/sitecustomize.py index 6640af0e99..894d58539f 100644 --- a/newrelic/bootstrap/sitecustomize.py +++ b/newrelic/bootstrap/sitecustomize.py @@ -15,14 +15,11 @@ import os import sys import time +from importlib.machinery import PathFinder # Define some debug logging routines to help sort out things when this # all doesn't work as expected. - -# Avoiding additional imports by defining PY2 manually -PY2 = sys.version_info[0] == 2 - startup_debug = os.environ.get("NEW_RELIC_STARTUP_DEBUG", "off").lower() in ("on", "true", "1") @@ -31,10 +28,18 @@ def log_message(text, *args, **kwargs): if startup_debug or critical: text = text % args timestamp = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) - sys.stdout.write("NEWRELIC: %s (%d) - %s\n" % (timestamp, os.getpid(), text)) + sys.stdout.write(f"NEWRELIC: {timestamp} ({os.getpid()}) - {text}\n") sys.stdout.flush() +def del_sys_path_entry(path): + if path and path in sys.path: + try: + del sys.path[sys.path.index(path)] + except Exception: + pass + + log_message("New Relic Bootstrap (%s)", __file__) log_message("working_directory = %r", os.getcwd()) @@ -70,36 +75,19 @@ def log_message(text, *args, **kwargs): # the search, and then load what was found. boot_directory = os.path.dirname(__file__) -root_directory = os.path.dirname(os.path.dirname(boot_directory)) - -log_message("root_directory = %r", root_directory) log_message("boot_directory = %r", boot_directory) -path = list(sys.path) - -if boot_directory in path: - del path[path.index(boot_directory)] +del_sys_path_entry(boot_directory) try: - if PY2: - import imp - - module_spec = imp.find_module("sitecustomize", path) - else: - from importlib.machinery import PathFinder - - module_spec = PathFinder.find_spec("sitecustomize", path=path) - + module_spec = PathFinder.find_spec("sitecustomize", path=sys.path) except ImportError: pass else: if module_spec is not None: # Import error not raised in importlib log_message("sitecustomize = %r", module_spec) - if PY2: - imp.load_module("sitecustomize", *module_spec) - else: - module_spec.loader.load_module("sitecustomize") + module_spec.loader.load_module("sitecustomize") # Because the PYTHONPATH environment variable has been amended and the # bootstrap directory added, if a Python application creates a sub @@ -118,11 +106,15 @@ def log_message(text, *args, **kwargs): python_prefix_matches = expected_python_prefix == actual_python_prefix python_version_matches = expected_python_version == actual_python_version +k8s_operator_enabled = os.environ.get("NEW_RELIC_K8S_OPERATOR_ENABLED", "off").lower() in ("on", "true", "1") +azure_operator_enabled = os.environ.get("NEW_RELIC_AZURE_OPERATOR_ENABLED", "off").lower() in ("on", "true", "1") log_message("python_prefix_matches = %r", python_prefix_matches) log_message("python_version_matches = %r", python_version_matches) +log_message("k8s_operator_enabled = %r", k8s_operator_enabled) +log_message("azure_operator_enabled = %r", azure_operator_enabled) -if python_prefix_matches and python_version_matches: +if k8s_operator_enabled or azure_operator_enabled or (python_prefix_matches and python_version_matches): # We also need to skip agent initialisation if neither the license # key or config file environment variables are set. We do this as # some people like to use a common startup script which always uses @@ -130,40 +122,80 @@ def log_message(text, *args, **kwargs): # actually run based on the presence of the environment variables. license_key = os.environ.get("NEW_RELIC_LICENSE_KEY", None) - + developer_mode = os.environ.get("NEW_RELIC_DEVELOPER_MODE", "off").lower() in ("on", "true", "1") config_file = os.environ.get("NEW_RELIC_CONFIG_FILE", None) environment = os.environ.get("NEW_RELIC_ENVIRONMENT", None) - - log_message("initialize_agent = %r", bool(license_key or config_file)) - - if license_key or config_file: - # When installed as an egg with buildout, the root directory for - # packages is not listed in sys.path and scripts instead set it - # after Python has started up. This will cause importing of - # 'newrelic' module to fail. What we do is see if the root - # directory where the package is held is in sys.path and if not - # insert it. For good measure we remove it after having imported - # 'newrelic' module to reduce chance that will cause any issues. - # If it is a buildout created script, it will replace the whole - # sys.path again later anyway. - - do_insert_path = root_directory not in sys.path - if do_insert_path: - sys.path.insert(0, root_directory) - - import newrelic.config - - log_message("agent_version = %r", newrelic.version) - - if do_insert_path: + initialize_agent = bool(license_key or config_file or developer_mode) + + log_message("initialize_agent = %r", initialize_agent) + + if initialize_agent: + if k8s_operator_enabled or azure_operator_enabled: + # When installed with either the kubernetes operator or the + # azure operator functionality enabled, we need to attempt to + # find a distribution from our initcontainer that matches the + # current environment. For wheels, this is platform dependent and we + # rely on pip to identify the correct wheel to use. If no suitable + # wheel can be found, we will fall back to the sdist and disable + # extensions. Once the appropriate distribution is found, we import + # it and leave the entry in sys.path. This allows users to import + # the 'newrelic' module later and use our APIs in their code. try: - del sys.path[sys.path.index(root_directory)] - except Exception: - pass + sys.path.insert(0, boot_directory) + # Will use the same file for k8s as well as Azure since the functionality + # will remain the same. File may be renamed in the near future. + from newrelic_k8s_operator import find_supported_newrelic_distribution + finally: + del_sys_path_entry(boot_directory) + + new_relic_path = find_supported_newrelic_distribution() + do_insert_path = True + else: + # When installed as an egg with buildout, the root directory for + # packages is not listed in sys.path and scripts instead set it + # after Python has started up. This will cause importing of + # 'newrelic' module to fail. What we do is see if the root + # directory where the package is held is in sys.path and if not + # insert it. For good measure we remove it after having imported + # 'newrelic' module to reduce chance that will cause any issues. + # If it is a buildout created script, it will replace the whole + # sys.path again later anyway. + root_directory = os.path.dirname(os.path.dirname(boot_directory)) + log_message("root_directory = %r", root_directory) + + new_relic_path = root_directory + do_insert_path = root_directory not in sys.path + + # Now that the appropriate location of the module has been identified, + # either by the kubernetes operator or this script, we are ready to import + # the 'newrelic' module to make it available in sys.modules. If the location + # containing it was not found on sys.path, do_insert_path will be set and + # the location will be inserted into sys.path. The module is then imported, + # and the sys.path entry is removed afterwards to reduce chance that will + # cause any issues. + + log_message(f"new_relic_path = {new_relic_path!r}") + log_message(f"do_insert_path = {do_insert_path!r}") + + try: + if do_insert_path: + sys.path.insert(0, new_relic_path) + + import newrelic + + log_message("agent_version = %r", newrelic.version) + finally: + if do_insert_path: + del_sys_path_entry(new_relic_path) # Finally initialize the agent. + import newrelic.config newrelic.config.initialize(config_file, environment) + else: + log_message( + "New Relic could not start due to missing configuration. Either NEW_RELIC_LICENSE_KEY or NEW_RELIC_CONFIG_FILE are required." + ) else: log_message( """New Relic could not start because the newrelic-admin script was called from a Python installation that is different from the Python installation that is currently running. To fix this problem, call the newrelic-admin script from the Python installation that is currently running (details below). diff --git a/newrelic/common/_monotonic.c b/newrelic/common/_monotonic.c index b393c51e1f..95f28f1a05 100644 --- a/newrelic/common/_monotonic.c +++ b/newrelic/common/_monotonic.c @@ -121,7 +121,6 @@ static PyMethodDef monotonic_methods[] = { { NULL, NULL } }; -#if PY_MAJOR_VERSION >= 3 static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "_monotonic", /* m_name */ @@ -133,18 +132,13 @@ static struct PyModuleDef moduledef = { NULL, /* m_clear */ NULL, /* m_free */ }; -#endif static PyObject * moduleinit(void) { PyObject *module; -#if PY_MAJOR_VERSION >= 3 module = PyModule_Create(&moduledef); -#else - module = Py_InitModule3("_monotonic", monotonic_methods, NULL); -#endif if (module == NULL) return NULL; @@ -152,17 +146,10 @@ moduleinit(void) return module; } -#if PY_MAJOR_VERSION < 3 -PyMODINIT_FUNC init_monotonic(void) -{ - moduleinit(); -} -#else PyMODINIT_FUNC PyInit__monotonic(void) { return moduleinit(); } -#endif /* ------------------------------------------------------------------------- */ diff --git a/newrelic/common/agent_http.py b/newrelic/common/agent_http.py index e9d9a00aac..eb5b210d33 100644 --- a/newrelic/common/agent_http.py +++ b/newrelic/common/agent_http.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import sys import time @@ -23,7 +21,11 @@ import newrelic.packages.urllib3 as urllib3 from newrelic import version from newrelic.common import certs -from newrelic.common.encoding_utils import json_decode, json_encode +from newrelic.common.encoding_utils import ( + json_decode, + json_encode, + obfuscate_license_key, +) from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import patch_function_wrapper from newrelic.core.internal_metrics import internal_count_metric, internal_metric @@ -33,7 +35,7 @@ from ssl import get_default_verify_paths except ImportError: - class _DEFAULT_CERT_PATH(object): + class _DEFAULT_CERT_PATH(): cafile = None capath = None @@ -41,15 +43,14 @@ def get_default_verify_paths(): return _DEFAULT_CERT_PATH +HEADER_AUDIT_LOGGING_DENYLIST = frozenset(("x-api-key", "api-key")) + + # User agent string that must be used in all requests. The data collector # does not rely on this, but is used to target specific agents if there # is a problem with data collector handling requests. -USER_AGENT = "NewRelic-PythonAgent/%s (Python %s %s)" % ( - version, - sys.version.split()[0], - sys.platform, -) +USER_AGENT = f"NewRelic-PythonAgent/{version} (Python {sys.version.split()[0]} {sys.platform})" # This is a monkey patch for urllib3 + python3.6 + gevent/eventlet. @@ -72,7 +73,7 @@ def _urllib3_ssl_recursion_workaround(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) -class BaseClient(object): +class BaseClient(): AUDIT_LOG_ID = 0 def __init__( @@ -92,6 +93,7 @@ def __init__( compression_method="gzip", max_payload_size_in_bytes=1000000, audit_log_fp=None, + default_content_encoding_header="Identity", ): self._audit_log_fp = audit_log_fp @@ -112,32 +114,39 @@ def _supportability_request(params, payload, body, compression_time): pass @classmethod - def log_request( - cls, fp, method, url, params, payload, headers, body=None, compression_time=None - ): + def log_request(cls, fp, method, url, params, payload, headers, body=None, compression_time=None): cls._supportability_request(params, payload, body, compression_time) if not fp: return + # Obfuscate license key from headers and URL params + if headers: + headers = {k: obfuscate_license_key(v) if k.lower() in HEADER_AUDIT_LOGGING_DENYLIST else v for k, v in headers.items()} + + if params and "license_key" in params: + params = params.copy() + params["license_key"] = obfuscate_license_key(params["license_key"]) + # Maintain a global AUDIT_LOG_ID attached to all class instances # NOTE: this is not thread safe so this class cannot be used # across threads when audit logging is on cls.AUDIT_LOG_ID += 1 print( - "TIME: %r" % time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), file=fp, + f"TIME: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())!r}", + file=fp, ) print(file=fp) - print("ID: %r" % cls.AUDIT_LOG_ID, file=fp) + print(f"ID: {cls.AUDIT_LOG_ID!r}", file=fp) print(file=fp) - print("PID: %r" % os.getpid(), file=fp) + print(f"PID: {os.getpid()!r}", file=fp) print(file=fp) - print("URL: %r" % url, file=fp) + print(f"URL: {url!r}", file=fp) print(file=fp) - print("PARAMS: %r" % params, file=fp) + print(f"PARAMS: {params!r}", file=fp) print(file=fp) - print("HEADERS: %r" % headers, file=fp) + print(f"HEADERS: {headers!r}", file=fp) print(file=fp) print("DATA:", end=" ", file=fp) @@ -178,20 +187,18 @@ def log_response(cls, fp, log_id, status, headers, data, connection="direct"): except Exception: result = data - print( - "TIME: %r" % time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()), file=fp - ) + print(f"TIME: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())!r}", file=fp) print(file=fp) - print("ID: %r" % log_id, file=fp) + print(f"ID: {log_id!r}", file=fp) print(file=fp) - print("PID: %r" % os.getpid(), file=fp) + print(f"PID: {os.getpid()!r}", file=fp) print(file=fp) if exc_info: - print("Exception: %r" % exc_info[1], file=fp) + print(f"Exception: {exc_info[1]!r}", file=fp) print(file=fp) else: - print("STATUS: %r" % status, file=fp) + print(f"STATUS: {status!r}", file=fp) print(file=fp) print("HEADERS:", end=" ", file=fp) pprint(dict(headers), stream=fp) @@ -219,9 +226,7 @@ def send_request( class HttpClient(BaseClient): CONNECTION_CLS = urllib3.HTTPSConnectionPool PREFIX_SCHEME = "https://" - BASE_HEADERS = urllib3.make_headers( - keep_alive=True, accept_encoding=True, user_agent=USER_AGENT - ) + BASE_HEADERS = urllib3.make_headers(keep_alive=True, accept_encoding=True, user_agent=USER_AGENT) def __init__( self, @@ -240,6 +245,7 @@ def __init__( compression_method="gzip", max_payload_size_in_bytes=1000000, audit_log_fp=None, + default_content_encoding_header="Identity", ): self._host = host port = self._port = port @@ -248,6 +254,7 @@ def __init__( self._compression_method = compression_method self._max_payload_size_in_bytes = max_payload_size_in_bytes self._audit_log_fp = audit_log_fp + self._default_content_encoding_header = default_content_encoding_header self._prefix = "" @@ -263,11 +270,9 @@ def __init__( # If there is no resolved cafile, assume the bundled certs are # required and report this condition as a supportability metric. - if not verify_path.cafile: + if not verify_path.cafile and not verify_path.capath: ca_bundle_path = certs.where() - internal_metric( - "Supportability/Python/Certificate/BundleRequired", 1 - ) + internal_metric("Supportability/Python/Certificate/BundleRequired", 1) if ca_bundle_path: if os.path.isdir(ca_bundle_path): @@ -279,11 +284,13 @@ def __init__( connection_kwargs["cert_reqs"] = "NONE" proxy = self._parse_proxy( - proxy_scheme, proxy_host, proxy_port, proxy_user, proxy_pass, - ) - proxy_headers = ( - proxy and proxy.auth and urllib3.make_headers(proxy_basic_auth=proxy.auth) + proxy_scheme, + proxy_host, + proxy_port, + proxy_user, + proxy_pass, ) + proxy_headers = proxy and proxy.auth and urllib3.make_headers(proxy_basic_auth=proxy.auth) if proxy: if self.CONNECTION_CLS.scheme == "https" and proxy.scheme != "https": @@ -292,7 +299,7 @@ def __init__( else: self._host = proxy.host self._port = proxy.port or 443 - self._prefix = self.PREFIX_SCHEME + host + ":" + str(port) + self._prefix = f"{self.PREFIX_SCHEME + host}:{str(port)}" urlopen_kwargs["assert_same_host"] = False if proxy_headers: self._headers.update(proxy_headers) @@ -317,7 +324,7 @@ def _parse_proxy(scheme, host, port, username, password): else: auth = username if auth and password is not None: - auth = auth + ":" + password + auth = f"{auth}:{password}" # Host must be defined if not host: @@ -343,15 +350,9 @@ def _connection(self): if self._connection_attr: return self._connection_attr - retries = urllib3.Retry( - total=False, connect=None, read=None, redirect=0, status=None - ) + retries = urllib3.Retry(total=False, connect=None, read=None, redirect=0, status=None) self._connection_attr = self.CONNECTION_CLS( - self._host, - self._port, - strict=True, - retries=retries, - **self._connection_kwargs + self._host, self._port, strict=True, retries=retries, **self._connection_kwargs ) return self._connection_attr @@ -372,11 +373,9 @@ def log_request( compression_time=None, ): if not self._prefix: - url = self.CONNECTION_CLS.scheme + "://" + self._host + url + url = f"{self.CONNECTION_CLS.scheme}://{self._host}{url}" - return super(HttpClient, self).log_request( - fp, method, url, params, payload, headers, body, compression_time - ) + return super(HttpClient, self).log_request(fp, method, url, params, payload, headers, body, compression_time) @staticmethod def _compress(data, method="gzip", level=None): @@ -402,7 +401,7 @@ def send_request( ): if self._proxy: proxy_scheme = self._proxy.scheme or "http" - connection = proxy_scheme + "-proxy" + connection = f"{proxy_scheme}-proxy" else: connection = "direct" @@ -419,11 +418,9 @@ def send_request( method=self._compression_method, level=self._compression_level, ) - content_encoding = self._compression_method - else: - content_encoding = "Identity" - - merged_headers["Content-Encoding"] = content_encoding + merged_headers["Content-Encoding"] = self._compression_method + elif self._default_content_encoding_header: + merged_headers["Content-Encoding"] = self._default_content_encoding_header request_id = self.log_request( self._audit_log_fp, @@ -441,16 +438,16 @@ def send_request( try: response = self._connection.request_encode_url( - method, - path, - fields=params, - body=body, - headers=merged_headers, - **self._urlopen_kwargs + method, path, fields=params, body=body, headers=merged_headers, **self._urlopen_kwargs ) except urllib3.exceptions.HTTPError as e: self.log_response( - self._audit_log_fp, request_id, 0, None, None, connection, + self._audit_log_fp, + request_id, + 0, + None, + None, + connection, ) # All urllib3 HTTP errors should be treated as a network # interface exception. @@ -489,6 +486,7 @@ def __init__( compression_method="gzip", max_payload_size_in_bytes=1000000, audit_log_fp=None, + default_content_encoding_header="Identity", ): proxy = self._parse_proxy(proxy_scheme, proxy_host, None, None, None) if proxy and proxy.scheme == "https": @@ -515,10 +513,11 @@ def __init__( compression_method, max_payload_size_in_bytes, audit_log_fp, + default_content_encoding_header, ) -class SupportabilityMixin(object): +class SupportabilityMixin(): @staticmethod def _supportability_request(params, payload, body, compression_time): # ********* @@ -533,43 +532,34 @@ def _supportability_request(params, payload, body, compression_time): # Compression was applied if compression_time is not None: internal_metric( - "Supportability/Python/Collector/%s/ZLIB/Bytes" % agent_method, + f"Supportability/Python/Collector/{agent_method}/ZLIB/Bytes", len(body), ) + internal_metric("Supportability/Python/Collector/ZLIB/Bytes", len(body)) internal_metric( - "Supportability/Python/Collector/ZLIB/Bytes", len(body) - ) - internal_metric( - "Supportability/Python/Collector/%s/ZLIB/Compress" % agent_method, + f"Supportability/Python/Collector/{agent_method}/ZLIB/Compress", compression_time, ) internal_metric( - "Supportability/Python/Collector/%s/Output/Bytes" % agent_method, + f"Supportability/Python/Collector/{agent_method}/Output/Bytes", len(payload), ) # Top level metric to aggregate overall bytes being sent - internal_metric( - "Supportability/Python/Collector/Output/Bytes", len(payload) - ) + internal_metric("Supportability/Python/Collector/Output/Bytes", len(payload)) @staticmethod def _supportability_response(status, exc, connection="direct"): if exc or not 200 <= status < 300: internal_count_metric("Supportability/Python/Collector/Failures", 1) - internal_count_metric( - "Supportability/Python/Collector/Failures/%s" % connection, 1 - ) + internal_count_metric(f"Supportability/Python/Collector/Failures/{connection}", 1) if exc: internal_count_metric( - "Supportability/Python/Collector/Exception/" - "%s" % callable_name(exc), + f"Supportability/Python/Collector/Exception/{callable_name(exc)}", 1, ) else: - internal_count_metric( - "Supportability/Python/Collector/HTTPError/%d" % status, 1 - ) + internal_count_metric(f"Supportability/Python/Collector/HTTPError/{status}", 1) class ApplicationModeClient(SupportabilityMixin, HttpClient): @@ -578,33 +568,31 @@ class ApplicationModeClient(SupportabilityMixin, HttpClient): class DeveloperModeClient(SupportabilityMixin, BaseClient): RESPONSES = { - "preconnect": {u"redirect_host": u"fake-collector.newrelic.com"}, + "preconnect": {"redirect_host": "fake-collector.newrelic.com"}, "agent_settings": [], "connect": { - u"js_agent_loader": u"", - u"js_agent_file": u"fake-js-agent.newrelic.com/nr-0.min.js", - u"browser_key": u"1234567890", - u"browser_monitoring.loader_version": u"0", - u"beacon": u"fake-beacon.newrelic.com", - u"error_beacon": u"fake-jserror.newrelic.com", - u"apdex_t": 0.5, - u"encoding_key": u"1111111111111111111111111111111111111111", - u"entity_guid": u"DEVELOPERMODEENTITYGUID", - u"agent_run_id": u"1234567", - u"product_level": 50, - u"trusted_account_ids": [12345], - u"trusted_account_key": u"12345", - u"url_rules": [], - u"collect_errors": True, - u"account_id": u"12345", - u"cross_process_id": u"12345#67890", - u"messages": [ - {u"message": u"Reporting to fake collector", u"level": u"INFO"} - ], - u"sampling_rate": 0, - u"collect_traces": True, - u"collect_span_events": True, - u"data_report_period": 60, + "js_agent_loader": "", + "js_agent_file": "fake-js-agent.newrelic.com/nr-0.min.js", + "browser_key": "1234567890", + "browser_monitoring.loader_version": "0", + "beacon": "fake-beacon.newrelic.com", + "error_beacon": "fake-jserror.newrelic.com", + "apdex_t": 0.5, + "encoding_key": "1111111111111111111111111111111111111111", + "entity_guid": "DEVELOPERMODEENTITYGUID", + "agent_run_id": "1234567", + "product_level": 50, + "trusted_account_ids": [12345], + "trusted_account_key": "12345", + "url_rules": [], + "collect_errors": True, + "account_id": "12345", + "cross_process_id": "12345#67890", + "messages": [{"message": "Reporting to fake collector", "level": "INFO"}], + "sampling_rate": 0, + "collect_traces": True, + "collect_span_events": True, + "data_report_period": 60, }, "metric_data": None, "get_agent_commands": [], @@ -632,7 +620,7 @@ def send_request( request_id = self.log_request( self._audit_log_fp, "POST", - "https://fake-collector.newrelic.com" + path, + f"https://fake-collector.newrelic.com{path}", params, payload, headers, @@ -648,7 +636,11 @@ def send_request( payload = {"return_value": result} response_data = json_encode(payload).encode("utf-8") self.log_response( - self._audit_log_fp, request_id, 200, {}, response_data, + self._audit_log_fp, + request_id, + 200, + {}, + response_data, ) return 200, response_data diff --git a/newrelic/common/async_proxy.py b/newrelic/common/async_proxy.py index 56eaba73c2..481f8df416 100644 --- a/newrelic/common/async_proxy.py +++ b/newrelic/common/async_proxy.py @@ -14,7 +14,6 @@ import logging import time -import newrelic.packages.six as six from newrelic.common.coroutine import (is_coroutine_callable, is_asyncio_coroutine, is_generator_function) @@ -26,7 +25,7 @@ CancelledError = None -class TransactionContext(object): +class TransactionContext(): def __init__(self, transaction_init): self.enter_time = None self.transaction = None @@ -102,7 +101,7 @@ def __exit__(self, exc, value, tb): self.transaction.__exit__(exc, value, tb) -class LoopContext(object): +class LoopContext(): def __enter__(self): self.enter_time = time.time() @@ -140,12 +139,8 @@ class GeneratorProxy(Coroutine): def __iter__(self): return self - if six.PY2: - def next(self): - return self.send(None) - else: - def __next__(self): - return self.send(None) + def __next__(self): + return self.send(None) class AwaitableGeneratorProxy(GeneratorProxy): diff --git a/newrelic/common/async_wrapper.py b/newrelic/common/async_wrapper.py index c5f95308da..0c17d68dc3 100644 --- a/newrelic/common/async_wrapper.py +++ b/newrelic/common/async_wrapper.py @@ -12,39 +12,25 @@ # See the License for the specific language governing permissions and # limitations under the License. -import textwrap import functools from newrelic.common.coroutine import ( is_coroutine_callable, is_asyncio_coroutine, is_generator_function, + is_async_generator_function, ) -def evaluate_wrapper(wrapper_string, wrapped, trace): - values = {'wrapper': None, 'wrapped': wrapped, - 'trace': trace, 'functools': functools} - exec(wrapper_string, values) - return values['wrapper'] - - def coroutine_wrapper(wrapped, trace): - - WRAPPER = textwrap.dedent(""" @functools.wraps(wrapped) async def wrapper(*args, **kwargs): with trace: return await wrapped(*args, **kwargs) - """) - - try: - return evaluate_wrapper(WRAPPER, wrapped, trace) - except Exception: - return wrapped + + return wrapper def awaitable_generator_wrapper(wrapped, trace): - WRAPPER = textwrap.dedent(""" import asyncio @functools.wraps(wrapped) @@ -53,30 +39,39 @@ def wrapper(*args, **kwargs): with trace: result = yield from wrapped(*args, **kwargs) return result - """) - try: - return evaluate_wrapper(WRAPPER, wrapped, trace) - except: - return wrapped + return wrapper def generator_wrapper(wrapped, trace): @functools.wraps(wrapped) def wrapper(*args, **kwargs): - g = wrapped(*args, **kwargs) - value = None with trace: - while True: - try: - yielded = g.send(value) - except StopIteration: - break + result = yield from wrapped(*args, **kwargs) + return result + + return wrapper + - try: - value = yield yielded - except BaseException as e: - value = yield g.throw(type(e), e) +def async_generator_wrapper(wrapped, trace): + @functools.wraps(wrapped) + async def wrapper(*args, **kwargs): + g = wrapped(*args, **kwargs) + with trace: + try: + yielded = await g.asend(None) + while True: + try: + sent = yield yielded + except GeneratorExit as e: + await g.aclose() + raise + except BaseException as e: + yielded = await g.athrow(e) + else: + yielded = await g.asend(sent) + except StopAsyncIteration: + return return wrapper @@ -84,6 +79,8 @@ def wrapper(*args, **kwargs): def async_wrapper(wrapped): if is_coroutine_callable(wrapped): return coroutine_wrapper + elif is_async_generator_function(wrapped): + return async_generator_wrapper elif is_generator_function(wrapped): if is_asyncio_coroutine(wrapped): return awaitable_generator_wrapper diff --git a/newrelic/common/coroutine.py b/newrelic/common/coroutine.py index cf4c91f85c..9df83da276 100644 --- a/newrelic/common/coroutine.py +++ b/newrelic/common/coroutine.py @@ -13,24 +13,15 @@ # limitations under the License. import inspect -import newrelic.packages.six as six -if hasattr(inspect, 'iscoroutinefunction'): - def is_coroutine_function(wrapped): - return inspect.iscoroutinefunction(wrapped) -else: - def is_coroutine_function(wrapped): - return False +def is_coroutine_function(wrapped): + return inspect.iscoroutinefunction(wrapped) -if six.PY3: - def is_asyncio_coroutine(wrapped): - """Return True if func is a decorated coroutine function.""" - return getattr(wrapped, '_is_coroutine', None) is not None -else: - def is_asyncio_coroutine(wrapped): - return False +def is_asyncio_coroutine(wrapped): + """Return True if func is a decorated coroutine function.""" + return getattr(wrapped, '_is_coroutine', None) is not None def is_generator_function(wrapped): @@ -43,3 +34,11 @@ def _iscoroutinefunction_tornado(fn): def is_coroutine_callable(wrapped): return is_coroutine_function(wrapped) or is_coroutine_function(getattr(wrapped, "__call__", None)) + + +if hasattr(inspect, 'isasyncgenfunction'): + def is_async_generator_function(wrapped): + return inspect.isasyncgenfunction(wrapped) +else: + def is_async_generator_function(wrapped): + return False diff --git a/newrelic/common/encoding_utils.py b/newrelic/common/encoding_utils.py index ef8624240f..59dabb346c 100644 --- a/newrelic/common/encoding_utils.py +++ b/newrelic/common/encoding_utils.py @@ -29,16 +29,15 @@ import zlib from collections import OrderedDict -from newrelic.packages import six -HEXDIGLC_RE = re.compile('^[0-9a-f]+$') -DELIMITER_FORMAT_RE = re.compile('[ \t]*,[ \t]*') +HEXDIGLC_RE = re.compile("^[0-9a-f]+$") +DELIMITER_FORMAT_RE = re.compile("[ \t]*,[ \t]*") PARENT_TYPE = { - '0': 'App', - '1': 'Browser', - '2': 'Mobile', + "0": "App", + "1": "Browser", + "2": "Mobile", } -BASE64_DECODE_STR = getattr(base64, 'decodestring', None) +BASE64_DECODE_STR = getattr(base64, "decodestring", None) # Functions for encoding/decoding JSON. These wrappers are used in order @@ -48,28 +47,18 @@ # be supplied as key word arguments to allow the wrappers to supply # defaults. + def json_encode(obj, **kwargs): _kwargs = {} # This wrapper function needs to deal with a few issues. # # The first is that when a byte string is provided, we need to - # ensure that it is interpreted as being Latin-1. This is necessary - # as by default JSON will treat it as UTF-8, which means if an - # invalid UTF-8 byte string is provided, a failure will occur when - # encoding the value. - # - # The json.dumps() function in Python 2 had an encoding argument - # which needs to be used to dictate what encoding a byte string - # should be interpreted as being. We need to supply this and set it - # to Latin-1 to avoid the failures if the byte string is not valid - # UTF-8. - # - # For Python 3, it will simply fail if provided any byte string. To - # be compatible with Python 2, we still want to accept them, but as - # before interpret it as being Latin-1. For Python 3 we can only do - # this by overriding the fallback encoder used when a type is - # encountered that the JSON encoder doesn't know what to do with. + # ensure that it is accepted as a string, and interpreted as + # being Latin-1. The default JSON encoder will not accept byte + # strings, so a we can do this by overriding the fallback encoder + # used when a type is encountered that the JSON encoder doesn't + # know what to do with. # # The second issue we want to deal with is allowing generators or # iterables to be supplied and for them to be automatically expanded @@ -79,21 +68,18 @@ def json_encode(obj, **kwargs): # The third is eliminate white space after separators to trim the # size of the data being sent. - if type(b'') is type(''): # NOQA - _kwargs['encoding'] = 'latin-1' - def _encode(o): if isinstance(o, bytes): - return o.decode('latin-1') + return o.decode("latin-1") elif isinstance(o, types.GeneratorType): return list(o) - elif hasattr(o, '__iter__'): + elif hasattr(o, "__iter__"): return list(iter(o)) - raise TypeError(repr(o) + ' is not JSON serializable') + raise TypeError(repr(o) + " is not JSON serializable") - _kwargs['default'] = _encode + _kwargs["default"] = _encode - _kwargs['separators'] = (',', ':') + _kwargs["separators"] = (",", ":") # We still allow supplied arguments to override internal defaults if # necessary, but the caller must be sure they aren't dependent on @@ -111,6 +97,7 @@ def json_decode(s, **kwargs): return json.loads(s, **kwargs) + # Functions for obfuscating/deobfuscating text string based on an XOR # cipher. @@ -124,7 +111,7 @@ def xor_cipher_genkey(key, length=None): """ - return bytearray(key[:length], encoding='ascii') + return bytearray(key[:length], encoding="ascii") def xor_cipher_encrypt(text, key): @@ -164,12 +151,9 @@ def xor_cipher_encrypt_base64(text, key): array using xor_cipher_genkey(). The key cannot be an empty byte array or string. Where the key is shorter than the text to be encrypted, the same key will continually be reapplied in succession. - In Python 2 either a byte string or Unicode string can be provided - for the text input. In the case of a byte string, it will be - interpreted as having Latin-1 encoding. In Python 3 only a Unicode - string can be provided for the text input. Having being encrypted, - the result will then be base64 encoded with the result being a - Unicode string. + In Python 3 only a Unicode string can be provided for the text input. + Having being encrypted, the result will then be base64 encoded with + the result being a Unicode string. """ @@ -190,8 +174,8 @@ def xor_cipher_encrypt_base64(text, key): # isn't UTF-8 and so fail with a Unicode decoding error. if isinstance(text, bytes): - text = text.decode('latin-1') - text = text.encode('utf-8').decode('latin-1') + text = text.decode("latin-1") + text = text.encode("utf-8").decode("latin-1") result = base64.b64encode(bytes(xor_cipher_encrypt(text, key))) @@ -201,10 +185,7 @@ def xor_cipher_encrypt_base64(text, key): # use ASCII when decoding the byte string as base64 encoding only # produces characters within that codeset. - if six.PY3: - return result.decode('ascii') - - return result + return result.decode("ascii") def xor_cipher_decrypt_base64(text, key): @@ -223,7 +204,7 @@ def xor_cipher_decrypt_base64(text, key): result = xor_cipher_decrypt(bytearray(base64.b64decode(text)), key) - return bytes(result).decode('utf-8') + return bytes(result).decode("utf-8") obfuscate = xor_cipher_encrypt_base64 @@ -240,13 +221,13 @@ def unpack_field(field): """ if not isinstance(field, bytes): - field = field.encode('UTF-8') + field = field.encode("UTF-8") - data = getattr(base64, 'decodebytes', BASE64_DECODE_STR)(field) + data = getattr(base64, "decodebytes", BASE64_DECODE_STR)(field) data = zlib.decompress(data) if isinstance(data, bytes): - data = data.decode('Latin-1') + data = data.decode("Latin-1") data = json_decode(data) return data @@ -260,13 +241,13 @@ def generate_path_hash(name, seed): """ - rotated = ((seed << 1) | (seed >> 31)) & 0xffffffff + rotated = ((seed << 1) | (seed >> 31)) & 0xFFFFFFFF if not isinstance(name, bytes): - name = name.encode('UTF-8') + name = name.encode("UTF-8") - path_hash = (rotated ^ int(hashlib.md5(name).hexdigest()[-8:], base=16)) - return '%08x' % path_hash + path_hash = rotated ^ int(hashlib.md5(name).hexdigest()[-8:], base=16) # nosec + return f"{path_hash:08x}" def base64_encode(text): @@ -291,11 +272,11 @@ def base64_encode(text): # and so fail with a Unicode decoding error. if isinstance(text, bytes): - text = text.decode('latin-1') - text = text.encode('utf-8').decode('latin-1') + text = text.decode("latin-1") + text = text.encode("utf-8").decode("latin-1") # Re-encode as utf-8 when passing to b64 encoder - result = base64.b64encode(text.encode('utf-8')) + result = base64.b64encode(text.encode("utf-8")) # The result from base64 encoding will be a byte string but since # dealing with byte strings in Python 2 and Python 3 is quite @@ -303,10 +284,7 @@ def base64_encode(text): # use ASCII when decoding the byte string as base64 encoding only # produces characters within that codeset. - if six.PY3: - return result.decode('ascii') - - return result + return result.decode("ascii") def base64_decode(text): @@ -314,7 +292,7 @@ def base64_decode(text): the decoded text is UTF-8 encoded. """ - return base64.b64decode(text).decode('utf-8') + return base64.b64decode(text).decode("utf-8") def gzip_compress(text): @@ -324,10 +302,10 @@ def gzip_compress(text): """ compressed_data = io.BytesIO() - if six.PY3 and isinstance(text, str): - text = text.encode('utf-8') + if isinstance(text, str): + text = text.encode("utf-8") - with gzip.GzipFile(fileobj=compressed_data, mode='wb') as f: + with gzip.GzipFile(fileobj=compressed_data, mode="wb") as f: f.write(text) return compressed_data.getvalue() @@ -340,7 +318,7 @@ def gzip_decompress(payload): """ data_bytes = io.BytesIO(payload) decoded_data = gzip.GzipFile(fileobj=data_bytes).read() - return decoded_data.decode('utf-8') + return decoded_data.decode("utf-8") def serverless_payload_encode(payload): @@ -356,9 +334,9 @@ def serverless_payload_encode(payload): def ensure_str(s): - if not isinstance(s, six.string_types): + if not isinstance(s, str): try: - s = s.decode('utf-8') + s = s.decode("utf-8") except Exception: return return s @@ -370,8 +348,8 @@ def serverless_payload_decode(text): Python object. """ - if hasattr(text, 'decode'): - text = text.decode('utf-8') + if hasattr(text, "decode"): + text = text.decode("utf-8") decoded_bytes = base64.b64decode(text) uncompressed_data = gzip_decompress(decoded_bytes) @@ -384,8 +362,7 @@ def decode_newrelic_header(encoded_header, encoding_key): decoded_header = None if encoded_header: try: - decoded_header = json_decode(deobfuscate( - encoded_header, encoding_key)) + decoded_header = json_decode(deobfuscate(encoded_header, encoding_key)) except Exception: pass @@ -402,7 +379,6 @@ def convert_to_cat_metadata_value(nr_headers): class DistributedTracePayload(dict): - version = (0, 1) def text(self): @@ -437,18 +413,13 @@ def decode(cls, payload): class W3CTraceParent(dict): - def text(self): - if 'id' in self: - guid = self['id'] + if "id" in self: + guid = self["id"] else: - guid = '{:016x}'.format(random.getrandbits(64)) + guid = f"{random.getrandbits(64):016x}" - return '00-{}-{}-{:02x}'.format( - self['tr'].lower().zfill(32), - guid, - int(self.get('sa', 0)), - ) + return f"00-{self['tr'].lower().zfill(32)}-{guid}-{int(self.get('sa', 0)):02x}" @classmethod def decode(cls, payload): @@ -456,7 +427,7 @@ def decode(cls, payload): if len(payload) < 55: return None - fields = payload.split('-', 4) + fields = payload.split("-", 4) # Expect that there are at least 4 fields if len(fields) < 4: @@ -469,11 +440,11 @@ def decode(cls, payload): return None # Version 255 is invalid - if version == 'ff': + if version == "ff": return None # Expect exactly 4 fields if version 00 - if version == '00' and len(fields) != 4: + if version == "00" and len(fields) != 4: return None # Check field lengths and values @@ -483,18 +454,15 @@ def decode(cls, payload): # trace_id or parent_id of all 0's are invalid trace_id, parent_id = fields[1:3] - if parent_id == '0' * 16 or trace_id == '0' * 32: + if parent_id == "0" * 16 or trace_id == "0" * 32: return None return cls(tr=trace_id, id=parent_id) class W3CTraceState(OrderedDict): - def text(self, limit=32): - return ','.join( - '{}={}'.format(k, v) - for k, v in itertools.islice(self.items(), limit)) + return ",".join(f"{k}={v}" for k, v in itertools.islice(self.items(), limit)) @classmethod def decode(cls, tracestate): @@ -502,9 +470,8 @@ def decode(cls, tracestate): vendors = cls() for entry in entries: - vendor_value = entry.split('=', 2) - if (len(vendor_value) != 2 or - any(len(v) > 256 for v in vendor_value)): + vendor_value = entry.split("=", 2) + if len(vendor_value) != 2 or any(len(v) > 256 for v in vendor_value): continue vendor, value = vendor_value @@ -514,36 +481,24 @@ def decode(cls, tracestate): class NrTraceState(dict): - FIELDS = ('ty', 'ac', 'ap', 'id', 'tx', 'sa', 'pr') + FIELDS = ("ty", "ac", "ap", "id", "tx", "sa", "pr") def text(self): - pr = self.get('pr', '') + pr = self.get("pr", "") if pr: - pr = ('%.6f' % pr).rstrip('0').rstrip('.') - - payload = '-'.join(( - '0-0', - self['ac'], - self['ap'], - self.get('id', ''), - self.get('tx', ''), - '1' if self.get('sa') else '0', - pr, - str(self['ti']), - )) - return '{}@nr={}'.format( - self.get('tk', self['ac']), - payload, - ) + pr = f"{pr:.6f}".rstrip("0").rstrip(".") + + payload = f"0-0-{self['ac']}-{self['ap']}-{self.get('id', '')}-{self.get('tx', '')}-{'1' if self.get('sa') else '0'}-{pr}-{str(self['ti'])}" + return f"{self.get('tk', self['ac'])}@nr={payload}" @classmethod def decode(cls, payload, tk): - fields = payload.split('-', 9) + fields = payload.split("-", 9) if len(fields) >= 9 and all(fields[:4]) and fields[8]: data = cls(tk=tk) try: - data['ti'] = int(fields[8]) + data["ti"] = int(fields[8]) except: return @@ -551,23 +506,85 @@ def decode(cls, payload, tk): if value: data[name] = value - if data['ty'] in PARENT_TYPE: - data['ty'] = PARENT_TYPE[data['ty']] + if data["ty"] in PARENT_TYPE: + data["ty"] = PARENT_TYPE[data["ty"]] else: return - if 'sa' in data: - if data['sa'] == '1': - data['sa'] = True - elif data['sa'] == '0': - data['sa'] = False + if "sa" in data: + if data["sa"] == "1": + data["sa"] = True + elif data["sa"] == "0": + data["sa"] = False else: - data['sa'] = None + data["sa"] = None - if 'pr' in data: + if "pr" in data: try: - data['pr'] = float(fields[7]) + data["pr"] = float(fields[7]) except: - data['pr'] = None + data["pr"] = None return data + + +def capitalize(string): + """Capitalize the first letter of a string.""" + if not string: + return string + elif len(string) == 1: + return string.capitalize() + else: + return f"{string[0].upper()}{string[1:]}" + + +def camel_case(string, upper=False): + """ + Convert a string of snake case to camel case. + + Setting upper=True will capitalize the first letter. Defaults to False, where no change is made to the first letter. + """ + string = ensure_str(string) + split_string = list(string.split("_")) + + if len(split_string) < 2: + if upper: + return capitalize(string) + else: + return string + else: + if upper: + camel_cased_string = "".join([capitalize(substr) for substr in split_string]) + else: + camel_cased_string = split_string[0] + "".join([capitalize(substr) for substr in split_string[1:]]) + + return camel_cased_string + + +_snake_case_re = re.compile(r"([A-Z]+[a-z]*)") + + +def snake_case(string): + """Convert a string of camel case to snake case. Assumes no repeated runs of capital letters.""" + string = ensure_str(string) + if "_" in string: + return string # Don't touch strings that are already snake cased + + return "_".join([s for s in _snake_case_re.split(string) if s]).lower() + + +_obfuscate_license_key_ending = "*" * 32 + + +def obfuscate_license_key(license_key): + """Obfuscate license key to allow it to be printed out.""" + + if not isinstance(license_key, str): + # For non-string values passed in such as None, return the original. + return license_key + elif len(license_key) == 40: + # For valid license keys of length 40, show the first 8 characters and then replace the remainder with **** + return license_key[:8] + _obfuscate_license_key_ending + else: + # For invalid lengths of license key, it's unclear how much is acceptable to show, so fully redact with **** + return "*" * len(license_key) diff --git a/newrelic/common/log_file.py b/newrelic/common/log_file.py index 583e24e5ce..91ffd8ef0a 100644 --- a/newrelic/common/log_file.py +++ b/newrelic/common/log_file.py @@ -76,7 +76,7 @@ def _initialize_file_logging(log_file, log_level): _agent_logger.setLevel(log_level) _agent_logger.debug("Initializing Python agent logging.") - _agent_logger.debug('Log file "%s".' % log_file) + _agent_logger.debug(f'Log file "{log_file}".') def initialize_logging(log_file, log_level): @@ -101,7 +101,7 @@ def initialize_logging(log_file, log_level): except Exception: _initialize_stderr_logging(log_level) - _agent_logger.exception("Falling back to stderr logging as unable to create log file %r." % log_file) + _agent_logger.exception(f"Falling back to stderr logging as unable to create log file {log_file!r}.") _initialized = True diff --git a/newrelic/common/metric_utils.py b/newrelic/common/metric_utils.py new file mode 100644 index 0000000000..ebffe83328 --- /dev/null +++ b/newrelic/common/metric_utils.py @@ -0,0 +1,35 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module implements functions for creating a unique identity from a name and set of tags for use in dimensional metrics. +""" + +from newrelic.core.attribute import process_user_attribute + + +def create_metric_identity(name, tags=None): + if tags: + # Convert dicts to an iterable of tuples, other iterables should already be in this form + if isinstance(tags, dict): + tags = tags.items() + + # Apply attribute system sanitization. + # process_user_attribute returns (None, None) for results that fail sanitization. + # The filter removes these results from the iterable before creating the frozenset. + tags = frozenset(filter(lambda args: args[0] is not None, map(lambda args: process_user_attribute(*args), tags))) + + tags = tags or None # Set empty iterables after filtering to None + + return (name, tags) diff --git a/newrelic/common/object_names.py b/newrelic/common/object_names.py index e37b03315e..d62371acc5 100644 --- a/newrelic/common/object_names.py +++ b/newrelic/common/object_names.py @@ -16,21 +16,13 @@ """ +import builtins import sys import types import inspect import functools -from newrelic.packages import six -if six.PY2: - import exceptions - _exceptions_module = exceptions -elif six.PY3: - import builtins - _exceptions_module = builtins -else: - _exceptions_module = None # Object model terminology for quick reference. # @@ -111,7 +103,7 @@ def _module_name(object): # happens for example with namedtuple classes in Python 3. if mname and mname not in sys.modules: - mname = '<%s>' % mname + mname = f'<{mname}>' # If unable to derive the module name, fallback to unknown. @@ -120,116 +112,7 @@ def _module_name(object): return mname -def _object_context_py2(object): - - cname = None - fname = None - - if inspect.isclass(object) or isinstance(object, type): - # Old and new style class types. - - cname = object.__name__ - - elif inspect.ismethod(object): - # Bound and unbound class methods. In the case of an - # unbound method the im_self attribute will be None. The - # rules around whether im_self is an instance or a class - # type are strange so need to cope with both. - - if object.im_self is not None: - cname = getattr(object.im_self, '__name__', None) - if cname is None: - cname = getattr(object.im_self.__class__, '__name__') - - else: - cname = object.im_class.__name__ - - fname = object.__name__ - - elif inspect.isfunction(object): - # Normal functions and static methods. For a static we - # method don't know of any way of being able to work out - # the name of the class the static method is against. - - fname = object.__name__ - - elif inspect.isbuiltin(object): - # Builtin function. Can also be be bound to class to - # create a method. Uses __self__ instead of im_self. The - # rules around whether __self__ is an instance or a class - # type are strange so need to cope with both. - - if object.__self__ is not None: - cname = getattr(object.__self__, '__name__', None) - if cname is None: - cname = getattr(object.__self__.__class__, '__name__') - - fname = object.__name__ - - elif isinstance(object, types.InstanceType): - # Instances of old style classes. Instances of a class - # don't normally have __name__. Where the object has a - # __name__, assume it is likely going to be a decorator - # implemented as a class and don't use the class name - # else it mucks things up. - - fname = getattr(object, '__name__', None) - - if fname is None: - cname = object.__class__.__name__ - - elif hasattr(object, '__class__'): - # Instances of new style classes. Instances of a class - # don't normally have __name__. Where the object has a - # __name__, assume it is likely going to be a decorator - # implemented as a class and don't use the class name - # else it mucks things up. The exception to this is when - # it is a descriptor and has __objclass__, in which case - # the class name from __objclass__ is used. - - fname = getattr(object, '__name__', None) - - if fname is not None: - if hasattr(object, '__objclass__'): - cname = object.__objclass__.__name__ - elif not hasattr(object, '__get__'): - cname = object.__class__.__name__ - else: - cname = object.__class__.__name__ - - # Calculate the qualified path from the class name and the - # function name. - - path = '' - - if cname: - path = cname - - if fname: - if path: - path += '.' - path += fname - - # Now calculate the name of the module object is defined in. - - owner = None - - if inspect.ismethod(object): - if object.__self__ is not None: - cname = getattr(object.__self__, '__name__', None) - if cname is None: - owner = object.__self__.__class__ # bound method - else: - owner = object.__self__ # class method - - else: - owner = getattr(object, 'im_class', None) # unbound method - - mname = _module_name(owner or object) - - return (mname, path) - -def _object_context_py3(object): +def _object_context(object): if inspect.ismethod(object): @@ -245,7 +128,7 @@ def _object_context_py3(object): if cname is None: cname = getattr(object.__self__.__class__, '__qualname__') - path = '%s.%s' % (cname, object.__name__) + path = f'{cname}.{object.__name__}' else: # For functions, the __qualname__ attribute gives us the name. @@ -297,11 +180,11 @@ def object_context(target): details = getattr(target, '_nr_object_path', None) - # Disallow cache lookup for python 3 methods. In the case where the method + # Disallow cache lookup for methods. In the case where the method # is defined on a parent class, the name of the parent class is incorrectly # returned. Avoid this by recalculating the details each time. - if details and not _is_py3_method(target): + if details and not inspect.ismethod(target): return details # Check whether the object is actually one of our own @@ -319,7 +202,7 @@ def object_context(target): if source: details = getattr(source, '_nr_object_path', None) - if details and not _is_py3_method(source): + if details and not inspect.ismethod(source): return details else: @@ -327,11 +210,7 @@ def object_context(target): # If it wasn't cached we generate the name details and then # attempt to cache them against the object. - - if six.PY3: - details = _object_context_py3(source) - else: - details = _object_context_py2(source) + details = _object_context(source) try: # If the original target is not the same as the source we @@ -395,7 +274,7 @@ def expand_builtin_exception_name(name): # Otherwise, return it unchanged. try: - exception = getattr(_exceptions_module, name) + exception = getattr(builtins, name) except AttributeError: pass else: @@ -404,9 +283,6 @@ def expand_builtin_exception_name(name): return name -def _is_py3_method(target): - return six.PY3 and inspect.ismethod(target) - def parse_exc_info(exc_info): """Parse exc_info and return commonly used strings.""" _, value, _ = exc_info @@ -415,27 +291,14 @@ def parse_exc_info(exc_info): name = value.__class__.__name__ if module: - fullnames = ("%s:%s" % (module, name), "%s.%s" % (module, name)) + fullnames = (f"{module}:{name}", f"{module}.{name}") else: fullnames = (name,) try: - - # Favor unicode in exception messages. - - message = six.text_type(value) - + # Ensure exception messages are strings + message = str(value) except Exception: - try: - - # If exception cannot be represented in unicode, this means - # that it is a byte string encoded with an encoding - # that is not compatible with the default system encoding. - # So, just pass this byte string along. - - message = str(value) - - except Exception: - message = "" % type(value).__name__ + message = f"" return (module, name, fullnames, message) diff --git a/newrelic/common/object_wrapper.py b/newrelic/common/object_wrapper.py index 7d9824fe0c..09c737fd2b 100644 --- a/newrelic/common/object_wrapper.py +++ b/newrelic/common/object_wrapper.py @@ -19,16 +19,19 @@ """ -import sys import inspect - -from newrelic.packages import six - -from newrelic.packages.wrapt import (ObjectProxy as _ObjectProxy, - FunctionWrapper as _FunctionWrapper, - BoundFunctionWrapper as _BoundFunctionWrapper) - -from newrelic.packages.wrapt.wrappers import _FunctionWrapperBase +import warnings + +from newrelic.packages.wrapt import BoundFunctionWrapper as _BoundFunctionWrapper +from newrelic.packages.wrapt import CallableObjectProxy as _CallableObjectProxy +from newrelic.packages.wrapt import FunctionWrapper as _FunctionWrapper +from newrelic.packages.wrapt import ObjectProxy as _ObjectProxy +from newrelic.packages.wrapt import ( # noqa: F401; pylint: disable=W0611 + apply_patch, + resolve_path, + wrap_object, + wrap_object_attribute, +) # We previously had our own pure Python implementation of the generic # object wrapper but we now defer to using the wrapt module as its C @@ -47,28 +50,36 @@ # ObjectProxy or FunctionWrapper should be used going forward. -class _ObjectWrapperBase(object): +class ObjectProxy(_ObjectProxy): + """ + This class provides method overrides for all object wrappers used by the + agent. These methods allow attributes to be defined with the special prefix + _nr_ to be interpretted as attributes on the wrapper, rather than the + wrapped object. Inheriting from the base class wrapt.ObjectProxy preserves + method resolution order (MRO) through multiple inheritance. + (See https://www.python.org/download/releases/2.3/mro/). + """ def __setattr__(self, name, value): - if name.startswith('_nr_'): - name = name.replace('_nr_', '_self_', 1) + if name.startswith("_nr_"): + name = name.replace("_nr_", "_self_", 1) setattr(self, name, value) else: - _ObjectProxy.__setattr__(self, name, value) + super(ObjectProxy, self).__setattr__(name, value) def __getattr__(self, name): - if name.startswith('_nr_'): - name = name.replace('_nr_', '_self_', 1) + if name.startswith("_nr_"): + name = name.replace("_nr_", "_self_", 1) return getattr(self, name) else: - return _ObjectProxy.__getattr__(self, name) + return super(ObjectProxy, self).__getattr__(name) def __delattr__(self, name): - if name.startswith('_nr_'): - name = name.replace('_nr_', '_self_', 1) + if name.startswith("_nr_"): + name = name.replace("_nr_", "_self_", 1) delattr(self, name) else: - _ObjectProxy.__delattr__(self, name) + super(ObjectProxy, self).__delattr__(name) @property def _nr_next_object(self): @@ -79,8 +90,7 @@ def _nr_last_object(self): try: return self._self_last_object except AttributeError: - self._self_last_object = getattr(self.__wrapped__, - '_nr_last_object', self.__wrapped__) + self._self_last_object = getattr(self.__wrapped__, "_nr_last_object", self.__wrapped__) return self._self_last_object @property @@ -96,166 +106,39 @@ def _nr_parent(self): return self._self_parent -class _NRBoundFunctionWrapper(_ObjectWrapperBase, _BoundFunctionWrapper): +class _NRBoundFunctionWrapper(ObjectProxy, _BoundFunctionWrapper): pass -class FunctionWrapper(_ObjectWrapperBase, _FunctionWrapper): +class FunctionWrapper(ObjectProxy, _FunctionWrapper): __bound_function_wrapper__ = _NRBoundFunctionWrapper -class ObjectProxy(_ObjectProxy): - - def __setattr__(self, name, value): - if name.startswith('_nr_'): - name = name.replace('_nr_', '_self_', 1) - setattr(self, name, value) - else: - _ObjectProxy.__setattr__(self, name, value) - - def __getattr__(self, name): - if name.startswith('_nr_'): - name = name.replace('_nr_', '_self_', 1) - return getattr(self, name) - else: - return _ObjectProxy.__getattr__(self, name) - - def __delattr__(self, name): - if name.startswith('_nr_'): - name = name.replace('_nr_', '_self_', 1) - delattr(self, name) - else: - _ObjectProxy.__delattr__(self, name) - - @property - def _nr_next_object(self): - return self.__wrapped__ - - @property - def _nr_last_object(self): - try: - return self._self_last_object - except AttributeError: - self._self_last_object = getattr(self.__wrapped__, - '_nr_last_object', self.__wrapped__) - return self._self_last_object - - -class CallableObjectProxy(ObjectProxy): +class CallableObjectProxy(ObjectProxy, _CallableObjectProxy): + pass - def __call__(self, *args, **kwargs): - return self.__wrapped__(*args, **kwargs) # The ObjectWrapper class needs to be deprecated and removed once all our # own code no longer uses it. It reaches down into what are wrapt internals # at present which shouldn't be doing. - -class ObjectWrapper(_ObjectWrapperBase, _FunctionWrapperBase): - __bound_function_wrapper__ = _NRBoundFunctionWrapper - +class ObjectWrapper(FunctionWrapper): def __init__(self, wrapped, instance, wrapper): - if isinstance(wrapped, classmethod): - binding = 'classmethod' - elif isinstance(wrapped, staticmethod): - binding = 'staticmethod' - else: - binding = 'function' - - super(ObjectWrapper, self).__init__(wrapped, instance, wrapper, - binding=binding) + warnings.warn( + ("The ObjectWrapper API is deprecated. Please use one of ObjectProxy, FunctionWrapper, or CallableObjectProxy instead."), + DeprecationWarning, + ) + super(ObjectWrapper, self).__init__(wrapped, wrapper) -# Helper functions for performing monkey patching. - - -def resolve_path(module, name): - if isinstance(module, six.string_types): - __import__(module) - module = sys.modules[module] - - parent = module - - path = name.split('.') - attribute = path[0] - - original = getattr(parent, attribute) - for attribute in path[1:]: - parent = original - - # We can't just always use getattr() because in doing - # that on a class it will cause binding to occur which - # will complicate things later and cause some things not - # to work. For the case of a class we therefore access - # the __dict__ directly. To cope though with the wrong - # class being given to us, or a method being moved into - # a base class, we need to walk the class hierarchy to - # work out exactly which __dict__ the method was defined - # in, as accessing it from __dict__ will fail if it was - # not actually on the class given. Fallback to using - # getattr() if we can't find it. If it truly doesn't - # exist, then that will fail. - - if inspect.isclass(original): - for cls in inspect.getmro(original): - if attribute in vars(cls): - original = vars(cls)[attribute] - break - else: - original = getattr(original, attribute) - - else: - original = getattr(original, attribute) - - return (parent, attribute, original) - - -def apply_patch(parent, attribute, replacement): - setattr(parent, attribute, replacement) - - -def wrap_object(module, name, factory, args=(), kwargs={}): - (parent, attribute, original) = resolve_path(module, name) - wrapper = factory(original, *args, **kwargs) - apply_patch(parent, attribute, wrapper) - return wrapper - -# Function for apply a proxy object to an attribute of a class instance. -# The wrapper works by defining an attribute of the same name on the -# class which is a descriptor and which intercepts access to the -# instance attribute. Note that this cannot be used on attributes which -# are themselves defined by a property object. - - -class AttributeWrapper(object): - - def __init__(self, attribute, factory, args, kwargs): - self.attribute = attribute - self.factory = factory - self.args = args - self.kwargs = kwargs - - def __get__(self, instance, owner): - value = instance.__dict__[self.attribute] - return self.factory(value, *self.args, **self.kwargs) - - def __set__(self, instance, value): - instance.__dict__[self.attribute] = value - - def __delete__(self, instance): - del instance.__dict__[self.attribute] - - -def wrap_object_attribute(module, name, factory, args=(), kwargs={}): - path, attribute = name.rsplit('.', 1) - parent = resolve_path(module, path)[2] - wrapper = AttributeWrapper(attribute, factory, args, kwargs) - apply_patch(parent, attribute, wrapper) - return wrapper - # Function for creating a decorator for applying to functions, as well as # short cut functions for applying wrapper functions via monkey patching. +# WARNING: These functions are reproduced directly from wrapt, but using +# our FunctionWrapper class which includes the _nr_ attriubte overrides +# that are inherited from our subclass of wrapt.ObjectProxy.These MUST be +# kept in sync with wrapt when upgrading, or drift may introduce bugs. + def function_wrapper(wrapper): def _wrapper(wrapped, instance, args, kwargs): @@ -267,6 +150,7 @@ def _wrapper(wrapped, instance, args, kwargs): else: target_wrapper = wrapper.__get__(instance, type(instance)) return FunctionWrapper(target_wrapped, target_wrapper) + return FunctionWrapper(wrapper, _wrapper) @@ -274,9 +158,10 @@ def wrap_function_wrapper(module, name, wrapper): return wrap_object(module, name, FunctionWrapper, (wrapper,)) -def patch_function_wrapper(module, name): +def patch_function_wrapper(module, name, enabled=None): def _wrapper(wrapper): - return wrap_object(module, name, FunctionWrapper, (wrapper,)) + return wrap_object(module, name, FunctionWrapper, (wrapper, enabled)) + return _wrapper @@ -299,10 +184,14 @@ def _execute(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) finally: setattr(parent, attribute, original) + return FunctionWrapper(target_wrapped, _execute) + return FunctionWrapper(wrapper, _wrapper) + return _decorator + # Generic decorators for performing actions before and after a wrapped # function is called, or modifying the inbound arguments or return value. @@ -315,6 +204,7 @@ def _wrapper(wrapped, instance, args, kwargs): else: function(*args, **kwargs) return wrapped(*args, **kwargs) + return _wrapper @@ -335,6 +225,7 @@ def _wrapper(wrapped, instance, args, kwargs): else: function(*args, **kwargs) return result + return _wrapper @@ -382,6 +273,7 @@ def out_function(function): @function_wrapper def _wrapper(wrapped, instance, args, kwargs): return function(wrapped(*args, **kwargs)) + return _wrapper diff --git a/newrelic/common/package_version_utils.py b/newrelic/common/package_version_utils.py index 13b8168780..9ab213ddd5 100644 --- a/newrelic/common/package_version_utils.py +++ b/newrelic/common/package_version_utils.py @@ -13,6 +13,10 @@ # limitations under the License. import sys +import warnings + +from functools import lru_cache + # Need to account for 4 possible variations of version declaration specified in (rejected) PEP 396 VERSION_ATTRS = ("__version__", "version", "__version_tuple__", "version_tuple") # nosec @@ -67,23 +71,40 @@ def int_or_str(value): return version +@lru_cache() def _get_package_version(name): module = sys.modules.get(name, None) version = None - for attr in VERSION_ATTRS: - try: - version = getattr(module, attr, None) - # Cast any version specified as a list into a tuple. - version = tuple(version) if isinstance(version, list) else version - if version not in NULL_VERSIONS: - return version - except Exception: - pass + + with warnings.catch_warnings(record=True): + for attr in VERSION_ATTRS: + try: + version = getattr(module, attr, None) + + # In certain cases like importlib_metadata.version, version is a callable + # function. + if callable(version): + continue + + # Cast any version specified as a list into a tuple. + version = tuple(version) if isinstance(version, list) else version + if version not in NULL_VERSIONS: + return version + except Exception: + pass # importlib was introduced into the standard library starting in Python3.8. if "importlib" in sys.modules and hasattr(sys.modules["importlib"], "metadata"): try: - version = sys.modules["importlib"].metadata.version(name) # pylint: disable=E1101 + # In Python3.10+ packages_distribution can be checked for as well + if hasattr(sys.modules["importlib"].metadata, "packages_distributions"): # pylint: disable=E1101 + distributions = sys.modules["importlib"].metadata.packages_distributions() # pylint: disable=E1101 + distribution_name = distributions.get(name, name) + distribution_name = distribution_name[0] if isinstance(distribution_name, list) else distribution_name + else: + distribution_name = name + + version = sys.modules["importlib"].metadata.version(distribution_name) # pylint: disable=E1101 if version not in NULL_VERSIONS: return version except Exception: @@ -95,4 +116,4 @@ def _get_package_version(name): if version not in NULL_VERSIONS: return version except Exception: - pass \ No newline at end of file + pass diff --git a/newrelic/common/signature.py b/newrelic/common/signature.py index 3149981962..68c9c6253a 100644 --- a/newrelic/common/signature.py +++ b/newrelic/common/signature.py @@ -12,20 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from newrelic.packages import six +from inspect import Signature -if six.PY3: - from inspect import Signature - def bind_args(func, args, kwargs): - """Bind arguments and apply defaults to missing arugments for a callable.""" - bound_args = Signature.from_callable(func).bind(*args, **kwargs) - bound_args.apply_defaults() - return bound_args.arguments - -else: - from inspect import getcallargs - - def bind_args(func, args, kwargs): - """Bind arguments and apply defaults to missing arugments for a callable.""" - return getcallargs(func, *args, **kwargs) +def bind_args(func, args, kwargs): + """Bind arguments and apply defaults to missing arguments for a callable.""" + bound_args = Signature.from_callable(func).bind(*args, **kwargs) + bound_args.apply_defaults() + return bound_args.arguments diff --git a/newrelic/common/stopwatch.py b/newrelic/common/stopwatch.py index 1f8de2f428..8038f3ec98 100644 --- a/newrelic/common/stopwatch.py +++ b/newrelic/common/stopwatch.py @@ -53,7 +53,7 @@ # wall clock time and duration based on a monotonic clock where # available. -class _Timer(object): +class _Timer(): def __init__(self): self._time_started = time.time() diff --git a/newrelic/common/streaming_utils.py b/newrelic/common/streaming_utils.py index ad1b371dc6..71863374c9 100644 --- a/newrelic/common/streaming_utils.py +++ b/newrelic/common/streaming_utils.py @@ -25,7 +25,7 @@ _logger = logging.getLogger(__name__) -class StreamBuffer(object): +class StreamBuffer(): def __init__(self, maxlen, batching=False): self._queue = collections.deque(maxlen=maxlen) self._notify = self.condition() @@ -80,7 +80,7 @@ def __iter__(self): return StreamBufferIterator(self) -class StreamBufferIterator(object): +class StreamBufferIterator(): MAX_BATCH_SIZE = 100 def __init__(self, stream_buffer): diff --git a/newrelic/common/system_info.py b/newrelic/common/system_info.py index 58a7118592..381b94a5e5 100644 --- a/newrelic/common/system_info.py +++ b/newrelic/common/system_info.py @@ -17,7 +17,6 @@ """ -import logging import multiprocessing import os import re @@ -25,36 +24,15 @@ import subprocess import sys import threading +from subprocess import check_output as _execute_program from newrelic.common.utilization import CommonUtilization -try: - from subprocess import check_output as _execute_program -except ImportError: - - def _execute_program(*popenargs, **kwargs): - # Replicates check_output() implementation from Python 2.7+. - # Should only be used for Python 2.6. - - if "stdout" in kwargs: - raise ValueError("stdout argument not allowed, it will be overridden.") - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) # nosec - output, unused_err = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - raise subprocess.CalledProcessError(retcode, cmd, output=output) - return output - - try: import resource except ImportError: pass -_logger = logging.getLogger(__name__) LOCALHOST_EQUIVALENTS = set( [ @@ -295,7 +273,7 @@ def _linux_physical_memory_used(filename=None): # data data + stack # dt dirty pages (unused in Linux 2.6) - filename = filename or "/proc/%d/statm" % os.getpid() + filename = filename or f"/proc/{os.getpid()}/statm" try: with open(filename, "r") as fp: @@ -351,7 +329,7 @@ def _resolve_hostname(use_dyno_names, dyno_shorten_prefixes): for prefix in dyno_shorten_prefixes: if prefix and dyno_name.startswith(prefix): - return "%s.*" % prefix + return f"{prefix}.*" return dyno_name diff --git a/newrelic/common/utilization.py b/newrelic/common/utilization.py index f205b4e132..06b045d3d7 100644 --- a/newrelic/common/utilization.py +++ b/newrelic/common/utilization.py @@ -16,15 +16,18 @@ import os import re import socket +import json import string -import threading +import newrelic.packages.urllib3 as urllib3 from newrelic.common.agent_http import InsecureHttpClient from newrelic.common.encoding_utils import json_decode from newrelic.core.internal_metrics import internal_count_metric + _logger = logging.getLogger(__name__) -VALID_CHARS_RE = re.compile(r'[0-9a-zA-Z_ ./-]') +VALID_CHARS_RE = re.compile(r"[0-9a-zA-Z_ ./-]") + class UtilizationHttpClient(InsecureHttpClient): SOCKET_TIMEOUT = 0.05 @@ -45,39 +48,36 @@ def send_request(self, *args, **kwargs): return super(UtilizationHttpClient, self).send_request(*args, **kwargs) -class CommonUtilization(object): - METADATA_HOST = '' - METADATA_PATH = '' +class CommonUtilization(): + METADATA_HOST = "" + METADATA_PATH = "" METADATA_QUERY = None HEADERS = None EXPECTED_KEYS = () - VENDOR_NAME = '' + VENDOR_NAME = "" FETCH_TIMEOUT = 0.4 CLIENT_CLS = UtilizationHttpClient @classmethod def record_error(cls, resource, data): # As per spec - internal_count_metric( - 'Supportability/utilization/%s/error' % cls.VENDOR_NAME, 1) - _logger.warning('Invalid %r data (%r): %r', - cls.VENDOR_NAME, resource, data) + internal_count_metric(f"Supportability/utilization/{cls.VENDOR_NAME}/error", 1) + _logger.warning("Invalid %r data (%r): %r", cls.VENDOR_NAME, resource, data) @classmethod def fetch(cls): try: - with cls.CLIENT_CLS(cls.METADATA_HOST, - timeout=cls.FETCH_TIMEOUT) as client: - resp = client.send_request(method='GET', - path=cls.METADATA_PATH, - params=cls.METADATA_QUERY, - headers=cls.HEADERS) + with cls.CLIENT_CLS(cls.METADATA_HOST, timeout=cls.FETCH_TIMEOUT) as client: + resp = client.send_request( + method="GET", path=cls.METADATA_PATH, params=cls.METADATA_QUERY, headers=cls.HEADERS + ) if not 200 <= resp[0] < 300: raise ValueError(resp[0]) return resp[1] except Exception as e: - _logger.debug('Unable to fetch %s data from %s%s: %r', - cls.VENDOR_NAME, cls.METADATA_HOST, cls.METADATA_PATH, e) + _logger.debug( + "Unable to fetch %s data from %s%s: %r", cls.VENDOR_NAME, cls.METADATA_HOST, cls.METADATA_PATH, e + ) return None @classmethod @@ -86,11 +86,9 @@ def get_values(cls, response): return try: - return json_decode(response.decode('utf-8')) + return json_decode(response.decode("utf-8")) except ValueError: - _logger.debug('Invalid %s data (%s%s): %r', - cls.VENDOR_NAME, cls.METADATA_HOST, - cls.METADATA_PATH, response) + _logger.debug("Invalid %s data (%s%s): %r", cls.VENDOR_NAME, cls.METADATA_HOST, cls.METADATA_PATH, response) @classmethod def valid_chars(cls, data): @@ -108,7 +106,7 @@ def valid_length(cls, data): if data is None: return False - b = data.encode('utf-8') + b = data.encode("utf-8") valid = len(b) <= 255 if valid: return True @@ -123,8 +121,7 @@ def normalize(cls, key, data): try: stripped = data.strip() - if (stripped and cls.valid_length(stripped) and - cls.valid_chars(stripped)): + if stripped and cls.valid_length(stripped) and cls.valid_chars(stripped): return stripped except: pass @@ -158,28 +155,27 @@ def detect(cls): class AWSUtilization(CommonUtilization): - EXPECTED_KEYS = ('availabilityZone', 'instanceId', 'instanceType') - METADATA_HOST = '169.254.169.254' - METADATA_PATH = '/latest/dynamic/instance-identity/document' - METADATA_TOKEN_PATH = '/latest/api/token' - HEADERS = {'X-aws-ec2-metadata-token-ttl-seconds': '21600'} - VENDOR_NAME = 'aws' + EXPECTED_KEYS = ("availabilityZone", "instanceId", "instanceType") + METADATA_HOST = "169.254.169.254" + METADATA_PATH = "/latest/dynamic/instance-identity/document" + METADATA_TOKEN_PATH = "/latest/api/token" + HEADERS = {"X-aws-ec2-metadata-token-ttl-seconds": "21600"} + VENDOR_NAME = "aws" @classmethod def fetchAuthToken(cls): try: - with cls.CLIENT_CLS(cls.METADATA_HOST, - timeout=cls.FETCH_TIMEOUT) as client: - resp = client.send_request(method='PUT', - path=cls.METADATA_TOKEN_PATH, - params=cls.METADATA_QUERY, - headers=cls.HEADERS) + with cls.CLIENT_CLS(cls.METADATA_HOST, timeout=cls.FETCH_TIMEOUT) as client: + resp = client.send_request( + method="PUT", path=cls.METADATA_TOKEN_PATH, params=cls.METADATA_QUERY, headers=cls.HEADERS + ) if not 200 <= resp[0] < 300: raise ValueError(resp[0]) return resp[1] except Exception as e: - _logger.debug('Unable to fetch %s data from %s%s: %r', - cls.VENDOR_NAME, cls.METADATA_HOST, cls.METADATA_PATH, e) + _logger.debug( + "Unable to fetch %s data from %s%s: %r", cls.VENDOR_NAME, cls.METADATA_HOST, cls.METADATA_PATH, e + ) return None @classmethod @@ -189,46 +185,45 @@ def fetch(cls): if authToken == None: return cls.HEADERS = {"X-aws-ec2-metadata-token": authToken} - with cls.CLIENT_CLS(cls.METADATA_HOST, - timeout=cls.FETCH_TIMEOUT) as client: - resp = client.send_request(method='GET', - path=cls.METADATA_PATH, - params=cls.METADATA_QUERY, - headers=cls.HEADERS) + with cls.CLIENT_CLS(cls.METADATA_HOST, timeout=cls.FETCH_TIMEOUT) as client: + resp = client.send_request( + method="GET", path=cls.METADATA_PATH, params=cls.METADATA_QUERY, headers=cls.HEADERS + ) if not 200 <= resp[0] < 300: raise ValueError(resp[0]) return resp[1] except Exception as e: - _logger.debug('Unable to fetch %s data from %s%s: %r', - cls.VENDOR_NAME, cls.METADATA_HOST, cls.METADATA_PATH, e) + _logger.debug( + "Unable to fetch %s data from %s%s: %r", cls.VENDOR_NAME, cls.METADATA_HOST, cls.METADATA_PATH, e + ) return None class AzureUtilization(CommonUtilization): - METADATA_HOST = '169.254.169.254' - METADATA_PATH = '/metadata/instance/compute' - METADATA_QUERY = {'api-version': '2017-03-01'} - EXPECTED_KEYS = ('location', 'name', 'vmId', 'vmSize') - HEADERS = {'Metadata': 'true'} - VENDOR_NAME = 'azure' + METADATA_HOST = "169.254.169.254" + METADATA_PATH = "/metadata/instance/compute" + METADATA_QUERY = {"api-version": "2017-03-01"} + EXPECTED_KEYS = ("location", "name", "vmId", "vmSize") + HEADERS = {"Metadata": "true"} + VENDOR_NAME = "azure" class GCPUtilization(CommonUtilization): - EXPECTED_KEYS = ('id', 'machineType', 'name', 'zone') - HEADERS = {'Metadata-Flavor': 'Google'} - METADATA_HOST = 'metadata.google.internal' - METADATA_PATH = '/computeMetadata/v1/instance/' - METADATA_QUERY = {'recursive': 'true'} - VENDOR_NAME = 'gcp' + EXPECTED_KEYS = ("id", "machineType", "name", "zone") + HEADERS = {"Metadata-Flavor": "Google"} + METADATA_HOST = "metadata.google.internal" + METADATA_PATH = "/computeMetadata/v1/instance/" + METADATA_QUERY = {"recursive": "true"} + VENDOR_NAME = "gcp" @classmethod def normalize(cls, key, data): if data is None: return - if key in ('machineType', 'zone'): - formatted = data.strip().split('/')[-1] - elif key == 'id': + if key in ("machineType", "zone"): + formatted = data.strip().split("/")[-1] + elif key == "id": formatted = str(data) else: formatted = data @@ -237,14 +232,14 @@ def normalize(cls, key, data): class PCFUtilization(CommonUtilization): - EXPECTED_KEYS = ('cf_instance_guid', 'cf_instance_ip', 'memory_limit') - VENDOR_NAME = 'pcf' + EXPECTED_KEYS = ("cf_instance_guid", "cf_instance_ip", "memory_limit") + VENDOR_NAME = "pcf" @staticmethod def fetch(): - cf_instance_guid = os.environ.get('CF_INSTANCE_GUID') - cf_instance_ip = os.environ.get('CF_INSTANCE_IP') - memory_limit = os.environ.get('MEMORY_LIMIT') + cf_instance_guid = os.environ.get("CF_INSTANCE_GUID") + cf_instance_ip = os.environ.get("CF_INSTANCE_IP") + memory_limit = os.environ.get("MEMORY_LIMIT") pcf_vars = (cf_instance_guid, cf_instance_ip, memory_limit) if all(pcf_vars): return pcf_vars @@ -256,30 +251,51 @@ def get_values(cls, response): values = {} for k, v in zip(cls.EXPECTED_KEYS, response): - if hasattr(v, 'decode'): - v = v.decode('utf-8') + if hasattr(v, "decode"): + v = v.decode("utf-8") values[k] = v return values class DockerUtilization(CommonUtilization): - VENDOR_NAME = 'docker' - EXPECTED_KEYS = ('id',) - METADATA_FILE = '/proc/self/cgroup' - DOCKER_RE = re.compile(r'([0-9a-f]{64,})') + VENDOR_NAME = "docker" + EXPECTED_KEYS = ("id",) + + METADATA_FILE_CGROUPS_V1 = "/proc/self/cgroup" + METADATA_RE_CGROUPS_V1 = re.compile(r"[0-9a-f]{64,}") + + METADATA_FILE_CGROUPS_V2 = "/proc/self/mountinfo" + METADATA_RE_CGROUPS_V2 = re.compile(r"^.*/docker/containers/([0-9a-f]{64,})/.*$") @classmethod def fetch(cls): + # Try to read from cgroups try: - with open(cls.METADATA_FILE, 'rb') as f: + with open(cls.METADATA_FILE_CGROUPS_V1, "rb") as f: for line in f: - stripped = line.decode('utf-8').strip() - cgroup = stripped.split(':') + stripped = line.decode("utf-8").strip() + cgroup = stripped.split(":") if len(cgroup) != 3: continue - subsystems = cgroup[1].split(',') - if 'cpu' in subsystems: - return cgroup[2] + subsystems = cgroup[1].split(",") + if "cpu" in subsystems: + contents = cgroup[2].split("/")[-1] + match = cls.METADATA_RE_CGROUPS_V1.search(contents) + if match: + return match.group(0) + except: + # There are all sorts of exceptions that can occur here + # (i.e. permissions, non-existent file, etc) + pass + + # Fallback to reading from mountinfo + try: + with open(cls.METADATA_FILE_CGROUPS_V2, "rb") as f: + for line in f: + stripped = line.decode("utf-8").strip() + match = cls.METADATA_RE_CGROUPS_V2.match(stripped) + if match: + return match.group(1) except: # There are all sorts of exceptions that can occur here # (i.e. permissions, non-existent file, etc) @@ -290,17 +306,12 @@ def get_values(cls, contents): if contents is None: return - value = contents.split('/')[-1] - match = cls.DOCKER_RE.search(value) - if match: - value = match.group(0) - return {'id': value} + return {"id": contents} @classmethod def valid_chars(cls, data): if data is None: return False - hex_digits = set(string.hexdigits) valid = all((c in hex_digits for c in data)) @@ -315,20 +326,67 @@ def valid_length(cls, data): return False # Must be exactly 64 characters - valid = len(data) == 64 - if valid: - return True + return bool(len(data) == 64) - return False + +class ECSUtilization(CommonUtilization): + VENDOR_NAME = "ecs" + EXPECTED_KEYS = ("ecsDockerId",) + + @classmethod + def fetch(cls): + # First, try checking ECS V4 metadata env var + try: + uri = os.environ.get("ECS_CONTAINER_METADATA_URI_V4") + if uri: + ecs_id = cls.get_ecs_container_id(uri) + if ecs_id: + return ecs_id + except: + # There are all sorts of exceptions that can occur here + # (i.e. permissions, non-existent file, etc) + pass + + # If V4 didn't work, try the older version + try: + uri = os.environ.get("ECS_CONTAINER_METADATA_URI") + if uri: + ecs_id = cls.get_ecs_container_id(uri) + if ecs_id: + return ecs_id + except: + # There are all sorts of exceptions that can occur here + # (i.e. permissions, non-existent file, etc) + pass + + @classmethod + def get_ecs_container_id(cls, metadata_uri): + try: + http = urllib3.PoolManager() + resp = http.request("GET", metadata_uri) + resp_dict = json.loads(resp.data) + docker_id = resp_dict.get("DockerId") + resp.release_conn() + return docker_id + except: + _logger.debug("Unable to fetch Docker container ID data from ECS endpoint: %s", metadata_uri) + return None + + @classmethod + def get_values(cls, contents): + if contents is None: + return + + return {"ecsDockerId": contents} class KubernetesUtilization(CommonUtilization): - EXPECTED_KEYS = ('kubernetes_service_host', ) - VENDOR_NAME = 'kubernetes' + EXPECTED_KEYS = ("kubernetes_service_host",) + VENDOR_NAME = "kubernetes" @staticmethod def fetch(): - kubernetes_service_host = os.environ.get('KUBERNETES_SERVICE_HOST') + kubernetes_service_host = os.environ.get("KUBERNETES_SERVICE_HOST") if kubernetes_service_host: return kubernetes_service_host @@ -337,7 +395,7 @@ def get_values(cls, v): if v is None: return - if hasattr(v, 'decode'): - v = v.decode('utf-8') + if hasattr(v, "decode"): + v = v.decode("utf-8") - return {'kubernetes_service_host': v} + return {"kubernetes_service_host": v} diff --git a/newrelic/config.py b/newrelic/config.py index df95db0290..37e190dc30 100644 --- a/newrelic/config.py +++ b/newrelic/config.py @@ -12,17 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import configparser import fnmatch import logging import os import sys import traceback -try: - import ConfigParser -except ImportError: - import configparser as ConfigParser - import newrelic.api.application import newrelic.api.background_task import newrelic.api.database_trace @@ -34,7 +30,6 @@ import newrelic.api.generator_trace import newrelic.api.import_hook import newrelic.api.memcache_trace -import newrelic.api.object_wrapper import newrelic.api.profile_trace import newrelic.api.settings import newrelic.api.transaction_name @@ -43,7 +38,7 @@ import newrelic.core.agent import newrelic.core.config from newrelic.common.log_file import initialize_logging -from newrelic.common.object_names import expand_builtin_exception_name +from newrelic.common.object_names import callable_name, expand_builtin_exception_name from newrelic.core import trace_cache from newrelic.core.config import ( Settings, @@ -51,7 +46,6 @@ default_host, fetch_config_setting, ) -from newrelic.packages import six __all__ = ["initialize", "filter_app_factory"] @@ -94,7 +88,7 @@ # modules to look up customised settings defined in the loaded # configuration file. -_config_object = ConfigParser.RawConfigParser() +_config_object = configparser.RawConfigParser() # Cache of the parsed global settings found in the configuration # file. We cache these so can dump them out to the log file once @@ -102,6 +96,14 @@ _cache_object = [] + +def _reset_config_parser(): + global _config_object + global _cache_object + _config_object = configparser.RawConfigParser() + _cache_object = [] + + # Mechanism for extracting settings from the configuration for use in # instrumentation modules and extensions. @@ -234,9 +236,7 @@ def _raise_configuration_error(section, option=None): if not _ignore_errors: if section: raise newrelic.api.exceptions.ConfigurationError( - 'Invalid configuration for section "%s". ' - "Check New Relic agent log file for further " - "details." % section + f'Invalid configuration for section "{section}". Check New Relic agent log file for further details.' ) raise newrelic.api.exceptions.ConfigurationError( "Invalid configuration. Check New Relic agent log file for further details." @@ -249,14 +249,10 @@ def _raise_configuration_error(section, option=None): if not _ignore_errors: if section: raise newrelic.api.exceptions.ConfigurationError( - 'Invalid configuration for option "%s" in ' - 'section "%s". Check New Relic agent log ' - "file for further details." % (option, section) + f'Invalid configuration for option "{option}" in section "{section}". Check New Relic agent log file for further details.' ) raise newrelic.api.exceptions.ConfigurationError( - 'Invalid configuration for option "%s". ' - "Check New Relic agent log file for further " - "details." % option + f'Invalid configuration for option "{option}". Check New Relic agent log file for further details.' ) @@ -297,10 +293,10 @@ def _process_setting(section, option, getter, mapper): _cache_object.append((option, value)) - except ConfigParser.NoSectionError: + except configparser.NoSectionError: pass - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: @@ -320,6 +316,8 @@ def _process_configuration(section): _process_setting(section, "api_key", "get", None) _process_setting(section, "host", "get", None) _process_setting(section, "port", "getint", None) + _process_setting(section, "otlp_host", "get", None) + _process_setting(section, "otlp_port", "getint", None) _process_setting(section, "ssl", "getboolean", None) _process_setting(section, "proxy_scheme", "get", None) _process_setting(section, "proxy_host", "get", None) @@ -329,6 +327,13 @@ def _process_configuration(section): _process_setting(section, "ca_bundle_path", "get", None) _process_setting(section, "audit_log_file", "get", None) _process_setting(section, "monitor_mode", "getboolean", None) + _process_setting(section, "security.agent.enabled", "getboolean", None) + _process_setting(section, "security.enabled", "getboolean", None) + _process_setting(section, "security.mode", "get", None) + _process_setting(section, "security.validator_service_url", "get", None) + _process_setting(section, "security.detection.rci.enabled", "getboolean", None) + _process_setting(section, "security.detection.rxss.enabled", "getboolean", None) + _process_setting(section, "security.detection.deserialization.enabled", "getboolean", None) _process_setting(section, "developer_mode", "getboolean", None) _process_setting(section, "high_security", "getboolean", None) _process_setting(section, "capture_params", "getboolean", None) @@ -345,6 +350,7 @@ def _process_configuration(section): _process_setting(section, "transaction_name.naming_scheme", "get", None) _process_setting(section, "gc_runtime_metrics.enabled", "getboolean", None) _process_setting(section, "gc_runtime_metrics.top_object_count_limit", "getint", None) + _process_setting(section, "memory_runtime_pid_metrics.enabled", "getboolean", None) _process_setting(section, "thread_profiler.enabled", "getboolean", None) _process_setting(section, "transaction_tracer.enabled", "getboolean", None) _process_setting( @@ -433,6 +439,8 @@ def _process_configuration(section): ) _process_setting(section, "custom_insights_events.enabled", "getboolean", None) _process_setting(section, "custom_insights_events.max_samples_stored", "getint", None) + _process_setting(section, "custom_insights_events.max_attribute_value", "getint", None) + _process_setting(section, "ml_insights_events.enabled", "getboolean", None) _process_setting(section, "distributed_tracing.enabled", "getboolean", None) _process_setting(section, "distributed_tracing.exclude_newrelic_header", "getboolean", None) _process_setting(section, "span_events.enabled", "getboolean", None) @@ -492,6 +500,7 @@ def _process_configuration(section): _process_setting(section, "debug.disable_certificate_validation", "getboolean", None) _process_setting(section, "debug.disable_harvest_until_shutdown", "getboolean", None) _process_setting(section, "debug.connect_span_stream_in_developer_mode", "getboolean", None) + _process_setting(section, "debug.otlp_content_encoding", "get", None) _process_setting(section, "cross_application_tracer.enabled", "getboolean", None) _process_setting(section, "message_tracer.segment_parameters_enabled", "getboolean", None) _process_setting(section, "process_host.display_name", "get", None) @@ -526,6 +535,7 @@ def _process_configuration(section): None, ) _process_setting(section, "event_harvest_config.harvest_limits.custom_event_data", "getint", None) + _process_setting(section, "event_harvest_config.harvest_limits.ml_event_data", "getint", None) _process_setting(section, "event_harvest_config.harvest_limits.span_event_data", "getint", None) _process_setting(section, "event_harvest_config.harvest_limits.error_event_data", "getint", None) _process_setting(section, "event_harvest_config.harvest_limits.log_event_data", "getint", None) @@ -539,9 +549,21 @@ def _process_configuration(section): _process_setting(section, "application_logging.enabled", "getboolean", None) _process_setting(section, "application_logging.forwarding.max_samples_stored", "getint", None) _process_setting(section, "application_logging.forwarding.enabled", "getboolean", None) + _process_setting(section, "application_logging.forwarding.context_data.enabled", "getboolean", None) + _process_setting(section, "application_logging.forwarding.context_data.include", "get", _map_inc_excl_attributes) + _process_setting(section, "application_logging.forwarding.context_data.exclude", "get", _map_inc_excl_attributes) _process_setting(section, "application_logging.metrics.enabled", "getboolean", None) _process_setting(section, "application_logging.local_decorating.enabled", "getboolean", None) + _process_setting(section, "machine_learning.enabled", "getboolean", None) + _process_setting(section, "machine_learning.inference_events_value.enabled", "getboolean", None) + _process_setting(section, "ai_monitoring.enabled", "getboolean", None) + _process_setting(section, "ai_monitoring.record_content.enabled", "getboolean", None) + _process_setting(section, "ai_monitoring.streaming.enabled", "getboolean", None) + _process_setting(section, "k8s_operator.enabled", "getboolean", None) + _process_setting(section, "azure_operator.enabled", "getboolean", None) + _process_setting(section, "package_reporting.enabled", "getboolean", None) + # Loading of configuration from specified file and for specified # deployment environment. Can also indicate whether configuration @@ -550,6 +572,11 @@ def _process_configuration(section): _configuration_done = False +def _reset_configuration_done(): + global _configuration_done + _configuration_done = False + + def _process_app_name_setting(): # Do special processing to handle the case where the application # name was actually a semicolon separated list of names. In this @@ -775,7 +802,7 @@ def translate_deprecated_settings(settings, cached_settings): ignored_params = fetch_config_setting(settings, "ignored_params") for p in ignored_params: - attr_value = "request.parameters." + p + attr_value = f"request.parameters.{p}" excluded_attrs = fetch_config_setting(settings, "attributes.exclude") if attr_value not in excluded_attrs: @@ -868,6 +895,10 @@ def apply_local_high_security_mode_setting(settings): settings.custom_insights_events.enabled = False _logger.info(log_template, "custom_insights_events.enabled", True, False) + if settings.ml_insights_events.enabled: + settings.ml_insights_events.enabled = False + _logger.info(log_template, "ml_insights_events.enabled", True, False) + if settings.message_tracer.segment_parameters_enabled: settings.message_tracer.segment_parameters_enabled = False _logger.info(log_template, "message_tracer.segment_parameters_enabled", True, False) @@ -876,6 +907,14 @@ def apply_local_high_security_mode_setting(settings): settings.application_logging.forwarding.enabled = False _logger.info(log_template, "application_logging.forwarding.enabled", True, False) + if settings.machine_learning.inference_events_value.enabled: + settings.machine_learning.inference_events_value.enabled = False + _logger.info(log_template, "machine_learning.inference_events_value.enabled", True, False) + + if settings.ai_monitoring.enabled: + settings.ai_monitoring.enabled = False + _logger.info(log_template, "ai_monitoring.enabled", True, False) + return settings @@ -903,10 +942,7 @@ def _load_configuration( if _configuration_done: if _config_file != config_file or _environment != environment: raise newrelic.api.exceptions.ConfigurationError( - "Configuration has already been done against " - "differing configuration file or environment. " - 'Prior configuration file used was "%s" and ' - 'environment "%s".' % (_config_file, _environment) + f'Configuration has already been done against differing configuration file or environment. Prior configuration file used was "{_config_file}" and environment "{_environment}".' ) return @@ -967,7 +1003,7 @@ def _load_configuration( # name in internal settings object as indication of succeeding. if not _config_object.read([config_file]): - raise newrelic.api.exceptions.ConfigurationError("Unable to open configuration file %s." % config_file) + raise newrelic.api.exceptions.ConfigurationError(f"Unable to open configuration file {config_file}.") _settings.config_file = config_file @@ -977,7 +1013,7 @@ def _load_configuration( _process_setting("newrelic", "log_file", "get", None) if environment: - _process_setting("newrelic:%s" % environment, "log_file", "get", None) + _process_setting(f"newrelic:{environment}", "log_file", "get", None) if log_file is None: log_file = _settings.log_file @@ -985,7 +1021,7 @@ def _load_configuration( _process_setting("newrelic", "log_level", "get", _map_log_level) if environment: - _process_setting("newrelic:%s" % environment, "log_level", "get", _map_log_level) + _process_setting(f"newrelic:{environment}", "log_level", "get", _map_log_level) if log_level is None: log_level = _settings.log_level @@ -1005,7 +1041,7 @@ def _load_configuration( if environment: _settings.environment = environment - _process_configuration("newrelic:%s" % environment) + _process_configuration(f"newrelic:{environment}") # Log details of the configuration options which were # read and the values they have as would be applied @@ -1150,7 +1186,7 @@ def _process_module_configuration(): try: enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -1202,7 +1238,7 @@ def _module_function_glob(module, object_path): # Skip adding individual class's methods on failure available_functions.update( { - "%s.%s" % (cls, k): v + f"{cls}.{k}": v for k, v in available_classes.get(cls).__dict__.items() if callable(v) and not isinstance(v, type) } @@ -1238,12 +1274,11 @@ def _process_wsgi_application_configuration(): for section in _config_object.sections(): if not section.startswith("wsgi-application:"): continue - enabled = False try: enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -1292,7 +1327,7 @@ def _process_background_task_configuration(): try: enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -1316,7 +1351,7 @@ def _process_background_task_configuration(): group = _config_object.get(section, "group") if name and name.startswith("lambda "): - callable_vars = {"callable_name": newrelic.api.object_wrapper.callable_name} + callable_vars = {"callable_name": callable_name} name = eval(name, callable_vars) # nosec, pylint: disable=W0123 _logger.debug("register background-task %s", ((module, object_path, application, name, group),)) @@ -1351,7 +1386,7 @@ def _process_database_trace_configuration(): try: enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -1366,7 +1401,7 @@ def _process_database_trace_configuration(): sql = _config_object.get(section, "sql") if sql.startswith("lambda "): - callable_vars = {"callable_name": newrelic.api.object_wrapper.callable_name} + callable_vars = {"callable_name": callable_name} sql = eval(sql, callable_vars) # nosec, pylint: disable=W0123 _logger.debug("register database-trace %s", ((module, object_path, sql),)) @@ -1401,7 +1436,7 @@ def _process_external_trace_configuration(): try: enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -1421,11 +1456,11 @@ def _process_external_trace_configuration(): method = _config_object.get(section, "method") if url.startswith("lambda "): - callable_vars = {"callable_name": newrelic.api.object_wrapper.callable_name} + callable_vars = {"callable_name": callable_name} url = eval(url, callable_vars) # nosec, pylint: disable=W0123 if method and method.startswith("lambda "): - callable_vars = {"callable_name": newrelic.api.object_wrapper.callable_name} + callable_vars = {"callable_name": callable_name} method = eval(method, callable_vars) # nosec, pylint: disable=W0123 _logger.debug("register external-trace %s", ((module, object_path, library, url, method),)) @@ -1462,7 +1497,7 @@ def _process_function_trace_configuration(): try: enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -1493,7 +1528,7 @@ def _process_function_trace_configuration(): rollup = _config_object.get(section, "rollup") if name and name.startswith("lambda "): - callable_vars = {"callable_name": newrelic.api.object_wrapper.callable_name} + callable_vars = {"callable_name": callable_name} name = eval(name, callable_vars) # nosec, pylint: disable=W0123 _logger.debug( @@ -1530,7 +1565,7 @@ def _process_generator_trace_configuration(): try: enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -1551,7 +1586,7 @@ def _process_generator_trace_configuration(): group = _config_object.get(section, "group") if name and name.startswith("lambda "): - callable_vars = {"callable_name": newrelic.api.object_wrapper.callable_name} + callable_vars = {"callable_name": callable_name} name = eval(name, callable_vars) # nosec, pylint: disable=W0123 _logger.debug("register generator-trace %s", ((module, object_path, name, group),)) @@ -1586,7 +1621,7 @@ def _process_profile_trace_configuration(): try: enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -1610,7 +1645,7 @@ def _process_profile_trace_configuration(): depth = _config_object.get(section, "depth") if name and name.startswith("lambda "): - callable_vars = {"callable_name": newrelic.api.object_wrapper.callable_name} + callable_vars = {"callable_name": callable_name} name = eval(name, callable_vars) # nosec, pylint: disable=W0123 _logger.debug("register profile-trace %s", ((module, object_path, name, group, depth),)) @@ -1645,7 +1680,7 @@ def _process_memcache_trace_configuration(): try: enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -1660,7 +1695,7 @@ def _process_memcache_trace_configuration(): command = _config_object.get(section, "command") if command.startswith("lambda "): - callable_vars = {"callable_name": newrelic.api.object_wrapper.callable_name} + callable_vars = {"callable_name": callable_name} command = eval(command, callable_vars) # nosec, pylint: disable=W0123 _logger.debug("register memcache-trace %s", (module, object_path, command)) @@ -1696,7 +1731,7 @@ def _process_transaction_name_configuration(): try: enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -1720,7 +1755,7 @@ def _process_transaction_name_configuration(): priority = _config_object.getint(section, "priority") if name and name.startswith("lambda "): - callable_vars = {"callable_name": newrelic.api.object_wrapper.callable_name} + callable_vars = {"callable_name": callable_name} name = eval(name, callable_vars) # nosec, pylint: disable=W0123 _logger.debug("register transaction-name %s", ((module, object_path, name, group, priority),)) @@ -1755,7 +1790,7 @@ def _process_error_trace_configuration(): try: enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -1805,7 +1840,7 @@ def _process_data_source_configuration(): try: enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -1916,7 +1951,7 @@ def _process_function_profile_configuration(): try: enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -1959,10 +1994,10 @@ def _process_module_definition(target, module, function="instrument"): return try: - section = "import-hook:%s" % target + section = f"import-hook:{target}" if _config_object.has_section(section): enabled = _config_object.getboolean(section, "enabled") - except ConfigParser.NoOptionError: + except configparser.NoOptionError: pass except Exception: _raise_configuration_error(section) @@ -2010,362 +2045,1067 @@ def _process_trace_cache_import_hooks(): def _process_module_builtin_defaults(): _process_module_definition( - "asyncio.base_events", - "newrelic.hooks.coroutines_asyncio", - "instrument_asyncio_base_events", + "openai.api_resources.embedding", + "newrelic.hooks.mlmodel_openai", + "instrument_openai_api_resources_embedding", ) _process_module_definition( - "asyncio.events", - "newrelic.hooks.coroutines_asyncio", - "instrument_asyncio_events", + "openai.api_resources.chat_completion", + "newrelic.hooks.mlmodel_openai", + "instrument_openai_api_resources_chat_completion", + ) + _process_module_definition( + "openai.resources.embeddings", + "newrelic.hooks.mlmodel_openai", + "instrument_openai_resources_embeddings", + ) + _process_module_definition( + "openai.util", + "newrelic.hooks.mlmodel_openai", + "instrument_openai_util", + ) + _process_module_definition( + "openai.api_resources.abstract.engine_api_resource", + "newrelic.hooks.mlmodel_openai", + "instrument_openai_api_resources_abstract_engine_api_resource", + ) + _process_module_definition( + "openai._streaming", + "newrelic.hooks.mlmodel_openai", + "instrument_openai__streaming", ) - _process_module_definition("asgiref.sync", "newrelic.hooks.adapter_asgiref", "instrument_asgiref_sync") + _process_module_definition( + "openai.resources.chat.completions", + "newrelic.hooks.mlmodel_openai", + "instrument_openai_resources_chat_completions", + ) _process_module_definition( - "django.core.handlers.base", - "newrelic.hooks.framework_django", - "instrument_django_core_handlers_base", + "openai.resources.completions", + "newrelic.hooks.mlmodel_openai", + "instrument_openai_resources_chat_completions", ) _process_module_definition( - "django.core.handlers.asgi", - "newrelic.hooks.framework_django", - "instrument_django_core_handlers_asgi", + "openai._base_client", + "newrelic.hooks.mlmodel_openai", + "instrument_openai_base_client", ) + _process_module_definition( - "django.core.handlers.wsgi", - "newrelic.hooks.framework_django", - "instrument_django_core_handlers_wsgi", + "asyncio.base_events", + "newrelic.hooks.coroutines_asyncio", + "instrument_asyncio_base_events", ) + _process_module_definition( - "django.core.urlresolvers", - "newrelic.hooks.framework_django", - "instrument_django_core_urlresolvers", + "langchain_core.runnables.base", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_runnables_chains_base", ) _process_module_definition( - "django.template", - "newrelic.hooks.framework_django", - "instrument_django_template", + "langchain_core.runnables.config", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_core_runnables_config", ) _process_module_definition( - "django.template.loader_tags", - "newrelic.hooks.framework_django", - "instrument_django_template_loader_tags", + "langchain.chains.base", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_chains_base", ) _process_module_definition( - "django.core.servers.basehttp", - "newrelic.hooks.framework_django", - "instrument_django_core_servers_basehttp", + "langchain_core.callbacks.manager", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_callbacks_manager", ) _process_module_definition( - "django.contrib.staticfiles.views", - "newrelic.hooks.framework_django", - "instrument_django_contrib_staticfiles_views", + "langchain_community.vectorstores.docarray.hnsw", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "django.contrib.staticfiles.handlers", - "newrelic.hooks.framework_django", - "instrument_django_contrib_staticfiles_handlers", + "langchain_community.vectorstores.docarray.in_memory", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "django.views.debug", - "newrelic.hooks.framework_django", - "instrument_django_views_debug", + "langchain_community.vectorstores.alibabacloud_opensearch", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "django.http.multipartparser", - "newrelic.hooks.framework_django", - "instrument_django_http_multipartparser", + "langchain_community.vectorstores.redis.base", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "django.core.mail", - "newrelic.hooks.framework_django", - "instrument_django_core_mail", + "langchain_community.vectorstores.aerospike", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "django.core.mail.message", - "newrelic.hooks.framework_django", - "instrument_django_core_mail_message", + "langchain_community.vectorstores.analyticdb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "django.views.generic.base", - "newrelic.hooks.framework_django", - "instrument_django_views_generic_base", + "langchain_community.vectorstores.annoy", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "django.core.management.base", - "newrelic.hooks.framework_django", - "instrument_django_core_management_base", + "langchain_community.vectorstores.apache_doris", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "django.template.base", - "newrelic.hooks.framework_django", - "instrument_django_template_base", + "langchain_community.vectorstores.aperturedb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "django.middleware.gzip", - "newrelic.hooks.framework_django", - "instrument_django_gzip_middleware", + "langchain_community.vectorstores.astradb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) - - # New modules in Django 1.10 _process_module_definition( - "django.urls.resolvers", - "newrelic.hooks.framework_django", - "instrument_django_core_urlresolvers", + "langchain_community.vectorstores.atlas", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "django.urls.base", - "newrelic.hooks.framework_django", - "instrument_django_urls_base", + "langchain_community.vectorstores.awadb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "django.core.handlers.exception", - "newrelic.hooks.framework_django", - "instrument_django_core_handlers_exception", + "langchain_community.vectorstores.azure_cosmos_db_no_sql", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) - - _process_module_definition("falcon.api", "newrelic.hooks.framework_falcon", "instrument_falcon_api") - _process_module_definition("falcon.app", "newrelic.hooks.framework_falcon", "instrument_falcon_app") _process_module_definition( - "falcon.routing.util", - "newrelic.hooks.framework_falcon", - "instrument_falcon_routing_util", + "langchain_community.vectorstores.azure_cosmos_db", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) - _process_module_definition( - "fastapi.routing", - "newrelic.hooks.framework_fastapi", - "instrument_fastapi_routing", + "langchain_community.vectorstores.azuresearch", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) - - _process_module_definition("flask.app", "newrelic.hooks.framework_flask", "instrument_flask_app") _process_module_definition( - "flask.templating", - "newrelic.hooks.framework_flask", - "instrument_flask_templating", + "langchain_community.vectorstores.bageldb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "flask.blueprints", - "newrelic.hooks.framework_flask", - "instrument_flask_blueprints", + "langchain_community.vectorstores.baiduvectordb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) - _process_module_definition("flask.views", "newrelic.hooks.framework_flask", "instrument_flask_views") - _process_module_definition( - "flask_compress", - "newrelic.hooks.middleware_flask_compress", - "instrument_flask_compress", + "langchain_community.vectorstores.baiducloud_vector_search", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) - - _process_module_definition("flask_restful", "newrelic.hooks.component_flask_rest", "instrument_flask_rest") _process_module_definition( - "flask_restplus.api", - "newrelic.hooks.component_flask_rest", - "instrument_flask_rest", + "langchain_community.vectorstores.bigquery_vector_search", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "flask_restx.api", - "newrelic.hooks.component_flask_rest", - "instrument_flask_rest", + "langchain_community.vectorstores.cassandra", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "graphql_server", - "newrelic.hooks.component_graphqlserver", - "instrument_graphqlserver", + "langchain_community.vectorstores.chroma", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "sentry_sdk.integrations.asgi", "newrelic.hooks.component_sentry", "instrument_sentry_sdk_integrations_asgi" + "langchain_community.vectorstores.clarifai", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) - # _process_module_definition('web.application', - # 'newrelic.hooks.framework_webpy') - # _process_module_definition('web.template', - # 'newrelic.hooks.framework_webpy') - _process_module_definition( - "gluon.compileapp", - "newrelic.hooks.framework_web2py", - "instrument_gluon_compileapp", + "langchain_community.vectorstores.clickhouse", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "gluon.restricted", - "newrelic.hooks.framework_web2py", - "instrument_gluon_restricted", + "langchain_community.vectorstores.couchbase", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) - _process_module_definition("gluon.main", "newrelic.hooks.framework_web2py", "instrument_gluon_main") - _process_module_definition("gluon.template", "newrelic.hooks.framework_web2py", "instrument_gluon_template") - _process_module_definition("gluon.tools", "newrelic.hooks.framework_web2py", "instrument_gluon_tools") - _process_module_definition("gluon.http", "newrelic.hooks.framework_web2py", "instrument_gluon_http") - _process_module_definition("httpx._client", "newrelic.hooks.external_httpx", "instrument_httpx_client") + _process_module_definition( + "langchain_community.vectorstores.dashvector", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) - _process_module_definition("gluon.contrib.feedparser", "newrelic.hooks.external_feedparser") - _process_module_definition("gluon.contrib.memcache.memcache", "newrelic.hooks.memcache_memcache") + _process_module_definition( + "langchain_community.vectorstores.databricks_vector_search", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) _process_module_definition( - "graphene.types.schema", - "newrelic.hooks.framework_graphene", - "instrument_graphene_types_schema", + "langchain_community.vectorstores.deeplake", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "graphql.graphql", - "newrelic.hooks.framework_graphql", - "instrument_graphql", + "langchain_community.vectorstores.dingo", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "graphql.execution.execute", - "newrelic.hooks.framework_graphql", - "instrument_graphql_execute", + "langchain_community.vectorstores.documentdb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "graphql.execution.executor", - "newrelic.hooks.framework_graphql", - "instrument_graphql_execute", + "langchain_community.vectorstores.duckdb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "graphql.execution.middleware", - "newrelic.hooks.framework_graphql", - "instrument_graphql_execution_middleware", + "langchain_community.vectorstores.ecloud_vector_search", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "graphql.execution.utils", - "newrelic.hooks.framework_graphql", - "instrument_graphql_execution_utils", + "langchain_community.vectorstores.elastic_vector_search", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "graphql.error.located_error", - "newrelic.hooks.framework_graphql", - "instrument_graphql_error_located_error", + "langchain_community.vectorstores.elasticsearch", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "graphql.language.parser", - "newrelic.hooks.framework_graphql", - "instrument_graphql_parser", + "langchain_community.vectorstores.epsilla", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "graphql.validation.validate", - "newrelic.hooks.framework_graphql", - "instrument_graphql_validate", + "langchain_community.vectorstores.faiss", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "graphql.validation.validation", - "newrelic.hooks.framework_graphql", - "instrument_graphql_validate", + "langchain_community.vectorstores.hanavector", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "ariadne.asgi", - "newrelic.hooks.framework_ariadne", - "instrument_ariadne_asgi", + "langchain_community.vectorstores.hippo", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "ariadne.graphql", - "newrelic.hooks.framework_ariadne", - "instrument_ariadne_execute", + "langchain_community.vectorstores.hologres", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "ariadne.wsgi", - "newrelic.hooks.framework_ariadne", - "instrument_ariadne_wsgi", + "langchain_community.vectorstores.infinispanvs", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) - _process_module_definition("grpc._channel", "newrelic.hooks.framework_grpc", "instrument_grpc__channel") - _process_module_definition("grpc._server", "newrelic.hooks.framework_grpc", "instrument_grpc_server") + _process_module_definition( + "langchain_community.vectorstores.inmemory", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) - _process_module_definition("pylons.wsgiapp", "newrelic.hooks.framework_pylons") - _process_module_definition("pylons.controllers.core", "newrelic.hooks.framework_pylons") - _process_module_definition("pylons.templating", "newrelic.hooks.framework_pylons") + _process_module_definition( + "langchain_community.vectorstores.kdbai", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) - _process_module_definition("bottle", "newrelic.hooks.framework_bottle", "instrument_bottle") + _process_module_definition( + "langchain_community.vectorstores.kinetica", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) _process_module_definition( - "cherrypy._cpreqbody", - "newrelic.hooks.framework_cherrypy", - "instrument_cherrypy__cpreqbody", + "langchain_community.vectorstores.lancedb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "cherrypy._cprequest", - "newrelic.hooks.framework_cherrypy", - "instrument_cherrypy__cprequest", + "langchain_community.vectorstores.lantern", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "cherrypy._cpdispatch", - "newrelic.hooks.framework_cherrypy", - "instrument_cherrypy__cpdispatch", + "langchain_community.vectorstores.llm_rails", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "cherrypy._cpwsgi", - "newrelic.hooks.framework_cherrypy", - "instrument_cherrypy__cpwsgi", + "langchain_community.vectorstores.manticore_search", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "cherrypy._cptree", - "newrelic.hooks.framework_cherrypy", - "instrument_cherrypy__cptree", + "langchain_community.vectorstores.marqo", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "confluent_kafka.cimpl", - "newrelic.hooks.messagebroker_confluentkafka", - "instrument_confluentkafka_cimpl", + "langchain_community.vectorstores.matching_engine", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "confluent_kafka.serializing_producer", - "newrelic.hooks.messagebroker_confluentkafka", - "instrument_confluentkafka_serializing_producer", + "langchain_community.vectorstores.meilisearch", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "confluent_kafka.deserializing_consumer", - "newrelic.hooks.messagebroker_confluentkafka", - "instrument_confluentkafka_deserializing_consumer", + "langchain_community.vectorstores.milvus", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "kafka.consumer.group", - "newrelic.hooks.messagebroker_kafkapython", - "instrument_kafka_consumer_group", + "langchain_community.vectorstores.momento_vector_index", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "kafka.producer.kafka", - "newrelic.hooks.messagebroker_kafkapython", - "instrument_kafka_producer", + "langchain_community.vectorstores.mongodb_atlas", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "kafka.coordinator.heartbeat", - "newrelic.hooks.messagebroker_kafkapython", - "instrument_kafka_heartbeat", + "langchain_community.vectorstores.myscale", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "kafka.consumer.group", - "newrelic.hooks.messagebroker_kafkapython", - "instrument_kafka_consumer_group", + "langchain_community.vectorstores.neo4j_vector", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "logging", - "newrelic.hooks.logger_logging", - "instrument_logging", + "langchain_community.vectorstores.thirdai_neuraldb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "loguru", - "newrelic.hooks.logger_loguru", - "instrument_loguru", + "langchain_community.vectorstores.nucliadb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) + _process_module_definition( - "loguru._logger", - "newrelic.hooks.logger_loguru", - "instrument_loguru_logger", + "langchain_community.vectorstores.opensearch_vector_search", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", ) _process_module_definition( - "paste.httpserver", - "newrelic.hooks.adapter_paste", - "instrument_paste_httpserver", + "langchain_community.vectorstores.oraclevs", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.pathway", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.pgembedding", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.pgvecto_rs", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.pgvector", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.pinecone", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.qdrant", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.relyt", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.rocksetdb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.scann", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.semadb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.singlestoredb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.sklearn", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.sqlitevec", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.sqlitevss", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.starrocks", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.supabase", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.surrealdb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.tair", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.tencentvectordb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.thirdai_neuraldb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.tidb_vector", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.tigris", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.tiledb", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.timescalevector", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.typesense", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.upstash", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.usearch", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.vald", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.vdms", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.vearch", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.vectara", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.vespa", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.vlite", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.weaviate", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.xata", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.yellowbrick", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.zep_cloud", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_community.vectorstores.zep", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_vectorstore_similarity_search", + ) + + _process_module_definition( + "langchain_core.tools", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_core_tools", + ) + + _process_module_definition( + "langchain_core.callbacks.manager", + "newrelic.hooks.mlmodel_langchain", + "instrument_langchain_callbacks_manager", + ) + + _process_module_definition( + "asyncio.events", + "newrelic.hooks.coroutines_asyncio", + "instrument_asyncio_events", + ) + + _process_module_definition("asgiref.sync", "newrelic.hooks.adapter_asgiref", "instrument_asgiref_sync") + + _process_module_definition( + "django.core.handlers.base", + "newrelic.hooks.framework_django", + "instrument_django_core_handlers_base", + ) + _process_module_definition( + "django.core.handlers.asgi", + "newrelic.hooks.framework_django", + "instrument_django_core_handlers_asgi", + ) + _process_module_definition( + "django.core.handlers.wsgi", + "newrelic.hooks.framework_django", + "instrument_django_core_handlers_wsgi", + ) + _process_module_definition( + "django.core.urlresolvers", + "newrelic.hooks.framework_django", + "instrument_django_core_urlresolvers", + ) + _process_module_definition( + "django.template", + "newrelic.hooks.framework_django", + "instrument_django_template", + ) + _process_module_definition( + "django.template.loader_tags", + "newrelic.hooks.framework_django", + "instrument_django_template_loader_tags", + ) + _process_module_definition( + "django.core.servers.basehttp", + "newrelic.hooks.framework_django", + "instrument_django_core_servers_basehttp", + ) + _process_module_definition( + "django.contrib.staticfiles.views", + "newrelic.hooks.framework_django", + "instrument_django_contrib_staticfiles_views", + ) + _process_module_definition( + "django.contrib.staticfiles.handlers", + "newrelic.hooks.framework_django", + "instrument_django_contrib_staticfiles_handlers", + ) + _process_module_definition( + "django.views.debug", + "newrelic.hooks.framework_django", + "instrument_django_views_debug", + ) + _process_module_definition( + "django.http.multipartparser", + "newrelic.hooks.framework_django", + "instrument_django_http_multipartparser", + ) + _process_module_definition( + "django.core.mail", + "newrelic.hooks.framework_django", + "instrument_django_core_mail", + ) + _process_module_definition( + "django.core.mail.message", + "newrelic.hooks.framework_django", + "instrument_django_core_mail_message", + ) + _process_module_definition( + "django.views.generic.base", + "newrelic.hooks.framework_django", + "instrument_django_views_generic_base", + ) + _process_module_definition( + "django.core.management.base", + "newrelic.hooks.framework_django", + "instrument_django_core_management_base", + ) + _process_module_definition( + "django.template.base", + "newrelic.hooks.framework_django", + "instrument_django_template_base", + ) + _process_module_definition( + "django.middleware.gzip", + "newrelic.hooks.framework_django", + "instrument_django_gzip_middleware", + ) + + # New modules in Django 1.10 + _process_module_definition( + "django.urls.resolvers", + "newrelic.hooks.framework_django", + "instrument_django_core_urlresolvers", + ) + _process_module_definition( + "django.urls.base", + "newrelic.hooks.framework_django", + "instrument_django_urls_base", + ) + _process_module_definition( + "django.core.handlers.exception", + "newrelic.hooks.framework_django", + "instrument_django_core_handlers_exception", + ) + + _process_module_definition("falcon.api", "newrelic.hooks.framework_falcon", "instrument_falcon_api") + _process_module_definition("falcon.app", "newrelic.hooks.framework_falcon", "instrument_falcon_app") + _process_module_definition( + "falcon.routing.util", + "newrelic.hooks.framework_falcon", + "instrument_falcon_routing_util", + ) + + _process_module_definition( + "fastapi.routing", + "newrelic.hooks.framework_fastapi", + "instrument_fastapi_routing", + ) + + _process_module_definition("flask.app", "newrelic.hooks.framework_flask", "instrument_flask_app") + _process_module_definition( + "flask.templating", + "newrelic.hooks.framework_flask", + "instrument_flask_templating", + ) + _process_module_definition( + "flask.blueprints", + "newrelic.hooks.framework_flask", + "instrument_flask_blueprints", + ) + _process_module_definition("flask.views", "newrelic.hooks.framework_flask", "instrument_flask_views") + + _process_module_definition( + "flask_compress", + "newrelic.hooks.middleware_flask_compress", + "instrument_flask_compress", + ) + + _process_module_definition("flask_restful", "newrelic.hooks.component_flask_rest", "instrument_flask_rest") + _process_module_definition( + "flask_restplus.api", + "newrelic.hooks.component_flask_rest", + "instrument_flask_rest", + ) + _process_module_definition( + "flask_restx.api", + "newrelic.hooks.component_flask_rest", + "instrument_flask_rest", + ) + + _process_module_definition( + "graphql_server", + "newrelic.hooks.component_graphqlserver", + "instrument_graphqlserver", + ) + + _process_module_definition( + "sentry_sdk.integrations.asgi", "newrelic.hooks.component_sentry", "instrument_sentry_sdk_integrations_asgi" + ) + + _process_module_definition("httpx._client", "newrelic.hooks.external_httpx", "instrument_httpx_client") + + _process_module_definition("gluon.contrib.feedparser", "newrelic.hooks.external_feedparser") + _process_module_definition("gluon.contrib.memcache.memcache", "newrelic.hooks.memcache_memcache") + + _process_module_definition( + "graphene.types.schema", + "newrelic.hooks.framework_graphene", + "instrument_graphene_types_schema", + ) + + _process_module_definition( + "graphql.graphql", + "newrelic.hooks.framework_graphql", + "instrument_graphql", + ) + _process_module_definition( + "graphql.execution.execute", + "newrelic.hooks.framework_graphql", + "instrument_graphql_execute", + ) + _process_module_definition( + "graphql.execution.executor", + "newrelic.hooks.framework_graphql", + "instrument_graphql_execute", + ) + _process_module_definition( + "graphql.execution.middleware", + "newrelic.hooks.framework_graphql", + "instrument_graphql_execution_middleware", + ) + _process_module_definition( + "graphql.execution.utils", + "newrelic.hooks.framework_graphql", + "instrument_graphql_execution_utils", + ) + _process_module_definition( + "graphql.error.located_error", + "newrelic.hooks.framework_graphql", + "instrument_graphql_error_located_error", + ) + _process_module_definition( + "graphql.language.parser", + "newrelic.hooks.framework_graphql", + "instrument_graphql_parser", + ) + _process_module_definition( + "graphql.validation.validate", + "newrelic.hooks.framework_graphql", + "instrument_graphql_validate", + ) + _process_module_definition( + "graphql.validation.validation", + "newrelic.hooks.framework_graphql", + "instrument_graphql_validate", + ) + + _process_module_definition( + "graphql.type.schema", + "newrelic.hooks.framework_graphql", + "instrument_graphql_schema_get_field", + ) + + _process_module_definition( + "google.cloud.firestore_v1.base_client", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_base_client", + ) + _process_module_definition( + "google.cloud.firestore_v1.client", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_client", + ) + _process_module_definition( + "google.cloud.firestore_v1.async_client", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_async_client", + ) + _process_module_definition( + "google.cloud.firestore_v1.document", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_document", + ) + _process_module_definition( + "google.cloud.firestore_v1.async_document", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_async_document", + ) + _process_module_definition( + "google.cloud.firestore_v1.collection", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_collection", + ) + _process_module_definition( + "google.cloud.firestore_v1.async_collection", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_async_collection", + ) + _process_module_definition( + "google.cloud.firestore_v1.query", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_query", + ) + _process_module_definition( + "google.cloud.firestore_v1.async_query", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_async_query", + ) + _process_module_definition( + "google.cloud.firestore_v1.aggregation", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_aggregation", + ) + _process_module_definition( + "google.cloud.firestore_v1.async_aggregation", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_async_aggregation", + ) + _process_module_definition( + "google.cloud.firestore_v1.batch", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_batch", + ) + _process_module_definition( + "google.cloud.firestore_v1.async_batch", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_async_batch", + ) + _process_module_definition( + "google.cloud.firestore_v1.bulk_batch", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_bulk_batch", + ) + _process_module_definition( + "google.cloud.firestore_v1.transaction", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_transaction", + ) + _process_module_definition( + "google.cloud.firestore_v1.async_transaction", + "newrelic.hooks.datastore_firestore", + "instrument_google_cloud_firestore_v1_async_transaction", + ) + + _process_module_definition( + "ariadne.asgi", + "newrelic.hooks.framework_ariadne", + "instrument_ariadne_asgi", + ) + _process_module_definition( + "ariadne.graphql", + "newrelic.hooks.framework_ariadne", + "instrument_ariadne_execute", + ) + _process_module_definition( + "ariadne.wsgi", + "newrelic.hooks.framework_ariadne", + "instrument_ariadne_wsgi", + ) + + _process_module_definition("grpc._channel", "newrelic.hooks.framework_grpc", "instrument_grpc__channel") + _process_module_definition("grpc._server", "newrelic.hooks.framework_grpc", "instrument_grpc_server") + + _process_module_definition("bottle", "newrelic.hooks.framework_bottle", "instrument_bottle") + + _process_module_definition( + "cherrypy._cpreqbody", + "newrelic.hooks.framework_cherrypy", + "instrument_cherrypy__cpreqbody", + ) + _process_module_definition( + "cherrypy._cprequest", + "newrelic.hooks.framework_cherrypy", + "instrument_cherrypy__cprequest", + ) + _process_module_definition( + "cherrypy._cpdispatch", + "newrelic.hooks.framework_cherrypy", + "instrument_cherrypy__cpdispatch", + ) + _process_module_definition( + "cherrypy._cpwsgi", + "newrelic.hooks.framework_cherrypy", + "instrument_cherrypy__cpwsgi", + ) + _process_module_definition( + "cherrypy._cptree", + "newrelic.hooks.framework_cherrypy", + "instrument_cherrypy__cptree", + ) + + _process_module_definition( + "confluent_kafka.cimpl", + "newrelic.hooks.messagebroker_confluentkafka", + "instrument_confluentkafka_cimpl", + ) + _process_module_definition( + "confluent_kafka.serializing_producer", + "newrelic.hooks.messagebroker_confluentkafka", + "instrument_confluentkafka_serializing_producer", + ) + _process_module_definition( + "confluent_kafka.deserializing_consumer", + "newrelic.hooks.messagebroker_confluentkafka", + "instrument_confluentkafka_deserializing_consumer", + ) + + _process_module_definition( + "kafka.consumer.group", + "newrelic.hooks.messagebroker_kafkapython", + "instrument_kafka_consumer_group", + ) + _process_module_definition( + "kafka.producer.kafka", + "newrelic.hooks.messagebroker_kafkapython", + "instrument_kafka_producer", + ) + _process_module_definition( + "kafka.coordinator.heartbeat", + "newrelic.hooks.messagebroker_kafkapython", + "instrument_kafka_heartbeat", + ) + + _process_module_definition( + "logging", + "newrelic.hooks.logger_logging", + "instrument_logging", + ) + + _process_module_definition( + "loguru", + "newrelic.hooks.logger_loguru", + "instrument_loguru", + ) + _process_module_definition( + "loguru._logger", + "newrelic.hooks.logger_loguru", + "instrument_loguru_logger", + ) + _process_module_definition( + "structlog._base", + "newrelic.hooks.logger_structlog", + "instrument_structlog__base", + ) + _process_module_definition( + "structlog._frames", + "newrelic.hooks.logger_structlog", + "instrument_structlog__frames", + ) + _process_module_definition( + "paste.httpserver", + "newrelic.hooks.adapter_paste", + "instrument_paste_httpserver", ) _process_module_definition( @@ -2374,514 +3114,1286 @@ def _process_module_builtin_defaults(): "instrument_gunicorn_app_base", ) - _process_module_definition("cx_Oracle", "newrelic.hooks.database_cx_oracle", "instrument_cx_oracle") + _process_module_definition("cx_Oracle", "newrelic.hooks.database_cx_oracle", "instrument_cx_oracle") + + _process_module_definition("ibm_db_dbi", "newrelic.hooks.database_ibm_db_dbi", "instrument_ibm_db_dbi") + + _process_module_definition("mysql.connector", "newrelic.hooks.database_mysql", "instrument_mysql_connector") + _process_module_definition("MySQLdb", "newrelic.hooks.database_mysqldb", "instrument_mysqldb") + _process_module_definition("pymysql", "newrelic.hooks.database_pymysql", "instrument_pymysql") + + _process_module_definition("pyodbc", "newrelic.hooks.database_pyodbc", "instrument_pyodbc") + + _process_module_definition("pymssql", "newrelic.hooks.database_pymssql", "instrument_pymssql") + + _process_module_definition("psycopg", "newrelic.hooks.database_psycopg", "instrument_psycopg") + _process_module_definition("psycopg.sql", "newrelic.hooks.database_psycopg", "instrument_psycopg_sql") + + _process_module_definition("psycopg2", "newrelic.hooks.database_psycopg2", "instrument_psycopg2") + _process_module_definition( + "psycopg2._psycopg2", + "newrelic.hooks.database_psycopg2", + "instrument_psycopg2__psycopg2", + ) + _process_module_definition( + "psycopg2.extensions", + "newrelic.hooks.database_psycopg2", + "instrument_psycopg2_extensions", + ) + _process_module_definition( + "psycopg2._json", + "newrelic.hooks.database_psycopg2", + "instrument_psycopg2__json", + ) + _process_module_definition( + "psycopg2._range", + "newrelic.hooks.database_psycopg2", + "instrument_psycopg2__range", + ) + _process_module_definition("psycopg2.sql", "newrelic.hooks.database_psycopg2", "instrument_psycopg2_sql") + + _process_module_definition("psycopg2ct", "newrelic.hooks.database_psycopg2ct", "instrument_psycopg2ct") + _process_module_definition( + "psycopg2ct.extensions", + "newrelic.hooks.database_psycopg2ct", + "instrument_psycopg2ct_extensions", + ) + + _process_module_definition( + "psycopg2cffi", + "newrelic.hooks.database_psycopg2cffi", + "instrument_psycopg2cffi", + ) + _process_module_definition( + "psycopg2cffi.extensions", + "newrelic.hooks.database_psycopg2cffi", + "instrument_psycopg2cffi_extensions", + ) + + _process_module_definition( + "asyncpg.connect_utils", + "newrelic.hooks.database_asyncpg", + "instrument_asyncpg_connect_utils", + ) + _process_module_definition( + "asyncpg.protocol", + "newrelic.hooks.database_asyncpg", + "instrument_asyncpg_protocol", + ) + + _process_module_definition( + "postgresql.driver.dbapi20", + "newrelic.hooks.database_postgresql", + "instrument_postgresql_driver_dbapi20", + ) + + _process_module_definition( + "postgresql.interface.proboscis.dbapi2", + "newrelic.hooks.database_postgresql", + "instrument_postgresql_interface_proboscis_dbapi2", + ) + + _process_module_definition("sqlite3", "newrelic.hooks.database_sqlite", "instrument_sqlite3") + _process_module_definition("sqlite3.dbapi2", "newrelic.hooks.database_sqlite", "instrument_sqlite3_dbapi2") + + _process_module_definition("pysqlite2", "newrelic.hooks.database_sqlite", "instrument_sqlite3") + _process_module_definition( + "pysqlite2.dbapi2", + "newrelic.hooks.database_sqlite", + "instrument_sqlite3_dbapi2", + ) + + _process_module_definition("memcache", "newrelic.hooks.datastore_memcache", "instrument_memcache") + _process_module_definition( + "pylibmc.client", + "newrelic.hooks.datastore_pylibmc", + "instrument_pylibmc_client", + ) + _process_module_definition( + "bmemcached.client", + "newrelic.hooks.datastore_bmemcached", + "instrument_bmemcached_client", + ) + _process_module_definition( + "pymemcache.client", + "newrelic.hooks.datastore_pymemcache", + "instrument_pymemcache_client", + ) + _process_module_definition( + "aiomcache.client", + "newrelic.hooks.datastore_aiomcache", + "instrument_aiomcache_client", + ) + + _process_module_definition("jinja2.environment", "newrelic.hooks.template_jinja2") + + _process_module_definition("mako.runtime", "newrelic.hooks.template_mako", "instrument_mako_runtime") + _process_module_definition("mako.template", "newrelic.hooks.template_mako", "instrument_mako_template") + + _process_module_definition("genshi.template.base", "newrelic.hooks.template_genshi") + + _process_module_definition("http.client", "newrelic.hooks.external_httplib") + + _process_module_definition("httplib2", "newrelic.hooks.external_httplib2") + + _process_module_definition("urllib.request", "newrelic.hooks.external_urllib") + + _process_module_definition( + "urllib3.connectionpool", + "newrelic.hooks.external_urllib3", + "instrument_urllib3_connectionpool", + ) + _process_module_definition( + "urllib3.connection", + "newrelic.hooks.external_urllib3", + "instrument_urllib3_connection", + ) + _process_module_definition( + "requests.packages.urllib3.connection", + "newrelic.hooks.external_urllib3", + "instrument_urllib3_connection", + ) + + _process_module_definition( + "starlette.requests", + "newrelic.hooks.framework_starlette", + "instrument_starlette_requests", + ) + _process_module_definition( + "starlette.routing", + "newrelic.hooks.framework_starlette", + "instrument_starlette_routing", + ) + _process_module_definition( + "starlette.applications", + "newrelic.hooks.framework_starlette", + "instrument_starlette_applications", + ) + _process_module_definition( + "starlette.middleware.errors", + "newrelic.hooks.framework_starlette", + "instrument_starlette_middleware_errors", + ) + _process_module_definition( + "starlette.middleware.exceptions", + "newrelic.hooks.framework_starlette", + "instrument_starlette_middleware_exceptions", + ) + _process_module_definition( + "starlette.exceptions", + "newrelic.hooks.framework_starlette", + "instrument_starlette_exceptions", + ) + _process_module_definition( + "starlette.background", + "newrelic.hooks.framework_starlette", + "instrument_starlette_background_task", + ) + _process_module_definition( + "starlette.concurrency", + "newrelic.hooks.framework_starlette", + "instrument_starlette_concurrency", + ) + + _process_module_definition( + "strawberry.asgi", + "newrelic.hooks.framework_strawberry", + "instrument_strawberry_asgi", + ) + + _process_module_definition( + "strawberry.schema.schema", + "newrelic.hooks.framework_strawberry", + "instrument_strawberry_schema", + ) + + _process_module_definition( + "strawberry.schema.schema_converter", + "newrelic.hooks.framework_strawberry", + "instrument_strawberry_schema_converter", + ) + + _process_module_definition("uvicorn.config", "newrelic.hooks.adapter_uvicorn", "instrument_uvicorn_config") + + _process_module_definition( + "hypercorn.asyncio.run", "newrelic.hooks.adapter_hypercorn", "instrument_hypercorn_asyncio_run" + ) + _process_module_definition( + "hypercorn.trio.run", "newrelic.hooks.adapter_hypercorn", "instrument_hypercorn_trio_run" + ) + _process_module_definition("hypercorn.utils", "newrelic.hooks.adapter_hypercorn", "instrument_hypercorn_utils") + + _process_module_definition("daphne.server", "newrelic.hooks.adapter_daphne", "instrument_daphne_server") + + _process_module_definition("sanic.app", "newrelic.hooks.framework_sanic", "instrument_sanic_app") + _process_module_definition("sanic.response", "newrelic.hooks.framework_sanic", "instrument_sanic_response") + _process_module_definition( + "sanic.touchup.service", "newrelic.hooks.framework_sanic", "instrument_sanic_touchup_service" + ) + + _process_module_definition("aiohttp.wsgi", "newrelic.hooks.framework_aiohttp", "instrument_aiohttp_wsgi") + _process_module_definition("aiohttp.web", "newrelic.hooks.framework_aiohttp", "instrument_aiohttp_web") + _process_module_definition( + "aiohttp.web_reqrep", + "newrelic.hooks.framework_aiohttp", + "instrument_aiohttp_web_response", + ) + _process_module_definition( + "aiohttp.web_response", + "newrelic.hooks.framework_aiohttp", + "instrument_aiohttp_web_response", + ) + _process_module_definition( + "aiohttp.web_urldispatcher", + "newrelic.hooks.framework_aiohttp", + "instrument_aiohttp_web_urldispatcher", + ) + _process_module_definition( + "aiohttp.client", + "newrelic.hooks.framework_aiohttp", + "instrument_aiohttp_client", + ) + _process_module_definition( + "aiohttp.client_reqrep", + "newrelic.hooks.framework_aiohttp", + "instrument_aiohttp_client_reqrep", + ) + _process_module_definition( + "aiohttp.protocol", + "newrelic.hooks.framework_aiohttp", + "instrument_aiohttp_protocol", + ) + + _process_module_definition("requests.api", "newrelic.hooks.external_requests", "instrument_requests_api") + _process_module_definition( + "requests.sessions", + "newrelic.hooks.external_requests", + "instrument_requests_sessions", + ) + + _process_module_definition("feedparser", "newrelic.hooks.external_feedparser") + + _process_module_definition("xmlrpclib", "newrelic.hooks.external_xmlrpclib") + + _process_module_definition("dropbox", "newrelic.hooks.external_dropbox") + + _process_module_definition("facepy.graph_api", "newrelic.hooks.external_facepy") + + _process_module_definition("pysolr", "newrelic.hooks.datastore_pysolr", "instrument_pysolr") + + _process_module_definition("solr", "newrelic.hooks.datastore_solrpy", "instrument_solrpy") + + _process_module_definition("aredis.client", "newrelic.hooks.datastore_aredis", "instrument_aredis_client") + + _process_module_definition( + "aredis.connection", + "newrelic.hooks.datastore_aredis", + "instrument_aredis_connection", + ) + + _process_module_definition("aioredis.client", "newrelic.hooks.datastore_aioredis", "instrument_aioredis_client") + + _process_module_definition("aioredis.commands", "newrelic.hooks.datastore_aioredis", "instrument_aioredis_client") + + _process_module_definition( + "aioredis.connection", "newrelic.hooks.datastore_aioredis", "instrument_aioredis_connection" + ) + + # v7 and below + _process_module_definition( + "elasticsearch.client", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client", + ) + # v8 and above + _process_module_definition( + "elasticsearch._sync.client", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_v8", + ) + + # v7 and below + _process_module_definition( + "elasticsearch.client.cat", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_cat", + ) + # v8 and above + _process_module_definition( + "elasticsearch._sync.client.cat", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_cat_v8", + ) + + # v7 and below + _process_module_definition( + "elasticsearch.client.cluster", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_cluster", + ) + # v8 and above + _process_module_definition( + "elasticsearch._sync.client.cluster", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_cluster_v8", + ) + + # v7 and below + _process_module_definition( + "elasticsearch.client.indices", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_indices", + ) + # v8 and above + _process_module_definition( + "elasticsearch._sync.client.indices", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_indices_v8", + ) + + # v7 and below + _process_module_definition( + "elasticsearch.client.nodes", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_nodes", + ) + # v8 and above + _process_module_definition( + "elasticsearch._sync.client.nodes", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_nodes_v8", + ) + + # v7 and below + _process_module_definition( + "elasticsearch.client.snapshot", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_snapshot", + ) + # v8 and above + _process_module_definition( + "elasticsearch._sync.client.snapshot", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_snapshot_v8", + ) + + # v7 and below + _process_module_definition( + "elasticsearch.client.tasks", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_tasks", + ) + # v8 and above + _process_module_definition( + "elasticsearch._sync.client.tasks", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_tasks_v8", + ) + + # v7 and below + _process_module_definition( + "elasticsearch.client.ingest", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_ingest", + ) + # v8 and above + _process_module_definition( + "elasticsearch._sync.client.ingest", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_client_ingest_v8", + ) + + # v7 and below + _process_module_definition( + "elasticsearch.connection.base", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_connection_base", + ) + # v8 and above + _process_module_definition( + "elastic_transport._node._base", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elastic_transport__node__base", + ) + + # v7 and below + _process_module_definition( + "elasticsearch.transport", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elasticsearch_transport", + ) + # v8 and above + _process_module_definition( + "elastic_transport._transport", + "newrelic.hooks.datastore_elasticsearch", + "instrument_elastic_transport__transport", + ) + + _process_module_definition("pika.adapters", "newrelic.hooks.messagebroker_pika", "instrument_pika_adapters") + _process_module_definition("pika.channel", "newrelic.hooks.messagebroker_pika", "instrument_pika_channel") + _process_module_definition("pika.spec", "newrelic.hooks.messagebroker_pika", "instrument_pika_spec") + + _process_module_definition( + "pyelasticsearch.client", + "newrelic.hooks.datastore_pyelasticsearch", + "instrument_pyelasticsearch_client", + ) + + # Newer pymongo module locations + _process_module_definition( + "pymongo.synchronous.pool", + "newrelic.hooks.datastore_pymongo", + "instrument_pymongo_pool", + ) + _process_module_definition( + "pymongo.synchronous.collection", + "newrelic.hooks.datastore_pymongo", + "instrument_pymongo_collection", + ) + _process_module_definition( + "pymongo.synchronous.mongo_client", + "newrelic.hooks.datastore_pymongo", + "instrument_pymongo_mongo_client", + ) + + # Older pymongo module locations + _process_module_definition( + "pymongo.connection", + "newrelic.hooks.datastore_pymongo", + "instrument_pymongo_pool", + ) + _process_module_definition( + "pymongo.collection", + "newrelic.hooks.datastore_pymongo", + "instrument_pymongo_collection", + ) + _process_module_definition( + "pymongo.mongo_client", + "newrelic.hooks.datastore_pymongo", + "instrument_pymongo_mongo_client", + ) + + # Redis v4.2+ + _process_module_definition( + "redis.asyncio.client", "newrelic.hooks.datastore_redis", "instrument_asyncio_redis_client" + ) + + # Redis v4.2+ + _process_module_definition( + "redis.asyncio.commands", "newrelic.hooks.datastore_redis", "instrument_asyncio_redis_client" + ) + + # Redis v4.2+ + _process_module_definition( + "redis.asyncio.connection", "newrelic.hooks.datastore_redis", "instrument_asyncio_redis_connection" + ) - _process_module_definition("ibm_db_dbi", "newrelic.hooks.database_ibm_db_dbi", "instrument_ibm_db_dbi") + _process_module_definition( + "redis.connection", + "newrelic.hooks.datastore_redis", + "instrument_redis_connection", + ) + _process_module_definition("redis.client", "newrelic.hooks.datastore_redis", "instrument_redis_client") - _process_module_definition("mysql.connector", "newrelic.hooks.database_mysql", "instrument_mysql_connector") - _process_module_definition("MySQLdb", "newrelic.hooks.database_mysqldb", "instrument_mysqldb") - _process_module_definition("oursql", "newrelic.hooks.database_oursql", "instrument_oursql") - _process_module_definition("pymysql", "newrelic.hooks.database_pymysql", "instrument_pymysql") + _process_module_definition( + "redis.commands.cluster", "newrelic.hooks.datastore_redis", "instrument_redis_commands_cluster" + ) - _process_module_definition("pyodbc", "newrelic.hooks.database_pyodbc", "instrument_pyodbc") + _process_module_definition( + "redis.commands.core", "newrelic.hooks.datastore_redis", "instrument_redis_commands_core" + ) - _process_module_definition("pymssql", "newrelic.hooks.database_pymssql", "instrument_pymssql") + _process_module_definition( + "redis.commands.sentinel", "newrelic.hooks.datastore_redis", "instrument_redis_commands_sentinel" + ) - _process_module_definition("psycopg2", "newrelic.hooks.database_psycopg2", "instrument_psycopg2") _process_module_definition( - "psycopg2._psycopg2", - "newrelic.hooks.database_psycopg2", - "instrument_psycopg2__psycopg2", + "redis.commands.json.commands", "newrelic.hooks.datastore_redis", "instrument_redis_commands_json_commands" ) + _process_module_definition( - "psycopg2.extensions", - "newrelic.hooks.database_psycopg2", - "instrument_psycopg2_extensions", + "redis.commands.search.commands", "newrelic.hooks.datastore_redis", "instrument_redis_commands_search_commands" ) + _process_module_definition( - "psycopg2._json", - "newrelic.hooks.database_psycopg2", - "instrument_psycopg2__json", + "redis.commands.timeseries.commands", + "newrelic.hooks.datastore_redis", + "instrument_redis_commands_timeseries_commands", ) + _process_module_definition( - "psycopg2._range", - "newrelic.hooks.database_psycopg2", - "instrument_psycopg2__range", + "redis.commands.bf.commands", "newrelic.hooks.datastore_redis", "instrument_redis_commands_bf_commands" ) - _process_module_definition("psycopg2.sql", "newrelic.hooks.database_psycopg2", "instrument_psycopg2_sql") - _process_module_definition("psycopg2ct", "newrelic.hooks.database_psycopg2ct", "instrument_psycopg2ct") _process_module_definition( - "psycopg2ct.extensions", - "newrelic.hooks.database_psycopg2ct", - "instrument_psycopg2ct_extensions", + "redis.commands.graph.commands", "newrelic.hooks.datastore_redis", "instrument_redis_commands_graph_commands" ) + _process_module_definition("motor", "newrelic.hooks.datastore_motor", "patch_motor") + _process_module_definition( - "psycopg2cffi", - "newrelic.hooks.database_psycopg2cffi", - "instrument_psycopg2cffi", + "piston.resource", + "newrelic.hooks.component_piston", + "instrument_piston_resource", ) + _process_module_definition("piston.doc", "newrelic.hooks.component_piston", "instrument_piston_doc") + _process_module_definition( - "psycopg2cffi.extensions", - "newrelic.hooks.database_psycopg2cffi", - "instrument_psycopg2cffi_extensions", + "tastypie.resources", + "newrelic.hooks.component_tastypie", + "instrument_tastypie_resources", ) + _process_module_definition("tastypie.api", "newrelic.hooks.component_tastypie", "instrument_tastypie_api") _process_module_definition( - "asyncpg.connect_utils", - "newrelic.hooks.database_asyncpg", - "instrument_asyncpg_connect_utils", + "sklearn.metrics", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_metrics", + ) + + _process_module_definition( + "sklearn.tree._classes", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_tree_models", ) + # In scikit-learn < 0.21 the model classes are in tree.py instead of _classes.py. _process_module_definition( - "asyncpg.protocol", - "newrelic.hooks.database_asyncpg", - "instrument_asyncpg_protocol", + "sklearn.tree.tree", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_tree_models", ) _process_module_definition( - "postgresql.driver.dbapi20", - "newrelic.hooks.database_postgresql", - "instrument_postgresql_driver_dbapi20", + "sklearn.compose._column_transformer", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_compose_models", ) _process_module_definition( - "postgresql.interface.proboscis.dbapi2", - "newrelic.hooks.database_postgresql", - "instrument_postgresql_interface_proboscis_dbapi2", + "sklearn.compose._target", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_compose_models", ) - _process_module_definition("sqlite3", "newrelic.hooks.database_sqlite", "instrument_sqlite3") - _process_module_definition("sqlite3.dbapi2", "newrelic.hooks.database_sqlite", "instrument_sqlite3_dbapi2") + _process_module_definition( + "sklearn.covariance._empirical_covariance", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_models", + ) - _process_module_definition("pysqlite2", "newrelic.hooks.database_sqlite", "instrument_sqlite3") _process_module_definition( - "pysqlite2.dbapi2", - "newrelic.hooks.database_sqlite", - "instrument_sqlite3_dbapi2", + "sklearn.covariance.empirical_covariance_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_models", ) - _process_module_definition("memcache", "newrelic.hooks.datastore_memcache", "instrument_memcache") - _process_module_definition("umemcache", "newrelic.hooks.datastore_umemcache", "instrument_umemcache") _process_module_definition( - "pylibmc.client", - "newrelic.hooks.datastore_pylibmc", - "instrument_pylibmc_client", + "sklearn.covariance.shrunk_covariance_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_shrunk_models", ) + _process_module_definition( - "bmemcached.client", - "newrelic.hooks.datastore_bmemcached", - "instrument_bmemcached_client", + "sklearn.covariance._shrunk_covariance", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_shrunk_models", ) + _process_module_definition( - "pymemcache.client", - "newrelic.hooks.datastore_pymemcache", - "instrument_pymemcache_client", + "sklearn.covariance.robust_covariance_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_models", ) - _process_module_definition("jinja2.environment", "newrelic.hooks.template_jinja2") + _process_module_definition( + "sklearn.covariance._robust_covariance", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_models", + ) - _process_module_definition("mako.runtime", "newrelic.hooks.template_mako", "instrument_mako_runtime") - _process_module_definition("mako.template", "newrelic.hooks.template_mako", "instrument_mako_template") + _process_module_definition( + "sklearn.covariance.graph_lasso_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_graph_models", + ) - _process_module_definition("genshi.template.base", "newrelic.hooks.template_genshi") + _process_module_definition( + "sklearn.covariance._graph_lasso", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_graph_models", + ) - if six.PY2: - _process_module_definition("httplib", "newrelic.hooks.external_httplib") - else: - _process_module_definition("http.client", "newrelic.hooks.external_httplib") + _process_module_definition( + "sklearn.covariance.elliptic_envelope", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_models", + ) - _process_module_definition("httplib2", "newrelic.hooks.external_httplib2") + _process_module_definition( + "sklearn.covariance._elliptic_envelope", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_covariance_models", + ) - if six.PY2: - _process_module_definition("urllib", "newrelic.hooks.external_urllib") - else: - _process_module_definition("urllib.request", "newrelic.hooks.external_urllib") + _process_module_definition( + "sklearn.ensemble._bagging", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_bagging_models", + ) - if six.PY2: - _process_module_definition("urllib2", "newrelic.hooks.external_urllib2") + _process_module_definition( + "sklearn.ensemble.bagging", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_bagging_models", + ) _process_module_definition( - "urllib3.connectionpool", - "newrelic.hooks.external_urllib3", - "instrument_urllib3_connectionpool", + "sklearn.ensemble._forest", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_forest_models", + ) + + _process_module_definition( + "sklearn.ensemble.forest", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_forest_models", ) + _process_module_definition( - "urllib3.connection", - "newrelic.hooks.external_urllib3", - "instrument_urllib3_connection", + "sklearn.ensemble._iforest", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_iforest_models", ) + _process_module_definition( - "requests.packages.urllib3.connection", - "newrelic.hooks.external_urllib3", - "instrument_urllib3_connection", + "sklearn.ensemble.iforest", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_iforest_models", ) _process_module_definition( - "starlette.requests", - "newrelic.hooks.framework_starlette", - "instrument_starlette_requests", + "sklearn.ensemble._weight_boosting", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_weight_boosting_models", ) + _process_module_definition( - "starlette.routing", - "newrelic.hooks.framework_starlette", - "instrument_starlette_routing", + "sklearn.ensemble.weight_boosting", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_weight_boosting_models", ) + _process_module_definition( - "starlette.applications", - "newrelic.hooks.framework_starlette", - "instrument_starlette_applications", + "sklearn.ensemble._gb", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_gradient_boosting_models", ) + _process_module_definition( - "starlette.middleware.errors", - "newrelic.hooks.framework_starlette", - "instrument_starlette_middleware_errors", + "sklearn.ensemble.gradient_boosting", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_gradient_boosting_models", ) + _process_module_definition( - "starlette.middleware.exceptions", - "newrelic.hooks.framework_starlette", - "instrument_starlette_middleware_exceptions", + "sklearn.ensemble._voting", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_voting_models", ) + _process_module_definition( - "starlette.exceptions", - "newrelic.hooks.framework_starlette", - "instrument_starlette_exceptions", + "sklearn.ensemble.voting_classifier", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_voting_models", ) + _process_module_definition( - "starlette.background", - "newrelic.hooks.framework_starlette", - "instrument_starlette_background_task", + "sklearn.ensemble._stacking", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_stacking_models", + ) + + _process_module_definition( + "sklearn.ensemble._hist_gradient_boosting.gradient_boosting", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_ensemble_hist_models", + ) + + _process_module_definition( + "sklearn.linear_model._base", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model.base", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model._bayes", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_bayes_models", + ) + + _process_module_definition( + "sklearn.linear_model.bayes", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_bayes_models", + ) + + _process_module_definition( + "sklearn.linear_model._least_angle", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_least_angle_models", + ) + + _process_module_definition( + "sklearn.linear_model.least_angle", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_least_angle_models", + ) + + _process_module_definition( + "sklearn.linear_model.coordinate_descent", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_coordinate_descent_models", + ) + + _process_module_definition( + "sklearn.linear_model._coordinate_descent", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_coordinate_descent_models", + ) + + _process_module_definition( + "sklearn.linear_model._glm", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_GLM_models", + ) + + _process_module_definition( + "sklearn.linear_model._huber", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model.huber", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model._stochastic_gradient", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_stochastic_gradient_models", + ) + + _process_module_definition( + "sklearn.linear_model.stochastic_gradient", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_stochastic_gradient_models", + ) + + _process_module_definition( + "sklearn.linear_model._ridge", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_ridge_models", + ) + + _process_module_definition( + "sklearn.linear_model.ridge", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_ridge_models", + ) + + _process_module_definition( + "sklearn.linear_model._logistic", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_logistic_models", + ) + + _process_module_definition( + "sklearn.linear_model.logistic", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_logistic_models", + ) + + _process_module_definition( + "sklearn.linear_model._omp", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_OMP_models", + ) + + _process_module_definition( + "sklearn.linear_model.omp", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_OMP_models", + ) + + _process_module_definition( + "sklearn.linear_model._passive_aggressive", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_passive_aggressive_models", + ) + + _process_module_definition( + "sklearn.linear_model.passive_aggressive", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_passive_aggressive_models", + ) + + _process_module_definition( + "sklearn.linear_model._perceptron", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model.perceptron", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model._quantile", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model._ransac", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model.ransac", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model._theil_sen", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.linear_model.theil_sen", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_linear_models", + ) + + _process_module_definition( + "sklearn.cross_decomposition._pls", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cross_decomposition_models", + ) + + _process_module_definition( + "sklearn.cross_decomposition.pls_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cross_decomposition_models", ) + _process_module_definition( - "starlette.concurrency", - "newrelic.hooks.framework_starlette", - "instrument_starlette_concurrency", + "sklearn.discriminant_analysis", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_discriminant_analysis_models", ) _process_module_definition( - "strawberry.asgi", - "newrelic.hooks.framework_strawberry", - "instrument_strawberry_asgi", + "sklearn.gaussian_process._gpc", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_gaussian_process_models", ) _process_module_definition( - "strawberry.schema.schema", - "newrelic.hooks.framework_strawberry", - "instrument_strawberry_schema", + "sklearn.gaussian_process.gpc", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_gaussian_process_models", ) _process_module_definition( - "strawberry.schema.schema_converter", - "newrelic.hooks.framework_strawberry", - "instrument_strawberry_schema_converter", + "sklearn.gaussian_process._gpr", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_gaussian_process_models", ) - _process_module_definition("uvicorn.config", "newrelic.hooks.adapter_uvicorn", "instrument_uvicorn_config") + _process_module_definition( + "sklearn.gaussian_process.gpr", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_gaussian_process_models", + ) _process_module_definition( - "hypercorn.asyncio.run", "newrelic.hooks.adapter_hypercorn", "instrument_hypercorn_asyncio_run" + "sklearn.dummy", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_dummy_models", ) + _process_module_definition( - "hypercorn.trio.run", "newrelic.hooks.adapter_hypercorn", "instrument_hypercorn_trio_run" + "sklearn.feature_selection._rfe", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_rfe_models", ) - _process_module_definition("hypercorn.utils", "newrelic.hooks.adapter_hypercorn", "instrument_hypercorn_utils") - _process_module_definition("daphne.server", "newrelic.hooks.adapter_daphne", "instrument_daphne_server") + _process_module_definition( + "sklearn.feature_selection.rfe", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_rfe_models", + ) - _process_module_definition("sanic.app", "newrelic.hooks.framework_sanic", "instrument_sanic_app") - _process_module_definition("sanic.response", "newrelic.hooks.framework_sanic", "instrument_sanic_response") _process_module_definition( - "sanic.touchup.service", "newrelic.hooks.framework_sanic", "instrument_sanic_touchup_service" + "sklearn.feature_selection._variance_threshold", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_models", ) - _process_module_definition("aiohttp.wsgi", "newrelic.hooks.framework_aiohttp", "instrument_aiohttp_wsgi") - _process_module_definition("aiohttp.web", "newrelic.hooks.framework_aiohttp", "instrument_aiohttp_web") _process_module_definition( - "aiohttp.web_reqrep", - "newrelic.hooks.framework_aiohttp", - "instrument_aiohttp_web_response", + "sklearn.feature_selection.variance_threshold", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_models", ) + _process_module_definition( - "aiohttp.web_response", - "newrelic.hooks.framework_aiohttp", - "instrument_aiohttp_web_response", + "sklearn.feature_selection._from_model", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_models", ) + _process_module_definition( - "aiohttp.web_urldispatcher", - "newrelic.hooks.framework_aiohttp", - "instrument_aiohttp_web_urldispatcher", + "sklearn.feature_selection.from_model", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_models", ) + _process_module_definition( - "aiohttp.client", - "newrelic.hooks.framework_aiohttp", - "instrument_aiohttp_client", + "sklearn.feature_selection._sequential", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_feature_selection_models", ) + _process_module_definition( - "aiohttp.client_reqrep", - "newrelic.hooks.framework_aiohttp", - "instrument_aiohttp_client_reqrep", + "sklearn.kernel_ridge", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_kernel_ridge_models", ) + _process_module_definition( - "aiohttp.protocol", - "newrelic.hooks.framework_aiohttp", - "instrument_aiohttp_protocol", + "sklearn.neural_network._multilayer_perceptron", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neural_network_models", ) - _process_module_definition("requests.api", "newrelic.hooks.external_requests", "instrument_requests_api") _process_module_definition( - "requests.sessions", - "newrelic.hooks.external_requests", - "instrument_requests_sessions", + "sklearn.neural_network.multilayer_perceptron", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neural_network_models", ) - _process_module_definition("feedparser", "newrelic.hooks.external_feedparser") + _process_module_definition( + "sklearn.neural_network._rbm", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neural_network_models", + ) - _process_module_definition("xmlrpclib", "newrelic.hooks.external_xmlrpclib") + _process_module_definition( + "sklearn.neural_network.rbm", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neural_network_models", + ) - _process_module_definition("dropbox", "newrelic.hooks.external_dropbox") + _process_module_definition( + "sklearn.calibration", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_calibration_models", + ) - _process_module_definition("facepy.graph_api", "newrelic.hooks.external_facepy") + _process_module_definition( + "sklearn.cluster._affinity_propagation", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", + ) - _process_module_definition("pysolr", "newrelic.hooks.datastore_pysolr", "instrument_pysolr") + _process_module_definition( + "sklearn.cluster.affinity_propagation_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", + ) - _process_module_definition("solr", "newrelic.hooks.datastore_solrpy", "instrument_solrpy") + _process_module_definition( + "sklearn.cluster._agglomerative", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_agglomerative_models", + ) - _process_module_definition("aredis.client", "newrelic.hooks.datastore_aredis", "instrument_aredis_client") + _process_module_definition( + "sklearn.cluster.hierarchical", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_agglomerative_models", + ) _process_module_definition( - "aredis.connection", - "newrelic.hooks.datastore_aredis", - "instrument_aredis_connection", + "sklearn.cluster._birch", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", ) - _process_module_definition("aioredis.client", "newrelic.hooks.datastore_aioredis", "instrument_aioredis_client") + _process_module_definition( + "sklearn.cluster.birch", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", + ) - _process_module_definition("aioredis.commands", "newrelic.hooks.datastore_aioredis", "instrument_aioredis_client") + _process_module_definition( + "sklearn.cluster._bisect_k_means", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_kmeans_models", + ) _process_module_definition( - "aioredis.connection", "newrelic.hooks.datastore_aioredis", "instrument_aioredis_connection" + "sklearn.cluster._dbscan", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", ) _process_module_definition( - "redis.asyncio.client", "newrelic.hooks.datastore_aioredis", "instrument_aioredis_client" + "sklearn.cluster.dbscan_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", ) _process_module_definition( - "redis.asyncio.commands", "newrelic.hooks.datastore_aioredis", "instrument_aioredis_client" + "sklearn.cluster._feature_agglomeration", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", ) _process_module_definition( - "redis.asyncio.connection", "newrelic.hooks.datastore_aioredis", "instrument_aioredis_connection" + "sklearn.cluster._kmeans", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_kmeans_models", ) - # v7 and below _process_module_definition( - "elasticsearch.client", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client", + "sklearn.cluster.k_means_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_kmeans_models", ) - # v8 and above + _process_module_definition( - "elasticsearch._sync.client", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_v8", + "sklearn.cluster._mean_shift", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", ) - # v7 and below _process_module_definition( - "elasticsearch.client.cat", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_cat", + "sklearn.cluster.mean_shift_", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", ) - # v8 and above + _process_module_definition( - "elasticsearch._sync.client.cat", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_cat_v8", + "sklearn.cluster._optics", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_models", ) - # v7 and below _process_module_definition( - "elasticsearch.client.cluster", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_cluster", + "sklearn.cluster._spectral", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_clustering_models", ) - # v8 and above + _process_module_definition( - "elasticsearch._sync.client.cluster", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_cluster_v8", + "sklearn.cluster.spectral", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_clustering_models", ) - # v7 and below _process_module_definition( - "elasticsearch.client.indices", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_indices", + "sklearn.cluster._bicluster", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_clustering_models", ) - # v8 and above + _process_module_definition( - "elasticsearch._sync.client.indices", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_indices_v8", + "sklearn.cluster.bicluster", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_cluster_clustering_models", ) - # v7 and below _process_module_definition( - "elasticsearch.client.nodes", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_nodes", + "sklearn.multiclass", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_multiclass_models", ) - # v8 and above + _process_module_definition( - "elasticsearch._sync.client.nodes", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_nodes_v8", + "sklearn.multioutput", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_multioutput_models", ) - # v7 and below _process_module_definition( - "elasticsearch.client.snapshot", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_snapshot", + "sklearn.naive_bayes", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_naive_bayes_models", ) - # v8 and above + _process_module_definition( - "elasticsearch._sync.client.snapshot", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_snapshot_v8", + "sklearn.model_selection._search", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_model_selection_models", ) - # v7 and below _process_module_definition( - "elasticsearch.client.tasks", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_tasks", + "sklearn.mixture._bayesian_mixture", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_mixture_models", ) - # v8 and above + _process_module_definition( - "elasticsearch._sync.client.tasks", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_tasks_v8", + "sklearn.mixture.bayesian_mixture", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_mixture_models", ) - # v7 and below _process_module_definition( - "elasticsearch.client.ingest", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_ingest", + "sklearn.mixture._gaussian_mixture", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_mixture_models", ) - # v8 and above + _process_module_definition( - "elasticsearch._sync.client.ingest", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_client_ingest_v8", + "sklearn.mixture.gaussian_mixture", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_mixture_models", ) - # v7 and below _process_module_definition( - "elasticsearch.connection.base", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_connection_base", + "sklearn.pipeline", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_pipeline_models", ) - # v8 and above + _process_module_definition( - "elastic_transport._node._base", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elastic_transport__node__base", + "sklearn.semi_supervised._label_propagation", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_semi_supervised_models", ) - # v7 and below _process_module_definition( - "elasticsearch.transport", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elasticsearch_transport", + "sklearn.semi_supervised._self_training", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_semi_supervised_models", ) - # v8 and above + _process_module_definition( - "elastic_transport._transport", - "newrelic.hooks.datastore_elasticsearch", - "instrument_elastic_transport__transport", + "sklearn.semi_supervised.label_propagation", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_semi_supervised_models", ) - _process_module_definition("pika.adapters", "newrelic.hooks.messagebroker_pika", "instrument_pika_adapters") - _process_module_definition("pika.channel", "newrelic.hooks.messagebroker_pika", "instrument_pika_channel") - _process_module_definition("pika.spec", "newrelic.hooks.messagebroker_pika", "instrument_pika_spec") + _process_module_definition( + "sklearn.svm._classes", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_svm_models", + ) _process_module_definition( - "pyelasticsearch.client", - "newrelic.hooks.datastore_pyelasticsearch", - "instrument_pyelasticsearch_client", + "sklearn.svm.classes", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_svm_models", ) _process_module_definition( - "pymongo.connection", - "newrelic.hooks.datastore_pymongo", - "instrument_pymongo_connection", + "sklearn.neighbors._classification", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_KRadius_models", ) + _process_module_definition( - "pymongo.mongo_client", - "newrelic.hooks.datastore_pymongo", - "instrument_pymongo_mongo_client", + "sklearn.neighbors.classification", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_KRadius_models", ) + _process_module_definition( - "pymongo.collection", - "newrelic.hooks.datastore_pymongo", - "instrument_pymongo_collection", + "sklearn.neighbors._graph", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_KRadius_models", ) _process_module_definition( - "redis.connection", - "newrelic.hooks.datastore_redis", - "instrument_redis_connection", + "sklearn.neighbors._kde", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", ) - _process_module_definition("redis.client", "newrelic.hooks.datastore_redis", "instrument_redis_client") _process_module_definition( - "redis.commands.core", "newrelic.hooks.datastore_redis", "instrument_redis_commands_core" + "sklearn.neighbors.kde", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", ) _process_module_definition( - "redis.commands.sentinel", "newrelic.hooks.datastore_redis", "instrument_redis_commands_sentinel" + "sklearn.neighbors._lof", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", ) _process_module_definition( - "redis.commands.json.commands", "newrelic.hooks.datastore_redis", "instrument_redis_commands_json_commands" + "sklearn.neighbors.lof", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", ) _process_module_definition( - "redis.commands.search.commands", "newrelic.hooks.datastore_redis", "instrument_redis_commands_search_commands" + "sklearn.neighbors._nca", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", ) _process_module_definition( - "redis.commands.timeseries.commands", - "newrelic.hooks.datastore_redis", - "instrument_redis_commands_timeseries_commands", + "sklearn.neighbors._nearest_centroid", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", ) _process_module_definition( - "redis.commands.bf.commands", "newrelic.hooks.datastore_redis", "instrument_redis_commands_bf_commands" + "sklearn.neighbors.nearest_centroid", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", ) _process_module_definition( - "redis.commands.graph.commands", "newrelic.hooks.datastore_redis", "instrument_redis_commands_graph_commands" + "sklearn.neighbors._regression", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_KRadius_models", ) - _process_module_definition("motor", "newrelic.hooks.datastore_motor", "patch_motor") + _process_module_definition( + "sklearn.neighbors.regression", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_KRadius_models", + ) _process_module_definition( - "piston.resource", - "newrelic.hooks.component_piston", - "instrument_piston_resource", + "sklearn.neighbors._unsupervised", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", ) - _process_module_definition("piston.doc", "newrelic.hooks.component_piston", "instrument_piston_doc") _process_module_definition( - "tastypie.resources", - "newrelic.hooks.component_tastypie", - "instrument_tastypie_resources", + "sklearn.neighbors.unsupervised", + "newrelic.hooks.mlmodel_sklearn", + "instrument_sklearn_neighbors_models", ) - _process_module_definition("tastypie.api", "newrelic.hooks.component_tastypie", "instrument_tastypie_api") _process_module_definition( "rest_framework.views", @@ -2904,6 +4416,11 @@ def _process_module_builtin_defaults(): "newrelic.hooks.application_celery", "instrument_celery_app_task", ) + _process_module_definition( + "celery.app.trace", + "newrelic.hooks.application_celery", + "instrument_celery_app_trace", + ) _process_module_definition("celery.worker", "newrelic.hooks.application_celery", "instrument_celery_worker") _process_module_definition( "celery.concurrency.processes", @@ -2915,23 +4432,11 @@ def _process_module_builtin_defaults(): "newrelic.hooks.application_celery", "instrument_celery_worker", ) - # _process_module_definition('celery.loaders.base', - # 'newrelic.hooks.application_celery', - # 'instrument_celery_loaders_base') - _process_module_definition( - "celery.execute.trace", - "newrelic.hooks.application_celery", - "instrument_celery_execute_trace", - ) - _process_module_definition( - "celery.task.trace", - "newrelic.hooks.application_celery", - "instrument_celery_execute_trace", - ) + _process_module_definition( - "celery.app.trace", + "celery.app.base", "newrelic.hooks.application_celery", - "instrument_celery_execute_trace", + "instrument_celery_app_base", ) _process_module_definition("billiard.pool", "newrelic.hooks.application_celery", "instrument_billiard_pool") @@ -2952,8 +4457,6 @@ def _process_module_builtin_defaults(): "instrument_flup_server_scgi_base", ) - _process_module_definition("pywapi", "newrelic.hooks.external_pywapi", "instrument_pywapi") - _process_module_definition( "meinheld.server", "newrelic.hooks.adapter_meinheld", @@ -3016,17 +4519,6 @@ def _process_module_builtin_defaults(): _process_module_definition("gevent.monkey", "newrelic.hooks.coroutines_gevent", "instrument_gevent_monkey") - _process_module_definition( - "weberror.errormiddleware", - "newrelic.hooks.middleware_weberror", - "instrument_weberror_errormiddleware", - ) - _process_module_definition( - "weberror.reporter", - "newrelic.hooks.middleware_weberror", - "instrument_weberror_reporter", - ) - _process_module_definition("thrift.transport.TSocket", "newrelic.hooks.external_thrift") _process_module_definition( @@ -3045,6 +4537,12 @@ def _process_module_builtin_defaults(): "instrument_gearman_worker", ) + _process_module_definition( + "aiobotocore.endpoint", + "newrelic.hooks.external_aiobotocore", + "instrument_aiobotocore_endpoint", + ) + _process_module_definition( "botocore.endpoint", "newrelic.hooks.external_botocore", @@ -3056,6 +4554,12 @@ def _process_module_builtin_defaults(): "instrument_botocore_client", ) + _process_module_definition( + "s3transfer.futures", + "newrelic.hooks.external_s3transfer", + "instrument_s3transfer_futures", + ) + _process_module_definition( "tornado.httpserver", "newrelic.hooks.framework_tornado", @@ -3081,13 +4585,21 @@ def _process_module_builtin_defaults(): def _process_module_entry_points(): try: - import pkg_resources + # Preferred after Python 3.10 + if sys.version_info >= (3, 10): + from importlib.metadata import entry_points + # Introduced in Python 3.8 + elif sys.version_info >= (3, 8) and sys.version_info <= (3, 9): + from importlib_metadata import entry_points + # Removed in Python 3.12 + else: + from pkg_resources import iter_entry_points as entry_points except ImportError: return group = "newrelic.hooks" - for entrypoint in pkg_resources.iter_entry_points(group=group): + for entrypoint in entry_points(group=group): target = entrypoint.name if target in _module_import_hook_registry: @@ -3106,6 +4618,11 @@ def _process_module_entry_points(): _instrumentation_done = False +def _reset_instrumentation_done(): + global _instrumentation_done + _instrumentation_done = False + + def _setup_instrumentation(): global _instrumentation_done @@ -3140,13 +4657,21 @@ def _setup_instrumentation(): def _setup_extensions(): try: - import pkg_resources + # Preferred after Python 3.10 + if sys.version_info >= (3, 10): + from importlib.metadata import entry_points + # Introduced in Python 3.8 + elif sys.version_info >= (3, 8) and sys.version_info <= (3, 9): + from importlib_metadata import entry_points + # Removed in Python 3.12 + else: + from pkg_resources import iter_entry_points as entry_points except ImportError: return group = "newrelic.extension" - for entrypoint in pkg_resources.iter_entry_points(group=group): + for entrypoint in entry_points(group=group): __import__(entrypoint.module_name) module = sys.modules[entrypoint.module_name] module.initialize() @@ -3169,6 +4694,23 @@ def _setup_agent_console(): newrelic.core.agent.Agent.run_on_startup(_startup_agent_console) +def _setup_security_module(): + """Initiates k2 security module and adds a + callback to agent startup to propagate NR config + """ + try: + if not _settings.security.agent.enabled: + return + from newrelic_security.api.agent import Agent as SecurityAgent + + # initialize security agent + security_agent = SecurityAgent() + # create a callback to reinitialise the security module + newrelic.core.agent.Agent.run_on_startup(security_agent.refresh_agent) + except Exception as k2error: + _logger.error("K2 Startup failed with error %s", k2error) + + def initialize( config_file=None, environment=None, @@ -3187,6 +4729,8 @@ def initialize( _load_configuration(config_file, environment, ignore_errors, log_file, log_level) + _setup_security_module() + if _settings.monitor_mode or _settings.developer_mode: _settings.enabled = True _setup_instrumentation() diff --git a/newrelic/console.py b/newrelic/console.py index 48cda6e7cc..2e527dd9e4 100644 --- a/newrelic/console.py +++ b/newrelic/console.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import atexit +import builtins import cmd import code +import configparser import functools import glob import inspect @@ -29,64 +29,31 @@ import time import traceback -try: - import ConfigParser -except ImportError: - import configparser as ConfigParser - -try: - import __builtin__ -except ImportError: - import builtins as __builtin__ - - -def _argspec_py2(func): - return inspect.getargspec(func) - - -def _argspec_py3(func): - a = inspect.getfullargspec(func) - return (a.args, a.varargs, a.varkw, a.defaults) - - -if hasattr(inspect, "getfullargspec"): - _argspec = _argspec_py3 -else: - _argspec = _argspec_py2 +from collections import OrderedDict +from inspect import signature -try: - from collections import OrderedDict - from inspect import signature - def doc_signature(func): - sig = signature(func) - sig._parameters = OrderedDict(list(sig._parameters.items())[1:]) - return str(sig) - - -except ImportError: - from inspect import formatargspec - - def doc_signature(func): - args, varargs, keywords, defaults = _argspec(func) - return formatargspec(args[1:], varargs, keywords, defaults) - - -from newrelic.api.object_wrapper import ObjectWrapper -from newrelic.api.transaction import Transaction +from newrelic.common.object_wrapper import ObjectProxy from newrelic.core.agent import agent_instance from newrelic.core.config import flatten_settings, global_settings from newrelic.core.trace_cache import trace_cache + +def doc_signature(func): + sig = signature(func) + sig._parameters = OrderedDict(list(sig._parameters.items())[1:]) + return str(sig) + + _trace_cache = trace_cache() def shell_command(wrapped): - args, varargs, keywords, defaults = _argspec(wrapped) + args = inspect.getfullargspec(wrapped).args parser = optparse.OptionParser() for name in args[1:]: - parser.add_option("--%s" % name, dest=name) + parser.add_option(f"--{name}", dest=name) @functools.wraps(wrapped) def wrapper(self, line): @@ -102,10 +69,10 @@ def wrapper(self, line): return wrapped(self, *args, **kwargs) if wrapper.__name__.startswith("do_"): - prototype = wrapper.__name__[3:] + " " + doc_signature(wrapped) + prototype = f"{wrapper.__name__[3:]} {doc_signature(wrapped)}" if hasattr(wrapper, "__doc__") and wrapper.__doc__ is not None: - wrapper.__doc__ = "\n".join((prototype, wrapper.__doc__.lstrip("\n"))) + wrapper.__doc__ = "\n".join((prototype, wrapper.__doc__.lstrip('\n'))) # noqa: flynt return wrapper @@ -133,12 +100,12 @@ def setquit(): else: eof = "Ctrl-D (i.e. EOF)" - class Quitter(object): + class Quitter(): def __init__(self, name): self.name = name def __repr__(self): - return "Use %s() or %s to exit" % (self.name, eof) + return f"Use {self.name}() or {eof} to exit" def __call__(self, code=None): # If executed with our interactive console, only raise the @@ -157,11 +124,11 @@ def __call__(self, code=None): pass raise SystemExit(code) - __builtin__.quit = Quitter("quit") - __builtin__.exit = Quitter("exit") + builtins.quit = Quitter("quit") + builtins.exit = Quitter("exit") -class OutputWrapper(ObjectWrapper): +class OutputWrapper(ObjectProxy): def flush(self): try: shell = _consoles.active @@ -187,8 +154,8 @@ def writelines(self, data): def intercept_console(): setquit() - sys.stdout = OutputWrapper(sys.stdout, None, None) - sys.stderr = OutputWrapper(sys.stderr, None, None) + sys.stdout = OutputWrapper(sys.stdout) + sys.stderr = OutputWrapper(sys.stderr) class EmbeddedConsole(code.InteractiveConsole): @@ -228,7 +195,7 @@ def do_prompt(self, flag=None): Enable or disable the console prompt.""" if flag == "on": - self.prompt = "(newrelic:%d) " % os.getpid() + self.prompt = f"(newrelic:{os.getpid()}) " elif flag == "off": self.prompt = "" @@ -268,7 +235,7 @@ def do_sys_modules(self): for name, module in sorted(sys.modules.items()): if module is not None: file = getattr(module, "__file__", None) - print("%s - %s" % (name, file), file=self.stdout) + print(f"{name} - {file}", file=self.stdout) @shell_command def do_sys_meta_path(self): @@ -283,7 +250,7 @@ def do_os_environ(self): Displays the set of user environment variables.""" for key, name in os.environ.items(): - print("%s = %r" % (key, name), file=self.stdout) + print(f"{key} = {name!r}", file=self.stdout) @shell_command def do_current_time(self): @@ -327,7 +294,7 @@ def do_dump_config(self, name=None): config = flatten_settings(config) keys = sorted(config.keys()) for key in keys: - print("%s = %r" % (key, config[key]), file=self.stdout) + print(f"{key} = {config[key]!r}", file=self.stdout) @shell_command def do_agent_status(self): @@ -377,13 +344,13 @@ def do_import_hooks(self): result = results[key] if result is None: if key[0] not in sys.modules: - print("%s: PENDING" % (key,), file=self.stdout) + print(f"{key}: PENDING", file=self.stdout) else: - print("%s: IMPORTED" % (key,), file=self.stdout) + print(f"{key}: IMPORTED", file=self.stdout) elif not result: - print("%s: INSTRUMENTED" % (key,), file=self.stdout) + print(f"{key}: INSTRUMENTED", file=self.stdout) else: - print("%s: FAILED" % (key,), file=self.stdout) + print(f"{key}: FAILED", file=self.stdout) for line in result: print(line, end="", file=self.stdout) @@ -445,21 +412,21 @@ def do_threads(self): all = [] for threadId, stack in sys._current_frames().items(): block = [] - block.append("# ThreadID: %s" % threadId) + block.append(f"# ThreadID: {threadId}") thr = threading._active.get(threadId) if thr: - block.append("# Type: %s" % type(thr).__name__) - block.append("# Name: %s" % thr.name) + block.append(f"# Type: {type(thr).__name__}") + block.append(f"# Name: {thr.name}") for filename, lineno, name, line in traceback.extract_stack(stack): - block.append("File: '%s', line %d, in %s" % (filename, lineno, name)) + block.append(f"File: '{filename}', line {int(lineno)}, in {name}") if line: - block.append(" %s" % (line.strip())) + block.append(f" {line.strip()}") all.append("\n".join(block)) print("\n\n".join(all), file=self.stdout) -class ConnectionManager(object): +class ConnectionManager(): def __init__(self, listener_socket): self.__listener_socket = listener_socket self.__console_initialized = False @@ -541,11 +508,11 @@ def __init__(self, config_file, stdin=None, stdout=None, log=None): cmd.Cmd.__init__(self, stdin=stdin, stdout=stdout) self.__config_file = config_file - self.__config_object = ConfigParser.RawConfigParser() + self.__config_object = configparser.RawConfigParser() self.__log_object = log if not self.__config_object.read([config_file]): - raise RuntimeError("Unable to open configuration file %s." % config_file) + raise RuntimeError(f"Unable to open configuration file {config_file}.") listener_socket = self.__config_object.get("newrelic", "console.listener_socket") % {"pid": "*"} @@ -578,7 +545,7 @@ def do_servers(self, line): Display a list of the servers which can be connected to.""" for i in range(len(self.__servers)): - print("%s: %s" % (i + 1, self.__servers[i]), file=self.stdout) + print(f"{i + 1}: {self.__servers[i]}", file=self.stdout) def do_connect(self, line): """connect [index] diff --git a/newrelic/core/_thread_utilization.c b/newrelic/core/_thread_utilization.c index 8688fdb143..c3cc6b5396 100644 --- a/newrelic/core/_thread_utilization.c +++ b/newrelic/core/_thread_utilization.c @@ -16,8 +16,8 @@ /* ------------------------------------------------------------------------- */ +#include #include - #include #ifndef PyVarObject_HEAD_INIT @@ -254,14 +254,10 @@ static PyObject *NRUtilization_enter(NRUtilizationObject *self, PyObject *args) PyObject *func = NULL; dict = PyModule_GetDict(module); -#if PY_MAJOR_VERSION >= 3 func = PyDict_GetItemString(dict, "current_thread"); -#else - func = PyDict_GetItemString(dict, "currentThread"); -#endif if (func) { Py_INCREF(func); - thread = PyEval_CallObject(func, (PyObject *)NULL); + thread = PyObject_Call(func, (PyObject *)NULL, (PyObject *)NULL); if (!thread) PyErr_Clear(); @@ -408,7 +404,6 @@ PyTypeObject NRUtilization_Type = { /* ------------------------------------------------------------------------- */ -#if PY_MAJOR_VERSION >= 3 static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "_thread_utilization", /* m_name */ @@ -420,18 +415,13 @@ static struct PyModuleDef moduledef = { NULL, /* m_clear */ NULL, /* m_free */ }; -#endif static PyObject * moduleinit(void) { PyObject *module; -#if PY_MAJOR_VERSION >= 3 module = PyModule_Create(&moduledef); -#else - module = Py_InitModule3("_thread_utilization", NULL, NULL); -#endif if (module == NULL) return NULL; @@ -446,16 +436,9 @@ moduleinit(void) return module; } -#if PY_MAJOR_VERSION < 3 -PyMODINIT_FUNC init_thread_utilization(void) -{ - moduleinit(); -} -#else PyMODINIT_FUNC PyInit__thread_utilization(void) { return moduleinit(); } -#endif /* ------------------------------------------------------------------------- */ diff --git a/newrelic/core/adaptive_sampler.py b/newrelic/core/adaptive_sampler.py index 5e93e0e8a4..d79aa0eca9 100644 --- a/newrelic/core/adaptive_sampler.py +++ b/newrelic/core/adaptive_sampler.py @@ -17,7 +17,7 @@ import threading -class AdaptiveSampler(object): +class AdaptiveSampler(): def __init__(self, sampling_target, sampling_period): self.adaptive_target = 0.0 self.period = sampling_period diff --git a/newrelic/core/agent.py b/newrelic/core/agent.py index 6ab9571a45..1b81bb9b4b 100644 --- a/newrelic/core/agent.py +++ b/newrelic/core/agent.py @@ -17,8 +17,6 @@ """ -from __future__ import print_function - import atexit import logging import os @@ -32,7 +30,6 @@ import newrelic import newrelic.core.application import newrelic.core.config -import newrelic.packages.six as six from newrelic.common.log_file import initialize_logging from newrelic.core.thread_utilization import thread_utilization_data_source from newrelic.samplers.cpu_usage import cpu_usage_data_source @@ -82,7 +79,7 @@ def check_environment(): ) -class Agent(object): +class Agent(): """Only one instance of the agent should ever exist and that can be obtained using the agent_instance() function. @@ -155,7 +152,7 @@ def agent_singleton(): initialize_logging(settings.log_file, settings.log_level) - _logger.info("New Relic Python Agent (%s)" % newrelic.version) + _logger.info(f"New Relic Python Agent ({newrelic.version})") check_environment() @@ -253,16 +250,16 @@ def uwsgi_atexit_callback(): def dump(self, file): """Dumps details about the agent to the file object.""" - print("Time Created: %s" % (time.asctime(time.localtime(self._creation_time))), file=file) - print("Initialization PID: %s" % (self._process_id), file=file) - print("Default Harvest Count: %d" % (self._default_harvest_count), file=file) - print("Flexible Harvest Count: %d" % (self._flexible_harvest_count), file=file) - print("Last Default Harvest: %s" % (time.asctime(time.localtime(self._last_default_harvest))), file=file) - print("Last Flexible Harvest: %s" % (time.asctime(time.localtime(self._last_flexible_harvest))), file=file) - print("Default Harvest Duration: %.2f" % (self._default_harvest_duration), file=file) - print("Flexible Harvest Duration: %.2f" % (self._flexible_harvest_duration), file=file) - print("Agent Shutdown: %s" % (self._harvest_shutdown.isSet()), file=file) - print("Applications: %r" % (sorted(self._applications.keys())), file=file) + print(f"Time Created: {time.asctime(time.localtime(self._creation_time))}", file=file) + print(f"Initialization PID: {self._process_id}", file=file) + print(f"Default Harvest Count: {self._default_harvest_count}", file=file) + print(f"Flexible Harvest Count: {self._flexible_harvest_count}", file=file) + print(f"Last Default Harvest: {time.asctime(time.localtime(self._last_default_harvest))}", file=file) + print(f"Last Flexible Harvest: {time.asctime(time.localtime(self._last_flexible_harvest))}", file=file) + print(f"Default Harvest Duration: {self._default_harvest_duration:.2f}", file=file) + print(f"Flexible Harvest Duration: {self._flexible_harvest_duration:.2f}", file=file) + print(f"Agent Shutdown: {self._harvest_shutdown.isSet()}", file=file) + print(f"Applications: {sorted(self._applications.keys())!r}", file=file) def global_settings(self): """Returns the global default settings object. If access is @@ -339,7 +336,6 @@ def activate_application(self, app_name, linked_applications=None, timeout=None, with self._lock: application = self._applications.get(app_name, None) if not application: - process_id = os.getpid() if process_id != self._process_id: @@ -437,7 +433,7 @@ def register_data_source(self, source, application=None, name=None, settings=Non if application is None: # Bind to any applications that already exist. - for application in list(six.itervalues(self._applications)): + for application in list(self._applications.values()): application.register_data_source(source, name, settings, **properties) else: @@ -449,7 +445,6 @@ def register_data_source(self, source, application=None, name=None, settings=Non instance.register_data_source(source, name, settings, **properties) def remove_thread_utilization(self): - _logger.debug("Removing thread utilization data source from all applications") source_name = thread_utilization_data_source.__name__ @@ -524,6 +519,33 @@ def record_custom_metrics(self, app_name, metrics): application.record_custom_metrics(metrics) + def record_dimensional_metric(self, app_name, name, value, tags=None): + """Records a basic metric for the named application. If there has + been no prior request to activate the application, the metric is + discarded. + + """ + + application = self._applications.get(app_name, None) + if application is None or not application.active: + return + + application.record_dimensional_metric(name, value, tags) + + def record_dimensional_metrics(self, app_name, metrics): + """Records the metrics for the named application. If there has + been no prior request to activate the application, the metric is + discarded. The metrics should be an iterable yielding tuples + consisting of the name and value. + + """ + + application = self._applications.get(app_name, None) + if application is None or not application.active: + return + + application.record_dimensional_metrics(metrics) + def record_custom_event(self, app_name, event_type, params): application = self._applications.get(app_name, None) if application is None or not application.active: @@ -531,12 +553,19 @@ def record_custom_event(self, app_name, event_type, params): application.record_custom_event(event_type, params) - def record_log_event(self, app_name, message, level=None, timestamp=None, priority=None): + def record_ml_event(self, app_name, event_type, params): + application = self._applications.get(app_name, None) + if application is None or not application.active: + return + + application.record_ml_event(event_type, params) + + def record_log_event(self, app_name, message, level=None, timestamp=None, attributes=None, priority=None): application = self._applications.get(app_name, None) if application is None or not application.active: return - application.record_log_event(message, level, timestamp, priority=priority) + application.record_log_event(message, level, timestamp, attributes=attributes, priority=priority) def record_transaction(self, app_name, data): """Processes the raw transaction data, generating and recording @@ -587,11 +616,11 @@ def _harvest_flexible(self, shutdown=False): self._flexible_harvest_count += 1 self._last_flexible_harvest = time.time() - for application in list(six.itervalues(self._applications)): + for application in list(self._applications.values()): try: application.harvest(shutdown=False, flexible=True) except Exception: - _logger.exception("Failed to harvest data for %s." % application.name) + _logger.exception(f"Failed to harvest data for {application.name}.") self._flexible_harvest_duration = time.time() - self._last_flexible_harvest @@ -611,11 +640,11 @@ def _harvest_default(self, shutdown=False): self._default_harvest_count += 1 self._last_default_harvest = time.time() - for application in list(six.itervalues(self._applications)): + for application in list(self._applications.values()): try: application.harvest(shutdown, flexible=False) except Exception: - _logger.exception("Failed to harvest data for %s." % application.name) + _logger.exception(f"Failed to harvest data for {application.name}.") self._default_harvest_duration = time.time() - self._last_default_harvest diff --git a/newrelic/core/agent_protocol.py b/newrelic/core/agent_protocol.py index ba277d4de1..ddbfa051d7 100644 --- a/newrelic/core/agent_protocol.py +++ b/newrelic/core/agent_protocol.py @@ -27,6 +27,7 @@ AWSUtilization, AzureUtilization, DockerUtilization, + ECSUtilization, GCPUtilization, KubernetesUtilization, PCFUtilization, @@ -38,6 +39,7 @@ global_settings_dump, ) from newrelic.core.internal_metrics import internal_count_metric +from newrelic.core.otlp_utils import OTLP_CONTENT_TYPE, otlp_encode from newrelic.network.exceptions import ( DiscardDataForRequest, ForceAgentDisconnect, @@ -49,7 +51,7 @@ _logger = logging.getLogger(__name__) -class AgentProtocol(object): +class AgentProtocol(): VERSION = 17 STATUS_CODE_RESPONSE = { @@ -143,7 +145,10 @@ class AgentProtocol(object): "transaction_tracer.record_sql", "strip_exception_messages.enabled", "custom_insights_events.enabled", + "ml_insights_events.enabled", "application_logging.forwarding.enabled", + "machine_learning.inference_events_value.enabled", + "ai_monitoring.enabled", ) LOGGER_FUNC_MAPPING = { @@ -215,11 +220,16 @@ def __exit__(self, exc, value, tb): def close_connection(self): self.client.close_connection() - def send(self, method, payload=()): + def send( + self, + method, + payload=(), + path="/agent_listener/invoke_raw_method", + ): params, headers, payload = self._to_http(method, payload) try: - response = self.client.send_request(params=params, headers=headers, payload=payload) + response = self.client.send_request(path=path, params=params, headers=headers, payload=payload) except NetworkInterfaceException: # All HTTP errors are currently retried raise RetryDataForRequest @@ -229,7 +239,7 @@ def send(self, method, payload=()): if not 200 <= status < 300: if status == 413: internal_count_metric( - "Supportability/Python/Collector/MaxPayloadSizeLimit/%s" % method, + f"Supportability/Python/Collector/MaxPayloadSizeLimit/{method}", 1, ) level, message = self.LOG_MESSAGES.get(status, self.LOG_MESSAGES["default"]) @@ -251,7 +261,10 @@ def send(self, method, payload=()): exception = self.STATUS_CODE_RESPONSE.get(status, DiscardDataForRequest) raise exception if status == 200: - return json_decode(data.decode("utf-8"))["return_value"] + return self.decode_response(data) + + def decode_response(self, response): + return json_decode(response.decode("utf-8"))["return_value"] def _to_http(self, method, payload=()): params = dict(self._params) @@ -275,6 +288,7 @@ def _connect_payload(app_name, linked_applications, environment, settings): connect_settings = {} connect_settings["browser_monitoring.loader"] = settings["browser_monitoring.loader"] connect_settings["browser_monitoring.debug"] = settings["browser_monitoring.debug"] + connect_settings["ai_monitoring.enabled"] = settings["ai_monitoring.enabled"] security_settings = {} security_settings["capture_params"] = settings["capture_params"] @@ -308,8 +322,15 @@ def _connect_payload(app_name, linked_applications, environment, settings): utilization_settings["config"] = utilization_conf vendors = [] + ecs_id = None + utilization_vendor_settings = {} + if settings["utilization.detect_aws"]: vendors.append(AWSUtilization) + ecs_id = ECSUtilization.detect() + if ecs_id: + utilization_vendor_settings["ecs"] = ecs_id + if settings["utilization.detect_pcf"]: vendors.append(PCFUtilization) if settings["utilization.detect_gcp"]: @@ -317,7 +338,6 @@ def _connect_payload(app_name, linked_applications, environment, settings): if settings["utilization.detect_azure"]: vendors.append(AzureUtilization) - utilization_vendor_settings = {} for vendor in vendors: metadata = vendor.detect() if metadata: @@ -325,9 +345,10 @@ def _connect_payload(app_name, linked_applications, environment, settings): break if settings["utilization.detect_docker"]: - docker = DockerUtilization.detect() - if docker: - utilization_vendor_settings["docker"] = docker + if not ecs_id: + docker = DockerUtilization.detect() + if docker: + utilization_vendor_settings["docker"] = docker if settings["utilization.detect_kubernetes"]: kubernetes = KubernetesUtilization.detect() @@ -478,6 +499,7 @@ def __init__(self, settings, host=None, client_cls=ServerlessModeClient): "protocol_version": self.VERSION, "execution_environment": os.environ.get("AWS_EXECUTION_ENV", None), "agent_version": version, + "agent_language": "python", } def finalize(self): @@ -514,3 +536,77 @@ def connect( # can be modified later settings.aws_lambda_metadata = aws_lambda_metadata return cls(settings, client_cls=client_cls) + + +class OtlpProtocol(AgentProtocol): + def __init__(self, settings, host=None, client_cls=ApplicationModeClient): + if settings.audit_log_file: + audit_log_fp = open(settings.audit_log_file, "a") + else: + audit_log_fp = None + + self.client = client_cls( + host=host or settings.otlp_host, + port=settings.otlp_port or 4318, + proxy_scheme=settings.proxy_scheme, + proxy_host=settings.proxy_host, + proxy_port=settings.proxy_port, + proxy_user=settings.proxy_user, + proxy_pass=settings.proxy_pass, + timeout=settings.agent_limits.data_collector_timeout, + ca_bundle_path=settings.ca_bundle_path, + disable_certificate_validation=settings.debug.disable_certificate_validation, + compression_threshold=settings.agent_limits.data_compression_threshold, + compression_level=settings.agent_limits.data_compression_level, + compression_method=settings.compressed_content_encoding, + max_payload_size_in_bytes=1000000, + audit_log_fp=audit_log_fp, + default_content_encoding_header=None, + ) + + self._params = {} + self._headers = { + "api-key": settings.license_key, + } + + # In Python 2, the JSON is loaded with unicode keys and values; + # however, the header name must be a non-unicode value when given to + # the HTTP library. This code converts the header name from unicode to + # non-unicode. + if settings.request_headers_map: + for k, v in settings.request_headers_map.items(): + if not isinstance(k, str): + k = k.encode("utf-8") + self._headers[k] = v + + # Content-Type should be protobuf, but falls back to JSON if protobuf is not installed. + self._headers["Content-Type"] = OTLP_CONTENT_TYPE + self._run_token = settings.agent_run_id + + # Logging + self._proxy_host = settings.proxy_host + self._proxy_port = settings.proxy_port + self._proxy_user = settings.proxy_user + + # Do not access configuration anywhere inside the class + self.configuration = settings + + @classmethod + def connect( + cls, + app_name, + linked_applications, + environment, + settings, + client_cls=ApplicationModeClient, + ): + with cls(settings, client_cls=client_cls) as protocol: + pass + + return protocol + + def _to_http(self, method, payload=()): + return {}, self._headers, otlp_encode(payload) + + def decode_response(self, response): + return response.decode("utf-8") diff --git a/newrelic/core/agent_streaming.py b/newrelic/core/agent_streaming.py index b581f5d17a..dea53f3644 100644 --- a/newrelic/core/agent_streaming.py +++ b/newrelic/core/agent_streaming.py @@ -25,7 +25,7 @@ _logger = logging.getLogger(__name__) -class StreamingRpc(object): +class StreamingRpc(): """Streaming Remote Procedure Call This class keeps a stream_stream RPC alive, retrying after a timeout when @@ -128,7 +128,7 @@ def process_responses(self): details = response_iterator.details() self.record_metric( - "Supportability/InfiniteTracing/Span/gRPC/%s" % code.name, + f"Supportability/InfiniteTracing/Span/gRPC/{code.name}", {"count": 1}, ) diff --git a/newrelic/core/application.py b/newrelic/core/application.py index 7be2174280..4a5632f807 100644 --- a/newrelic/core/application.py +++ b/newrelic/core/application.py @@ -16,8 +16,6 @@ """ -from __future__ import print_function - import logging import os import sys @@ -50,13 +48,12 @@ NetworkInterfaceException, RetryDataForRequest, ) -from newrelic.packages import six from newrelic.samplers.data_sampler import DataSampler _logger = logging.getLogger(__name__) -class Application(object): +class Application(): """Class which maintains recorded data for a single application.""" @@ -159,31 +156,31 @@ def compute_sampled(self): def dump(self, file): """Dumps details about the application to the file object.""" - print("Time Created: %s" % (time.asctime(time.localtime(self._creation_time))), file=file) - print("Linked Applications: %r" % (self._linked_applications), file=file) - print("Registration PID: %s" % (self._process_id), file=file) - print("Harvest Count: %d" % (self._harvest_count), file=file) - print("Agent Restart: %d" % (self._agent_restart), file=file) - print("Forced Shutdown: %s" % (self._agent_shutdown), file=file) + print(f"Time Created: {time.asctime(time.localtime(self._creation_time))}", file=file) + print(f"Linked Applications: {self._linked_applications!r}", file=file) + print(f"Registration PID: {self._process_id}", file=file) + print(f"Harvest Count: {self._harvest_count}", file=file) + print(f"Agent Restart: {self._agent_restart}", file=file) + print(f"Forced Shutdown: {self._agent_shutdown}", file=file) active_session = self._active_session if active_session: try: - print("Collector URL: %s" % (active_session._protocol.client._host), file=file) + print(f"Collector URL: {active_session._protocol.client._host}", file=file) except AttributeError: pass - print("Agent Run ID: %s" % (active_session.agent_run_id), file=file) - print("URL Normalization Rules: %r" % (self._rules_engine["url"].rules), file=file) - print("Metric Normalization Rules: %r" % (self._rules_engine["metric"].rules), file=file) - print("Transaction Normalization Rules: %r" % (self._rules_engine["transaction"].rules), file=file) - print("Transaction Segment Allowlist Rules: %r" % (self._rules_engine["segment"].rules), file=file) - print("Harvest Period Start: %s" % (time.asctime(time.localtime(self._period_start))), file=file) - print("Transaction Count: %d" % (self._transaction_count), file=file) - print("Last Transaction: %s" % (time.asctime(time.localtime(self._last_transaction))), file=file) - print("Global Events Count: %d" % (self._global_events_account), file=file) - print("Harvest Metrics Count: %d" % (self._stats_engine.metrics_count()), file=file) - print("Harvest Discard Count: %d" % (self._discard_count), file=file) + print(f"Agent Run ID: {active_session.agent_run_id}", file=file) + print(f"URL Normalization Rules: {self._rules_engine['url'].rules!r}", file=file) + print(f"Metric Normalization Rules: {self._rules_engine['metric'].rules!r}", file=file) + print(f"Transaction Normalization Rules: {self._rules_engine['transaction'].rules!r}", file=file) + print(f"Transaction Segment Allowlist Rules: {self._rules_engine['segment'].rules!r}", file=file) + print(f"Harvest Period Start: {time.asctime(time.localtime(self._period_start))}", file=file) + print(f"Transaction Count: {self._transaction_count}", file=file) + print(f"Last Transaction: {time.asctime(time.localtime(self._last_transaction))}", file=file) + print(f"Global Events Count: {self._global_events_account}", file=file) + print(f"Harvest Metrics Count: {self._stats_engine.metrics_count()}", file=file) + print(f"Harvest Discard Count: {self._discard_count}", file=file) def activate_session(self, activate_agent=None, timeout=0.0): """Creates a background thread to initiate registration of the @@ -227,7 +224,7 @@ def activate_session(self, activate_agent=None, timeout=0.0): self._detect_deadlock = True thread = threading.Thread( - target=self.connect_to_data_collector, name="NR-Activate-Session/%s" % self.name, args=(activate_agent,) + target=self.connect_to_data_collector, name=f"NR-Activate-Session/{self.name}", args=(activate_agent,) ) thread.daemon = True thread.start() @@ -319,14 +316,7 @@ def connect_to_data_collector(self, activate_agent): # code run from this thread performs a deferred module import. if self._detect_deadlock: - if six.PY2: - import imp - - imp.acquire_lock() - self._deadlock_event.set() - imp.release_lock() - else: - self._deadlock_event.set() + self._deadlock_event.set() # Register the application with the data collector. Any errors # that occur will be dealt with by create_session(). The result @@ -510,6 +500,9 @@ def connect_to_data_collector(self, activate_agent): with self._stats_custom_lock: self._stats_custom_engine.reset_stats(configuration) + with self._stats_lock: + self._stats_engine.reset_stats(configuration) + # Record an initial start time for the reporting period and # clear record of last transaction processed. @@ -549,20 +542,24 @@ def connect_to_data_collector(self, activate_agent): application_logging_local_decorating = ( configuration.application_logging.enabled and configuration.application_logging.local_decorating.enabled ) + ai_monitoring_streaming = configuration.ai_monitoring.streaming.enabled internal_metric( - "Supportability/Logging/Forwarding/Python/%s" - % ("enabled" if application_logging_forwarding else "disabled"), + f"Supportability/Logging/Forwarding/Python/{'enabled' if application_logging_forwarding else 'disabled'}", 1, ) internal_metric( - "Supportability/Logging/LocalDecorating/Python/%s" - % ("enabled" if application_logging_local_decorating else "disabled"), + f"Supportability/Logging/LocalDecorating/Python/{'enabled' if application_logging_local_decorating else 'disabled'}", 1, ) internal_metric( - "Supportability/Logging/Metrics/Python/%s" % ("enabled" if application_logging_metrics else "disabled"), + f"Supportability/Logging/Metrics/Python/{'enabled' if application_logging_metrics else 'disabled'}", 1, ) + if not ai_monitoring_streaming: + internal_metric( + "Supportability/Python/ML/Streaming/Disabled", + 1, + ) # Infinite tracing feature toggle metrics infinite_tracing = configuration.infinite_tracing.enabled # Property that checks trace observer host @@ -570,13 +567,11 @@ def connect_to_data_collector(self, activate_agent): infinite_tracing_batching = configuration.infinite_tracing.batching infinite_tracing_compression = configuration.infinite_tracing.compression internal_metric( - "Supportability/InfiniteTracing/gRPC/Batching/%s" - % ("enabled" if infinite_tracing_batching else "disabled"), + f"Supportability/InfiniteTracing/gRPC/Batching/{'enabled' if infinite_tracing_batching else 'disabled'}", 1, ) internal_metric( - "Supportability/InfiniteTracing/gRPC/Compression/%s" - % ("enabled" if infinite_tracing_compression else "disabled"), + f"Supportability/InfiniteTracing/gRPC/Compression/{'enabled' if infinite_tracing_compression else 'disabled'}", 1, ) @@ -860,6 +855,50 @@ def record_custom_metrics(self, metrics): self._global_events_account += 1 self._stats_custom_engine.record_custom_metric(name, value) + def record_dimensional_metric(self, name, value, tags=None): + """Record a dimensional metric against the application independent + of a specific transaction. + + NOTE that this will require locking of the stats engine for + dimensional metrics and so under heavy use will have performance + issues. It is better to record the dimensional metric against an + active transaction as they will then be aggregated at the end of + the transaction when all other metrics are aggregated and so no + additional locking will be required. + + """ + + if not self._active_session: + return + + with self._stats_lock: + self._global_events_account += 1 + self._stats_engine.record_dimensional_metric(name, value, tags) + + def record_dimensional_metrics(self, metrics): + """Record a set of dimensional metrics against the application + independent of a specific transaction. + + NOTE that this will require locking of the stats engine for + dimensional metrics and so under heavy use will have performance + issues. It is better to record the dimensional metric against an + active transaction as they will then be aggregated at the end of + the transaction when all other metrics are aggregated and so no + additional locking will be required. + + """ + + if not self._active_session: + return + + with self._stats_lock: + for metric in metrics: + name, value = metric[:2] + tags = metric[2] if len(metric) >= 3 else None + + self._global_events_account += 1 + self._stats_engine.record_dimensional_metric(name, value, tags) + def record_custom_event(self, event_type, params): if not self._active_session: return @@ -869,22 +908,39 @@ def record_custom_event(self, event_type, params): if settings is None or not settings.custom_insights_events.enabled: return - event = create_custom_event(event_type, params) + event = create_custom_event(event_type, params, settings=settings) if event: with self._stats_custom_lock: self._global_events_account += 1 self._stats_engine.record_custom_event(event) - def record_log_event(self, message, level=None, timestamp=None, priority=None): + def record_ml_event(self, event_type, params): if not self._active_session: return - if message: + settings = self._stats_engine.settings + + if settings is None or not settings.ml_insights_events.enabled: + return + + event = create_custom_event(event_type, params, settings=settings, is_ml_event=True) + + if event: with self._stats_custom_lock: - event = self._stats_engine.record_log_event(message, level, timestamp, priority=priority) - if event: - self._global_events_account += 1 + self._global_events_account += 1 + self._stats_engine.record_ml_event(event) + + def record_log_event(self, message, level=None, timestamp=None, attributes=None, priority=None): + if not self._active_session: + return + + with self._stats_custom_lock: + event = self._stats_engine.record_log_event( + message, level, timestamp, attributes=attributes, priority=priority + ) + if event: + self._global_events_account += 1 def record_transaction(self, data): """Record a single transaction against this application.""" @@ -1097,7 +1153,7 @@ def harvest(self, shutdown=False, flexible=False): call_metric = "flexible" if flexible else "default" with InternalTraceContext(internal_metrics): - with InternalTrace("Supportability/Python/Harvest/Calls/" + call_metric): + with InternalTrace(f"Supportability/Python/Harvest/Calls/{call_metric}"): self._harvest_count += 1 start = time.time() @@ -1193,7 +1249,7 @@ def harvest(self, shutdown=False, flexible=False): if self._uninstrumented: for uninstrumented in self._uninstrumented: internal_count_metric("Supportability/Python/Uninstrumented", 1) - internal_count_metric("Supportability/Uninstrumented/%s" % uninstrumented, 1) + internal_count_metric(f"Supportability/Uninstrumented/{uninstrumented}", 1) # Create our time stamp as to when this reporting period # ends and start reporting the data. @@ -1335,6 +1391,26 @@ def harvest(self, shutdown=False, flexible=False): stats.reset_custom_events() + # Send machine learning events + + if configuration.ml_insights_events.enabled: + ml_events = stats.ml_events + + if ml_events: + if ml_events.num_samples > 0: + ml_event_samples = list(ml_events) + + _logger.debug("Sending machine learning event data for harvest of %r.", self._app_name) + + self._active_session.send_ml_events(ml_events.sampling_info, ml_event_samples) + ml_event_samples = None + + # As per spec + internal_count_metric("Supportability/Events/Customer/Seen", ml_events.num_seen) + internal_count_metric("Supportability/Events/Customer/Sent", ml_events.num_samples) + + stats.reset_ml_events() + # Send log events if ( @@ -1416,11 +1492,16 @@ def harvest(self, shutdown=False, flexible=False): _logger.debug("Normalizing metrics for harvest of %r.", self._app_name) metric_data = stats.metric_data(metric_normalizer) + dimensional_metric_data = stats.dimensional_metric_data(metric_normalizer) _logger.debug("Sending metric data for harvest of %r.", self._app_name) # Send metrics self._active_session.send_metric_data(self._period_start, period_end, metric_data) + if dimensional_metric_data: + self._active_session.send_dimensional_metric_data( + self._period_start, period_end, dimensional_metric_data + ) _logger.debug("Done sending data for harvest of %r.", self._app_name) @@ -1506,7 +1587,7 @@ def harvest(self, shutdown=False, flexible=False): exc_type = sys.exc_info()[0] - internal_metric("Supportability/Python/Harvest/Exception/%s" % callable_name(exc_type), 1) + internal_metric(f"Supportability/Python/Harvest/Exception/{callable_name(exc_type)}", 1) if self._period_start != period_end: self._stats_engine.rollback(stats) @@ -1519,7 +1600,7 @@ def harvest(self, shutdown=False, flexible=False): exc_type = sys.exc_info()[0] - internal_metric("Supportability/Python/Harvest/Exception/%s" % callable_name(exc_type), 1) + internal_metric(f"Supportability/Python/Harvest/Exception/{callable_name(exc_type)}", 1) self._discard_count += 1 @@ -1529,7 +1610,7 @@ def harvest(self, shutdown=False, flexible=False): exc_type = sys.exc_info()[0] - internal_metric("Supportability/Python/Harvest/Exception/%s" % callable_name(exc_type), 1) + internal_metric(f"Supportability/Python/Harvest/Exception/{callable_name(exc_type)}", 1) _logger.exception( "Unexpected exception when attempting " @@ -1649,7 +1730,7 @@ def process_agent_commands(self): # we don't know about a specific agent command we just # ignore it. - func_name = "cmd_%s" % cmd_name + func_name = f"cmd_{cmd_name}" cmd_handler = getattr(self, func_name, None) diff --git a/newrelic/core/attribute.py b/newrelic/core/attribute.py index 372711369c..16dacb18a0 100644 --- a/newrelic/core/attribute.py +++ b/newrelic/core/attribute.py @@ -18,12 +18,12 @@ from newrelic.core.attribute_filter import ( DST_ALL, DST_ERROR_COLLECTOR, + DST_LOG_EVENT_CONTEXT_DATA, DST_SPAN_EVENTS, DST_TRANSACTION_EVENTS, DST_TRANSACTION_SEGMENTS, DST_TRANSACTION_TRACER, ) -from newrelic.packages import six _logger = logging.getLogger(__name__) @@ -47,6 +47,8 @@ "aws.lambda.eventSource.arn", "aws.operation", "aws.requestId", + "cloud.account.id", + "cloud.region", "code.filepath", "code.function", "code.lineno", @@ -58,8 +60,8 @@ "enduser.id", "error.class", "error.expected", - "error.message", "error.group.name", + "error.message", "graphql.field.name", "graphql.field.parentType", "graphql.field.path", @@ -70,8 +72,11 @@ "host.displayName", "http.statusCode", "http.url", + "llm", "message.queueName", "message.routingKey", + "messaging.destination.name", + "messaging.system", "peer.address", "peer.hostname", "request.headers.accept", @@ -84,11 +89,14 @@ "response.headers.contentLength", "response.headers.contentType", "response.status", + "server.address", ) ) MAX_NUM_USER_ATTRIBUTES = 128 MAX_ATTRIBUTE_LENGTH = 255 +MAX_NUM_ML_USER_ATTRIBUTES = 64 +MAX_ML_ATTRIBUTE_LENGTH = 4095 MAX_64_BIT_INT = 2**63 - 1 MAX_LOG_MESSAGE_LENGTH = 32768 @@ -109,9 +117,13 @@ class CastingFailureException(Exception): pass +class NullValueException(ValueError): + pass + + class Attribute(_Attribute): def __repr__(self): - return "Attribute(name=%r, value=%r, destinations=%r)" % (self.name, self.value, bin(self.destinations)) + return f"Attribute(name={self.name!r}, value={self.value!r}, destinations={bin(self.destinations)!r})" def create_attributes(attr_dict, destinations, attribute_filter): @@ -125,6 +137,13 @@ def create_attributes(attr_dict, destinations, attribute_filter): def create_agent_attributes(attr_dict, attribute_filter): + """ + Returns a dictionary of Attribute objects with appropriate destinations. + + If the attribute's key is in the known list of event attributes, it is assigned + to _DESTINATIONS_WITH_EVENTS, otherwise it is assigned to _DESTINATIONS. + Note attributes with a value of None are filtered out. + """ attributes = [] for k, v in attr_dict.items(): @@ -142,12 +161,15 @@ def create_agent_attributes(attr_dict, attribute_filter): def resolve_user_attributes(attr_dict, attribute_filter, target_destination, attr_class=dict): + """ + Returns an attr_class of key value attributes filtered to the target_destination. + + process_user_attribute MUST be called before this function to filter out invalid + attributes. + """ u_attrs = attr_class() for attr_name, attr_value in attr_dict.items(): - if attr_value is None: - continue - dest = attribute_filter.apply(attr_name, DST_ALL) if dest & target_destination: @@ -174,20 +196,39 @@ def resolve_agent_attributes(attr_dict, attribute_filter, target_destination, at return a_attrs -def create_user_attributes(attr_dict, attribute_filter): - destinations = DST_ALL - return create_attributes(attr_dict, destinations, attribute_filter) +def resolve_logging_context_attributes(attr_dict, attribute_filter, attr_prefix, attr_class=dict): + """ + Helper function for processing logging context attributes that require a prefix. Correctly filters attribute names + before applying the required prefix, and then applies the process_user_attribute after the prefix is applied to + correctly check length requirements. + """ + c_attrs = attr_class() + + for attr_name, attr_value in attr_dict.items(): + dest = attribute_filter.apply(attr_name, DST_LOG_EVENT_CONTEXT_DATA) + + if dest & DST_LOG_EVENT_CONTEXT_DATA: + try: + attr_name, attr_value = process_user_attribute(attr_prefix + attr_name, attr_value) + if attr_name: + c_attrs[attr_name] = attr_value + except Exception: + _logger.debug( + "Log event context attribute failed to validate for unknown reason. Dropping context attribute: %s. Check traceback for clues.", + attr_name, + exc_info=True, + ) + + return c_attrs def truncate(text, maxsize=MAX_ATTRIBUTE_LENGTH, encoding="utf-8", ending=None): - # Truncate text so that its byte representation # is no longer than maxsize bytes. - # If text is unicode (Python 2 or 3), return unicode. - # If text is a Python 2 string, return str. + # Handle unicode or bytes strings and return the same type as the input. - if isinstance(text, six.text_type): + if isinstance(text, str): truncated = _truncate_unicode(text, maxsize, encoding) else: truncated = _truncate_bytes(text, maxsize) @@ -215,17 +256,16 @@ def check_name_length(name, max_length=MAX_ATTRIBUTE_LENGTH, encoding="utf-8"): def check_name_is_string(name): - if not isinstance(name, (six.text_type, six.binary_type)): + if not isinstance(name, (str, bytes)): raise NameIsNotStringException() def check_max_int(value, max_int=MAX_64_BIT_INT): - if isinstance(value, six.integer_types) and value > max_int: + if isinstance(value, int) and value > max_int: raise IntTooLargeException() def process_user_attribute(name, value, max_length=MAX_ATTRIBUTE_LENGTH, ending=None): - # Perform all necessary checks on a potential attribute. # # Returns: @@ -245,32 +285,40 @@ def process_user_attribute(name, value, max_length=MAX_ATTRIBUTE_LENGTH, ending= value = sanitize(value) except NameIsNotStringException: - _logger.debug("Attribute name must be a string. Dropping " "attribute: %r=%r", name, value) + _logger.debug("Attribute name must be a string. Dropping attribute: %r=%r", name, value) return FAILED_RESULT except NameTooLongException: - _logger.debug("Attribute name exceeds maximum length. Dropping " "attribute: %r=%r", name, value) + _logger.debug("Attribute name exceeds maximum length. Dropping attribute: %r=%r", name, value) return FAILED_RESULT except IntTooLargeException: - _logger.debug("Attribute value exceeds maximum integer value. " "Dropping attribute: %r=%r", name, value) + _logger.debug("Attribute value exceeds maximum integer value. Dropping attribute: %r=%r", name, value) return FAILED_RESULT except CastingFailureException: - _logger.debug("Attribute value cannot be cast to a string. " "Dropping attribute: %r=%r", name, value) + _logger.debug("Attribute value cannot be cast to a string. Dropping attribute: %r=%r", name, value) return FAILED_RESULT - else: + except NullValueException: + _logger.debug( + "Attribute value is None. There is no difference between omitting the key " + "and sending None. Dropping attribute: %r=%r", + name, + value, + ) + return FAILED_RESULT + else: # Check length after casting - valid_types_text = (six.text_type, six.binary_type) + valid_types_text = (str, bytes) if isinstance(value, valid_types_text): trunc_value = truncate(value, maxsize=max_length, ending=ending) if value != trunc_value: _logger.debug( - "Attribute value exceeds maximum length " "(%r bytes). Truncating value: %r=%r.", + "Attribute value exceeds maximum length (%r bytes). Truncating value: %r=%r.", max_length, name, trunc_value, @@ -282,15 +330,40 @@ def process_user_attribute(name, value, max_length=MAX_ATTRIBUTE_LENGTH, ending= def sanitize(value): - - # Return value unchanged, if it's a valid type that is supported by - # Insights. Otherwise, convert value to a string. - # - # Raise CastingFailureException, if str(value) somehow fails. - - valid_value_types = (six.text_type, six.binary_type, bool, float, six.integer_types) - - if not isinstance(value, valid_value_types): + """ + Return value unchanged, if it's a valid type that is supported by + Insights. Otherwise, convert value to a string. + + Raise CastingFailureException, if str(value) somehow fails. + Raise NullValueException, if value is None (null values SHOULD NOT be reported). + """ + + valid_value_types = (str, bytes, bool, float, int) + # According to the agent spec, agents should not report None attribute values. + # There is no difference between omitting the key and sending a None, so we can + # reduce the payload size by not sending None values. + if value is None: + raise NullValueException( + "Attribute value is of type: None. Omitting value since there is " + "no difference between omitting the key and sending None." + ) + + # When working with numpy, note that numpy has its own `int`s, `str`s, + # et cetera. `numpy.str_` and `numpy.float_` inherit from Python's native + # `str` and `float`, respectively. However, some types, such as `numpy.int_` + # and `numpy.bool_`, do not inherit from `int` and `bool` (respectively). + # In those cases, the valid_value_types check fails and it will try to + # convert these to string, which is not the desired behavior. Checking for + # `type` in lieu of `isinstance` has the potential to impact performance. + + # numpy values have an attribute "item" that returns the closest + # equivalent Python native type. Ex: numpy.int64 -> int + # This is important to utilize in cases like int and bool where + # numpy does not inherit from those classes. This logic is + # determining whether or not the value is a valid_value_type (or + # inherited from one of those types) AND whether it is a numpy + # type (by determining if it has the attribute "item"). + if not isinstance(value, valid_value_types) and not hasattr(value, "item"): original = value try: @@ -298,8 +371,6 @@ def sanitize(value): except Exception: raise CastingFailureException() else: - _logger.debug( - "Attribute value is of type: %r. Casting %r to " "string: %s", type(original), original, value - ) + _logger.debug("Attribute value is of type: %r. Casting %r to string: %s", type(original), original, value) return value diff --git a/newrelic/core/attribute_filter.py b/newrelic/core/attribute_filter.py index 8d4a93843b..a07d016416 100644 --- a/newrelic/core/attribute_filter.py +++ b/newrelic/core/attribute_filter.py @@ -15,16 +15,17 @@ # Attribute "destinations" represented as bitfields. DST_NONE = 0x0 -DST_ALL = 0x3F -DST_TRANSACTION_EVENTS = 1 << 0 -DST_TRANSACTION_TRACER = 1 << 1 -DST_ERROR_COLLECTOR = 1 << 2 -DST_BROWSER_MONITORING = 1 << 3 -DST_SPAN_EVENTS = 1 << 4 +DST_ALL = 0x7F +DST_TRANSACTION_EVENTS = 1 << 0 +DST_TRANSACTION_TRACER = 1 << 1 +DST_ERROR_COLLECTOR = 1 << 2 +DST_BROWSER_MONITORING = 1 << 3 +DST_SPAN_EVENTS = 1 << 4 DST_TRANSACTION_SEGMENTS = 1 << 5 +DST_LOG_EVENT_CONTEXT_DATA = 1 << 6 -class AttributeFilter(object): +class AttributeFilter(): # Apply filtering rules to attributes. # # Upon initialization, an AttributeFilter object will take all attribute @@ -59,46 +60,45 @@ class AttributeFilter(object): # 4. Return the resulting bitfield after all rules have been applied. def __init__(self, flattened_settings): - self.enabled_destinations = self._set_enabled_destinations(flattened_settings) self.rules = self._build_rules(flattened_settings) self.cache = {} def __repr__(self): - return "" % ( - bin(self.enabled_destinations), self.rules) + return f"" def _set_enabled_destinations(self, settings): - # Determines and returns bitfield representing attribute destinations enabled. enabled_destinations = DST_NONE - if settings.get('transaction_segments.attributes.enabled', None): + if settings.get("transaction_segments.attributes.enabled", None): enabled_destinations |= DST_TRANSACTION_SEGMENTS - if settings.get('span_events.attributes.enabled', None): + if settings.get("span_events.attributes.enabled", None): enabled_destinations |= DST_SPAN_EVENTS - if settings.get('transaction_tracer.attributes.enabled', None): + if settings.get("transaction_tracer.attributes.enabled", None): enabled_destinations |= DST_TRANSACTION_TRACER - if settings.get('transaction_events.attributes.enabled', None): + if settings.get("transaction_events.attributes.enabled", None): enabled_destinations |= DST_TRANSACTION_EVENTS - if settings.get('error_collector.attributes.enabled', None): + if settings.get("error_collector.attributes.enabled", None): enabled_destinations |= DST_ERROR_COLLECTOR - if settings.get('browser_monitoring.attributes.enabled', None): + if settings.get("browser_monitoring.attributes.enabled", None): enabled_destinations |= DST_BROWSER_MONITORING - if not settings.get('attributes.enabled', None): + if settings.get("application_logging.forwarding.context_data.enabled", None): + enabled_destinations |= DST_LOG_EVENT_CONTEXT_DATA + + if not settings.get("attributes.enabled", None): enabled_destinations = DST_NONE return enabled_destinations def _build_rules(self, settings): - # "Rule Templates" below are used for building AttributeFilterRules. # # Each tuple includes: @@ -107,26 +107,27 @@ def _build_rules(self, settings): # 3. Boolean that represents whether the setting is an "include" or not. rule_templates = ( - ('attributes.include', DST_ALL, True), - ('attributes.exclude', DST_ALL, False), - ('transaction_events.attributes.include', DST_TRANSACTION_EVENTS, True), - ('transaction_events.attributes.exclude', DST_TRANSACTION_EVENTS, False), - ('transaction_tracer.attributes.include', DST_TRANSACTION_TRACER, True), - ('transaction_tracer.attributes.exclude', DST_TRANSACTION_TRACER, False), - ('error_collector.attributes.include', DST_ERROR_COLLECTOR, True), - ('error_collector.attributes.exclude', DST_ERROR_COLLECTOR, False), - ('browser_monitoring.attributes.include', DST_BROWSER_MONITORING, True), - ('browser_monitoring.attributes.exclude', DST_BROWSER_MONITORING, False), - ('span_events.attributes.include', DST_SPAN_EVENTS, True), - ('span_events.attributes.exclude', DST_SPAN_EVENTS, False), - ('transaction_segments.attributes.include', DST_TRANSACTION_SEGMENTS, True), - ('transaction_segments.attributes.exclude', DST_TRANSACTION_SEGMENTS, False), + ("attributes.include", DST_ALL, True), + ("attributes.exclude", DST_ALL, False), + ("transaction_events.attributes.include", DST_TRANSACTION_EVENTS, True), + ("transaction_events.attributes.exclude", DST_TRANSACTION_EVENTS, False), + ("transaction_tracer.attributes.include", DST_TRANSACTION_TRACER, True), + ("transaction_tracer.attributes.exclude", DST_TRANSACTION_TRACER, False), + ("error_collector.attributes.include", DST_ERROR_COLLECTOR, True), + ("error_collector.attributes.exclude", DST_ERROR_COLLECTOR, False), + ("browser_monitoring.attributes.include", DST_BROWSER_MONITORING, True), + ("browser_monitoring.attributes.exclude", DST_BROWSER_MONITORING, False), + ("span_events.attributes.include", DST_SPAN_EVENTS, True), + ("span_events.attributes.exclude", DST_SPAN_EVENTS, False), + ("transaction_segments.attributes.include", DST_TRANSACTION_SEGMENTS, True), + ("transaction_segments.attributes.exclude", DST_TRANSACTION_SEGMENTS, False), + ("application_logging.forwarding.context_data.include", DST_LOG_EVENT_CONTEXT_DATA, True), + ("application_logging.forwarding.context_data.exclude", DST_LOG_EVENT_CONTEXT_DATA, False), ) rules = [] - for (setting_name, destination, is_include) in rule_templates: - + for setting_name, destination, is_include in rule_templates: for setting in settings.get(setting_name) or (): rule = AttributeFilterRule(setting, destination, is_include) rules.append(rule) @@ -157,16 +158,15 @@ def apply(self, name, default_destinations): self.cache[cache_index] = destinations return destinations -class AttributeFilterRule(object): +class AttributeFilterRule(): def __init__(self, name, destinations, is_include): - self.name = name.rstrip('*') + self.name = name.rstrip("*") self.destinations = destinations self.is_include = is_include - self.is_wildcard = name.endswith('*') + self.is_wildcard = name.endswith("*") def _as_sortable(self): - # Represent AttributeFilterRule as a tuple that will sort properly. # # Sorting rules: @@ -207,8 +207,7 @@ def __ge__(self, other): return self._as_sortable() >= other._as_sortable() def __repr__(self): - return '(%s, %s, %s, %s)' % (self.name, bin(self.destinations), - self.is_wildcard, self.is_include) + return f"({self.name}, {bin(self.destinations)}, {self.is_wildcard}, {self.is_include})" def name_match(self, name): if self.is_wildcard: diff --git a/newrelic/core/code_level_metrics.py b/newrelic/core/code_level_metrics.py index ba00d93af7..846a95b4fd 100644 --- a/newrelic/core/code_level_metrics.py +++ b/newrelic/core/code_level_metrics.py @@ -34,7 +34,7 @@ def add_attrs(self, add_attr_function): # Add attributes for k, v in self._asdict().items(): if v is not None: - add_attr_function("code.%s" % k, v) + add_attr_function(f"code.{k}", v) def extract_code_from_callable(func): @@ -97,7 +97,7 @@ def extract_code_from_callable(func): func_name = func_path[-1] # function name is last in path if len(func_path) > 1: class_name = ".".join((func_path[:-1])) - namespace = ".".join((module_name, class_name)) + namespace = f"{module_name}.{class_name}" else: namespace = module_name diff --git a/newrelic/core/config.py b/newrelic/core/config.py index 7489be2229..e470a1b4db 100644 --- a/newrelic/core/config.py +++ b/newrelic/core/config.py @@ -28,16 +28,12 @@ import os import re import threading +import urllib.parse as urlparse -import newrelic.packages.six as six from newrelic.common.object_names import parse_exc_info +from newrelic.core.attribute import MAX_ATTRIBUTE_LENGTH from newrelic.core.attribute_filter import AttributeFilter -try: - import urlparse -except ImportError: - import urllib.parse as urlparse - try: import grpc @@ -51,11 +47,14 @@ # By default, Transaction Events and Custom Events have the same size # reservoir. Error Events have a different default size. +# Slow harvest (Every 60 seconds) DEFAULT_RESERVOIR_SIZE = 1200 -CUSTOM_EVENT_RESERVOIR_SIZE = 3600 ERROR_EVENT_RESERVOIR_SIZE = 100 SPAN_EVENT_RESERVOIR_SIZE = 2000 +# Fast harvest (Every 5 seconds, so divide by 12 to get average per minute value) +CUSTOM_EVENT_RESERVOIR_SIZE = 3600 LOG_EVENT_RESERVOIR_SIZE = 10000 +ML_EVENT_RESERVOIR_SIZE = 100000 # settings that should be completely ignored if set server side IGNORED_SERVER_SIDE_SETTINGS = [ @@ -82,7 +81,7 @@ def emit(self, record): # sub categories we don't know about. -class Settings(object): +class Settings: nested = False def __repr__(self): @@ -101,6 +100,7 @@ def create_settings(nested): class TopLevelSettings(Settings): _host = None + _otlp_host = None @property def host(self): @@ -112,6 +112,16 @@ def host(self): def host(self, value): self._host = value + @property + def otlp_host(self): + if self._otlp_host: + return self._otlp_host + return default_otlp_host(self.host) + + @otlp_host.setter + def otlp_host(self, value): + self._otlp_host = value + class AttributesSettings(Settings): pass @@ -121,6 +131,44 @@ class GCRuntimeMetricsSettings(Settings): enabled = False +class MemoryRuntimeMetricsSettings(Settings): + pass + + +class MachineLearningSettings(Settings): + pass + + +class MachineLearningInferenceEventsValueSettings(Settings): + pass + + +class AIMonitoringSettings(Settings): + @property + def llm_token_count_callback(self): + return self._llm_token_count_callback + + +class AIMonitoringStreamingSettings(Settings): + pass + + +class AIMonitoringRecordContentSettings(Settings): + pass + + +class K8sOperatorSettings(Settings): + pass + + +class AzureOperatorSettings(Settings): + pass + + +class PackageReportingSettings(Settings): + pass + + class CodeLevelMetricsSettings(Settings): pass @@ -199,6 +247,10 @@ class CustomInsightsEventsSettings(Settings): pass +class MlInsightsEventsSettings(Settings): + pass + + class ProcessHostSettings(Settings): pass @@ -271,6 +323,10 @@ class ApplicationLoggingForwardingSettings(Settings): pass +class ApplicationLoggingForwardingContextDataSettings(Settings): + pass + + class ApplicationLoggingMetricsSettings(Settings): pass @@ -279,6 +335,30 @@ class ApplicationLoggingLocalDecoratingSettings(Settings): pass +class SecuritySettings(Settings): + pass + + +class SecurityDetectionSettings(Settings): + pass + + +class SecurityAgentSettings(Settings): + pass + + +class SecurityDetectionRCISettings(Settings): + pass + + +class SecurityDetectionRXSSSettings(Settings): + pass + + +class SecurityDetectionDeserializationSettings(Settings): + pass + + class InfiniteTracingSettings(Settings): _trace_observer_host = None @@ -368,8 +448,18 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.agent_limits = AgentLimitsSettings() _settings.application_logging = ApplicationLoggingSettings() _settings.application_logging.forwarding = ApplicationLoggingForwardingSettings() +_settings.application_logging.forwarding.context_data = ApplicationLoggingForwardingContextDataSettings() +_settings.application_logging.metrics = ApplicationLoggingMetricsSettings() _settings.application_logging.local_decorating = ApplicationLoggingLocalDecoratingSettings() _settings.application_logging.metrics = ApplicationLoggingMetricsSettings() +_settings.machine_learning = MachineLearningSettings() +_settings.machine_learning.inference_events_value = MachineLearningInferenceEventsValueSettings() +_settings.ai_monitoring = AIMonitoringSettings() +_settings.ai_monitoring.streaming = AIMonitoringStreamingSettings() +_settings.ai_monitoring.record_content = AIMonitoringRecordContentSettings() +_settings.k8s_operator = K8sOperatorSettings() +_settings.azure_operator = AzureOperatorSettings() +_settings.package_reporting = PackageReportingSettings() _settings.attributes = AttributesSettings() _settings.browser_monitoring = BrowserMonitorSettings() _settings.browser_monitoring.attributes = BrowserMonitorAttributesSettings() @@ -377,6 +467,7 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.console = ConsoleSettings() _settings.cross_application_tracer = CrossApplicationTracerSettings() _settings.custom_insights_events = CustomInsightsEventsSettings() +_settings.ml_insights_events = MlInsightsEventsSettings() _settings.datastore_tracer = DatastoreTracerSettings() _settings.datastore_tracer.database_name_reporting = DatastoreTracerDatabaseNameReportingSettings() _settings.datastore_tracer.instance_reporting = DatastoreTracerInstanceReportingSettings() @@ -388,6 +479,7 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.event_harvest_config.harvest_limits = EventHarvestConfigHarvestLimitSettings() _settings.event_loop_visibility = EventLoopVisibilitySettings() _settings.gc_runtime_metrics = GCRuntimeMetricsSettings() +_settings.memory_runtime_pid_metrics = MemoryRuntimeMetricsSettings() _settings.heroku = HerokuSettings() _settings.infinite_tracing = InfiniteTracingSettings() _settings.instrumentation = InstrumentationSettings() @@ -395,6 +487,12 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.message_tracer = MessageTracerSettings() _settings.process_host = ProcessHostSettings() _settings.rum = RumSettings() +_settings.security = SecuritySettings() +_settings.security.agent = SecurityAgentSettings() +_settings.security.detection = SecurityDetectionSettings() +_settings.security.detection.deserialization = SecurityDetectionDeserializationSettings() +_settings.security.detection.rci = SecurityDetectionRCISettings() +_settings.security.detection.rxss = SecurityDetectionRXSSSettings() _settings.serverless_mode = ServerlessModeSettings() _settings.slow_sql = SlowSqlSettings() _settings.span_events = SpanEventSettings() @@ -412,7 +510,6 @@ class EventHarvestConfigHarvestLimitSettings(Settings): _settings.transaction_tracer.attributes = TransactionTracerAttributesSettings() _settings.utilization = UtilizationSettings() - _settings.log_file = os.environ.get("NEW_RELIC_LOG", None) _settings.audit_log_file = os.environ.get("NEW_RELIC_AUDIT_LOG", None) @@ -538,10 +635,28 @@ def default_host(license_key): return "collector.newrelic.com" region = region_aware_match.group(1) - host = "collector." + region + ".nr-data.net" + host = f"collector.{region}.nr-data.net" return host +def default_otlp_host(host): + HOST_MAP = { + "collector.newrelic.com": "otlp.nr-data.net", + "collector.eu.newrelic.com": "otlp.eu01.nr-data.net", + "gov-collector.newrelic.com": "gov-otlp.nr-data.net", + "staging-collector.newrelic.com": "staging-otlp.nr-data.net", + "staging-collector.eu.newrelic.com": "staging-otlp.eu01.nr-data.net", + "staging-gov-collector.newrelic.com": "staging-gov-otlp.nr-data.net", + "fake-collector.newrelic.com": "fake-otlp.nr-data.net", + } + otlp_host = HOST_MAP.get(host, None) + if not otlp_host: + default = HOST_MAP["collector.newrelic.com"] + _logger.warn(f"Unable to find corresponding OTLP host using default {default}") + otlp_host = default + return otlp_host + + _LOG_LEVEL = { "CRITICAL": logging.CRITICAL, "ERROR": logging.ERROR, @@ -567,7 +682,9 @@ def default_host(license_key): _settings.ssl = _environ_as_bool("NEW_RELIC_SSL", True) _settings.host = os.environ.get("NEW_RELIC_HOST") +_settings.otlp_host = os.environ.get("NEW_RELIC_OTLP_HOST") _settings.port = int(os.environ.get("NEW_RELIC_PORT", "0")) +_settings.otlp_port = int(os.environ.get("NEW_RELIC_OTLP_PORT", "0")) _settings.agent_run_id = None _settings.entity_guid = None @@ -659,15 +776,24 @@ def default_host(license_key): _settings.thread_profiler.enabled = True _settings.cross_application_tracer.enabled = False -_settings.gc_runtime_metrics.enabled = False +_settings.gc_runtime_metrics.enabled = _environ_as_bool("NEW_RELIC_GC_RUNTIME_METRICS_ENABLED", default=False) _settings.gc_runtime_metrics.top_object_count_limit = 5 +_settings.memory_runtime_pid_metrics.enabled = _environ_as_bool( + "NEW_RELIC_MEMORY_RUNTIME_PID_METRICS_ENABLED", default=True +) + _settings.transaction_events.enabled = True _settings.transaction_events.attributes.enabled = True _settings.transaction_events.attributes.exclude = [] _settings.transaction_events.attributes.include = [] _settings.custom_insights_events.enabled = True +_settings.custom_insights_events.max_attribute_value = _environ_as_int( + "NEW_RELIC_CUSTOM_INSIGHTS_EVENTS_MAX_ATTRIBUTE_VALUE", default=MAX_ATTRIBUTE_LENGTH +) + +_settings.ml_insights_events.enabled = False _settings.distributed_tracing.enabled = _environ_as_bool("NEW_RELIC_DISTRIBUTED_TRACING_ENABLED", default=True) _settings.distributed_tracing.exclude_newrelic_header = False @@ -760,6 +886,10 @@ def default_host(license_key): "NEW_RELIC_CUSTOM_INSIGHTS_EVENTS_MAX_SAMPLES_STORED", CUSTOM_EVENT_RESERVOIR_SIZE ) +_settings.event_harvest_config.harvest_limits.ml_event_data = _environ_as_int( + "NEW_RELIC_ML_INSIGHTS_EVENTS_MAX_SAMPLES_STORED", ML_EVENT_RESERVOIR_SIZE +) + _settings.event_harvest_config.harvest_limits.span_event_data = _environ_as_int( "NEW_RELIC_SPAN_EVENTS_MAX_SAMPLES_STORED", SPAN_EVENT_RESERVOIR_SIZE ) @@ -797,6 +927,7 @@ def default_host(license_key): _settings.debug.log_untrusted_distributed_trace_keys = False _settings.debug.disable_harvest_until_shutdown = False _settings.debug.connect_span_stream_in_developer_mode = False +_settings.debug.otlp_content_encoding = None _settings.message_tracer.segment_parameters_enabled = True @@ -833,12 +964,45 @@ def default_host(license_key): _settings.application_logging.forwarding.enabled = _environ_as_bool( "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_ENABLED", default=True ) +_settings.application_logging.forwarding.context_data.enabled = _environ_as_bool( + "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CONTEXT_DATA_ENABLED", default=False +) +_settings.application_logging.forwarding.context_data.include = _environ_as_set( + "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CONTEXT_DATA_INCLUDE", default="" +) +_settings.application_logging.forwarding.context_data.exclude = _environ_as_set( + "NEW_RELIC_APPLICATION_LOGGING_FORWARDING_CONTEXT_DATA_EXCLUDE", default="" +) _settings.application_logging.metrics.enabled = _environ_as_bool( "NEW_RELIC_APPLICATION_LOGGING_METRICS_ENABLED", default=True ) _settings.application_logging.local_decorating.enabled = _environ_as_bool( "NEW_RELIC_APPLICATION_LOGGING_LOCAL_DECORATING_ENABLED", default=False ) +_settings.machine_learning.enabled = _environ_as_bool("NEW_RELIC_MACHINE_LEARNING_ENABLED", default=False) +_settings.machine_learning.inference_events_value.enabled = _environ_as_bool( + "NEW_RELIC_MACHINE_LEARNING_INFERENCE_EVENT_VALUE_ENABLED", default=False +) +_settings.ai_monitoring.enabled = _environ_as_bool("NEW_RELIC_AI_MONITORING_ENABLED", default=False) +_settings.ai_monitoring.streaming.enabled = _environ_as_bool("NEW_RELIC_AI_MONITORING_STREAMING_ENABLED", default=True) +_settings.ai_monitoring.record_content.enabled = _environ_as_bool( + "NEW_RELIC_AI_MONITORING_RECORD_CONTENT_ENABLED", default=True +) +_settings.ai_monitoring._llm_token_count_callback = None +_settings.k8s_operator.enabled = _environ_as_bool("NEW_RELIC_K8S_OPERATOR_ENABLED", default=False) +_settings.azure_operator.enabled = _environ_as_bool("NEW_RELIC_AZURE_OPERATOR_ENABLED", default=False) +_settings.package_reporting.enabled = _environ_as_bool("NEW_RELIC_PACKAGE_REPORTING_ENABLED", default=True) +_settings.ml_insights_events.enabled = _environ_as_bool("NEW_RELIC_ML_INSIGHTS_EVENTS_ENABLED", default=False) + +_settings.security.agent.enabled = _environ_as_bool("NEW_RELIC_SECURITY_AGENT_ENABLED", False) +_settings.security.enabled = _environ_as_bool("NEW_RELIC_SECURITY_ENABLED", False) +_settings.security.mode = os.environ.get("NEW_RELIC_SECURITY_MODE", "IAST") +_settings.security.validator_service_url = os.environ.get("NEW_RELIC_SECURITY_VALIDATOR_SERVICE_URL", None) +_settings.security.detection.rci.enabled = _environ_as_bool("NEW_RELIC_SECURITY_DETECTION_RCI_ENABLED", True) +_settings.security.detection.rxss.enabled = _environ_as_bool("NEW_RELIC_SECURITY_DETECTION_RXSS_ENABLED", True) +_settings.security.detection.deserialization.enabled = _environ_as_bool( + "NEW_RELIC_SECURITY_DETECTION_DESERIALIZATION_ENABLED", True +) def global_settings(): @@ -874,7 +1038,7 @@ def _flatten(settings, o, name=None): key = key[1:] if name: - key = "%s.%s" % (name, key) + key = f"{name}.{key}" if isinstance(value, Settings): if value.nested: @@ -904,9 +1068,9 @@ def create_obfuscated_netloc(username, password, hostname, mask): password = mask if username and password: - netloc = "%s:%s@%s" % (username, password, hostname) + netloc = f"{username}:{password}@{hostname}" elif username: - netloc = "%s@%s" % (username, hostname) + netloc = f"{username}@{hostname}" else: netloc = hostname @@ -960,22 +1124,18 @@ def global_settings_dump(settings_object=None, serializable=False): netloc = create_obfuscated_netloc(components.username, components.password, components.hostname, obfuscated) if components.port: - uri = "%s://%s:%s%s" % (components.scheme, netloc, components.port, components.path) + uri = f"{components.scheme}://{netloc}:{components.port}{components.path}" else: - uri = "%s://%s%s" % (components.scheme, netloc, components.path) + uri = f"{components.scheme}://{netloc}{components.path}" settings["proxy_host"] = uri if serializable: - for key, value in list(six.iteritems(settings)): - if not isinstance(key, six.string_types): + for key, value in list(settings.items()): + if not isinstance(key, str): del settings[key] - if ( - not isinstance(value, six.string_types) - and not isinstance(value, float) - and not isinstance(value, six.integer_types) - ): + if not isinstance(value, str) and not isinstance(value, float) and not isinstance(value, int): settings[key] = repr(value) return settings @@ -1011,7 +1171,7 @@ def apply_config_setting(settings_object, name, value, nested=False): default_value = getattr(target, fields[0], None) if isinstance(value, dict) and value and not isinstance(default_value, dict): for k, v in value.items(): - k_name = "{}.{}".format(fields[0], k) + k_name = f"{fields[0]}.{k}" apply_config_setting(target, k_name, v, nested=True) else: setattr(target, fields[0], value) @@ -1083,8 +1243,8 @@ def apply_server_side_settings(server_side_config=None, settings=_settings): apply_config_setting(settings_snapshot, name, value) # Overlay with global server side configuration settings. - # global server side configuration always takes precedence over the global - # server side configuration settings. + # global server side configuration always takes precedence over the local + # agent configuration settings. for name, value in server_side_config.items(): apply_config_setting(settings_snapshot, name, value) @@ -1101,6 +1261,30 @@ def apply_server_side_settings(server_side_config=None, settings=_settings): settings_snapshot, "event_harvest_config.harvest_limits.span_event_data", span_event_harvest_limit ) + # Check to see if collect_ai appears in the connect response to handle account-level AIM toggling + collect_ai = server_side_config.get("collect_ai", None) + if collect_ai is not None: + apply_config_setting(settings_snapshot, "ai_monitoring.enabled", collect_ai) + _logger.debug("Setting ai_monitoring.enabled to value of collect_ai=%s", collect_ai) + + # Since the server does not override this setting as it's an OTLP setting, + # we must override it here manually by converting it into a per harvest cycle + # value. + apply_config_setting( + settings_snapshot, + "event_harvest_config.harvest_limits.ml_event_data", + # override ml_events / (60s/5s) harvest + settings_snapshot.event_harvest_config.harvest_limits.ml_event_data / 12, + ) + + # Since the server does not override this setting we must override it here manually + # by caping it at the max value of 4095. + apply_config_setting( + settings_snapshot, + "custom_insights_events.max_attribute_value", + min(settings_snapshot.custom_insights_events.max_attribute_value, 4095), + ) + # This will be removed at some future point # Special case for account_id which will be sent instead of # cross_process_id in the future @@ -1223,8 +1407,8 @@ def error_matches_rules( return None # Retrieve settings based on prefix - classes_rules = getattr(settings.error_collector, "%s_classes" % rules_prefix, set()) - status_codes_rules = getattr(settings.error_collector, "%s_status_codes" % rules_prefix, set()) + classes_rules = getattr(settings.error_collector, f"{rules_prefix}_classes", set()) + status_codes_rules = getattr(settings.error_collector, f"{rules_prefix}_status_codes", set()) _, _, fullnames, _ = parse_exc_info(exc_info) fullname = fullnames[0] @@ -1246,7 +1430,7 @@ def error_matches_rules( # Coerce into integer status_code = int(status_code) except: - _logger.error("Failed to coerce status code into integer. status_code: %s" % str(status_code)) + _logger.error(f"Failed to coerce status code into integer. status_code: {str(status_code)}") else: if status_code in status_codes_rules: return True diff --git a/newrelic/core/context.py b/newrelic/core/context.py index 7560855aef..c782350ff6 100644 --- a/newrelic/core/context.py +++ b/newrelic/core/context.py @@ -24,7 +24,7 @@ _logger = logging.getLogger(__name__) -class ContextOf(object): +class ContextOf(): def __init__(self, trace=None, request=None, trace_cache_id=None, strict=True): self.trace = None self.trace_cache = trace_cache() @@ -48,7 +48,7 @@ def log_propagation_failure(s): elif trace_cache_id is not None: self.trace = self.trace_cache.get(trace_cache_id, None) if self.trace is None: - log_propagation_failure("No trace with id %d." % trace_cache_id) + log_propagation_failure(f"No trace with id {trace_cache_id}.") elif hasattr(request, "_nr_trace") and request._nr_trace is not None: # Unpack traces from objects patched with them self.trace = request._nr_trace diff --git a/newrelic/core/custom_event.py b/newrelic/core/custom_event.py index 206fb84e68..3741165e0e 100644 --- a/newrelic/core/custom_event.py +++ b/newrelic/core/custom_event.py @@ -11,27 +11,41 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import logging import re import time -from newrelic.core.attribute import (check_name_is_string, check_name_length, - process_user_attribute, NameIsNotStringException, NameTooLongException, - MAX_NUM_USER_ATTRIBUTES) - +from newrelic.core.attribute import ( + MAX_ML_ATTRIBUTE_LENGTH, + MAX_NUM_ML_USER_ATTRIBUTES, + MAX_NUM_USER_ATTRIBUTES, + NameIsNotStringException, + NameTooLongException, + check_name_is_string, + check_name_length, + process_user_attribute, +) +from newrelic.core.config import global_settings _logger = logging.getLogger(__name__) -EVENT_TYPE_VALID_CHARS_REGEX = re.compile(r'^[a-zA-Z0-9:_ ]+$') +EVENT_TYPE_VALID_CHARS_REGEX = re.compile(r"^[a-zA-Z0-9:_ ]+$") +NO_LIMIT_LLM_EVENT_TYPE = { + "LlmChatCompletionMessage": "content", + "LlmEmbedding": "input", +} + + +class NameInvalidCharactersException(Exception): + pass -class NameInvalidCharactersException(Exception): pass def check_event_type_valid_chars(name): regex = EVENT_TYPE_VALID_CHARS_REGEX if not regex.match(name): raise NameInvalidCharactersException() + def process_event_type(name): """Perform all necessary validation on a potential event type. @@ -55,24 +69,22 @@ def process_event_type(name): check_event_type_valid_chars(name) except NameIsNotStringException: - _logger.debug('Event type must be a string. Dropping ' - 'event: %r', name) + _logger.debug("Event type must be a string. Dropping event: %r", name) return FAILED_RESULT except NameTooLongException: - _logger.debug('Event type exceeds maximum length. Dropping ' - 'event: %r', name) + _logger.debug("Event type exceeds maximum length. Dropping event: %r", name) return FAILED_RESULT except NameInvalidCharactersException: - _logger.debug('Event type has invalid characters. Dropping ' - 'event: %r', name) + _logger.debug("Event type has invalid characters. Dropping event: %r", name) return FAILED_RESULT else: return name -def create_custom_event(event_type, params): + +def create_custom_event(event_type, params, settings=None, is_ml_event=False): """Creates a valid custom event. Ensures that the custom event has a valid name, and also checks @@ -83,12 +95,16 @@ def create_custom_event(event_type, params): Args: event_type (str): The type (name) of the custom event. params (dict): Attributes to add to the event. + settings: Optional config settings. + is_ml_event (bool): Boolean indicating whether create_custom_event was called from + record_ml_event for truncation purposes Returns: Custom event (list of 2 dicts), if successful. None, if not successful. """ + settings = settings or global_settings() name = process_event_type(event_type) @@ -99,23 +115,38 @@ def create_custom_event(event_type, params): try: for k, v in params.items(): - key, value = process_user_attribute(k, v) + if is_ml_event: + max_length = MAX_ML_ATTRIBUTE_LENGTH + max_num_attrs = MAX_NUM_ML_USER_ATTRIBUTES + else: + max_length = ( + settings.custom_insights_events.max_attribute_value + if not (NO_LIMIT_LLM_EVENT_TYPE.get(name) == k) + else None + ) + max_num_attrs = MAX_NUM_USER_ATTRIBUTES + key, value = process_user_attribute(k, v, max_length=max_length) if key: - if len(attributes) >= MAX_NUM_USER_ATTRIBUTES: - _logger.debug('Maximum number of attributes already ' - 'added to event %r. Dropping attribute: %r=%r', - name, key, value) + if len(attributes) >= max_num_attrs: + _logger.debug( + "Maximum number of attributes already added to event %r. Dropping attribute: %r=%r", + name, + key, + value, + ) else: attributes[key] = value except Exception: - _logger.debug('Attributes failed to validate for unknown reason. ' - 'Check traceback for clues. Dropping event: %r.', name, - exc_info=True) + _logger.debug( + "Attributes failed to validate for unknown reason. Check traceback for clues. Dropping event: %r.", + name, + exc_info=True, + ) return None intrinsics = { - 'type': name, - 'timestamp': int(1000.0 * time.time()), + "type": name, + "timestamp": int(1000.0 * time.time()), } event = [intrinsics, attributes] diff --git a/newrelic/core/data_collector.py b/newrelic/core/data_collector.py index 985e37240d..2d312bc0d5 100644 --- a/newrelic/core/data_collector.py +++ b/newrelic/core/data_collector.py @@ -16,8 +16,6 @@ """ -from __future__ import print_function - import logging from newrelic.common.agent_http import ( @@ -25,21 +23,30 @@ DeveloperModeClient, ServerlessModeClient, ) -from newrelic.core.agent_protocol import AgentProtocol, ServerlessModeProtocol +from newrelic.core.agent_protocol import ( + AgentProtocol, + OtlpProtocol, + ServerlessModeProtocol, +) from newrelic.core.agent_streaming import StreamingRpc from newrelic.core.config import global_settings +from newrelic.core.otlp_utils import encode_metric_data, encode_ml_event_data _logger = logging.getLogger(__name__) -class Session(object): +class Session(): PROTOCOL = AgentProtocol + OTLP_PROTOCOL = OtlpProtocol CLIENT = ApplicationModeClient def __init__(self, app_name, linked_applications, environment, settings): self._protocol = self.PROTOCOL.connect( app_name, linked_applications, environment, settings, client_cls=self.CLIENT ) + self._otlp_protocol = self.OTLP_PROTOCOL.connect( + app_name, linked_applications, environment, settings, client_cls=self.CLIENT + ) self._rpc = None @property @@ -62,7 +69,7 @@ def connect_span_stream(self, span_iterator, record_metric): port = self.configuration.infinite_tracing.trace_observer_port ssl = self.configuration.infinite_tracing.ssl compression_setting = self.configuration.infinite_tracing.compression - endpoint = "{}:{}".format(host, port) + endpoint = f"{host}:{port}" if ( self.configuration.distributed_tracing.enabled @@ -112,6 +119,11 @@ def send_custom_events(self, sampling_info, custom_event_data): payload = (self.agent_run_id, sampling_info, custom_event_data) return self._protocol.send("custom_event_data", payload) + def send_ml_events(self, sampling_info, custom_event_data): + """Called to submit sample set for machine learning events.""" + payload = encode_ml_event_data(custom_event_data, str(self.agent_run_id)) + return self._otlp_protocol.send("ml_event_data", payload, path="/v1/logs") + def send_span_events(self, sampling_info, span_event_data): """Called to submit sample set for span events.""" @@ -128,6 +140,20 @@ def send_metric_data(self, start_time, end_time, metric_data): payload = (self.agent_run_id, start_time, end_time, metric_data) return self._protocol.send("metric_data", payload) + def send_dimensional_metric_data(self, start_time, end_time, metric_data): + """Called to submit dimensional metric data for specified period of time. + Time values are seconds since UNIX epoch as returned by the + time.time() function. The metric data should be iterable of + specific metrics. + + NOTE: This data is sent not sent to the normal agent endpoints but is sent + to the OTLP API endpoints to keep the entity separate. This is for use + with the machine learning integration only. + """ + + payload = encode_metric_data(metric_data, start_time, end_time) + return self._otlp_protocol.send("dimensional_metric_data", payload, path="/v1/metrics") + def send_log_events(self, sampling_info, log_event_data): """Called to submit sample set for log events.""" diff --git a/newrelic/core/database_node.py b/newrelic/core/database_node.py index 3db38152f2..7eb18c6b96 100644 --- a/newrelic/core/database_node.py +++ b/newrelic/core/database_node.py @@ -16,22 +16,35 @@ import newrelic.core.attribute as attribute import newrelic.core.trace_node - from newrelic.common import system_info -from newrelic.core.database_utils import sql_statement, explain_plan -from newrelic.core.node_mixin import DatastoreNodeMixin +from newrelic.core.database_utils import explain_plan, sql_statement from newrelic.core.metric import TimeMetric +from newrelic.core.node_mixin import DatastoreNodeMixin - -_SlowSqlNode = namedtuple('_SlowSqlNode', - ['duration', 'path', 'request_uri', 'sql', 'sql_format', - 'metric', 'dbapi2_module', 'stack_trace', 'connect_params', - 'cursor_params', 'sql_parameters', 'execute_params', - 'host', 'port_path_or_id', 'database_name', 'params']) +_SlowSqlNode = namedtuple( + "_SlowSqlNode", + [ + "duration", + "path", + "request_uri", + "sql", + "sql_format", + "metric", + "dbapi2_module", + "stack_trace", + "connect_params", + "cursor_params", + "sql_parameters", + "execute_params", + "host", + "port_path_or_id", + "database_name", + "params", + ], +) class SlowSqlNode(_SlowSqlNode): - def __new__(cls, *args, **kwargs): node = _SlowSqlNode.__new__(cls, *args, **kwargs) node.statement = sql_statement(node.sql, node.dbapi2_module) @@ -46,16 +59,33 @@ def identifier(self): return self.statement.identifier -_DatabaseNode = namedtuple('_DatabaseNode', - ['dbapi2_module', 'sql', 'children', 'start_time', 'end_time', - 'duration', 'exclusive', 'stack_trace', 'sql_format', - 'connect_params', 'cursor_params', 'sql_parameters', - 'execute_params', 'host', 'port_path_or_id', 'database_name', - 'guid', 'agent_attributes', 'user_attributes']) +_DatabaseNode = namedtuple( + "_DatabaseNode", + [ + "dbapi2_module", + "sql", + "children", + "start_time", + "end_time", + "duration", + "exclusive", + "stack_trace", + "sql_format", + "connect_params", + "cursor_params", + "sql_parameters", + "execute_params", + "host", + "port_path_or_id", + "database_name", + "guid", + "agent_attributes", + "user_attributes", + ], +) class DatabaseNode(_DatabaseNode, DatastoreNodeMixin): - def __new__(cls, *args, **kwargs): node = _DatabaseNode.__new__(cls, *args, **kwargs) node.statement = sql_statement(node.sql, node.dbapi2_module) @@ -86,9 +116,15 @@ def formatted(self): return self.statement.formatted(self.sql_format) def explain_plan(self, connections): - return explain_plan(connections, self.statement, self.connect_params, - self.cursor_params, self.sql_parameters, self.execute_params, - self.sql_format) + return explain_plan( + connections, + self.statement, + self.connect_params, + self.cursor_params, + self.sql_parameters, + self.execute_params, + self.sql_format, + ) def time_metrics(self, stats, root, parent): """Return a generator yielding the timed metrics for this @@ -97,80 +133,70 @@ def time_metrics(self, stats, root, parent): """ product = self.product - operation = self.operation or 'other' + operation = self.operation or "other" target = self.target # Determine the scoped metric - statement_metric_name = 'Datastore/statement/%s/%s/%s' % (product, - target, operation) + statement_metric_name = f"Datastore/statement/{product}/{target}/{operation}" - operation_metric_name = 'Datastore/operation/%s/%s' % (product, - operation) + operation_metric_name = f"Datastore/operation/{product}/{operation}" if target: scoped_metric_name = statement_metric_name else: scoped_metric_name = operation_metric_name - yield TimeMetric(name=scoped_metric_name, scope=root.path, - duration=self.duration, exclusive=self.exclusive) + yield TimeMetric(name=scoped_metric_name, scope=root.path, duration=self.duration, exclusive=self.exclusive) # Unscoped rollup metrics - yield TimeMetric(name='Datastore/all', scope='', - duration=self.duration, exclusive=self.exclusive) + yield TimeMetric(name="Datastore/all", scope="", duration=self.duration, exclusive=self.exclusive) - yield TimeMetric(name='Datastore/%s/all' % product, scope='', - duration=self.duration, exclusive=self.exclusive) + yield TimeMetric(name=f"Datastore/{product}/all", scope="", duration=self.duration, exclusive=self.exclusive) - if root.type == 'WebTransaction': - yield TimeMetric(name='Datastore/allWeb', scope='', - duration=self.duration, exclusive=self.exclusive) + if root.type == "WebTransaction": + yield TimeMetric(name="Datastore/allWeb", scope="", duration=self.duration, exclusive=self.exclusive) - yield TimeMetric(name='Datastore/%s/allWeb' % product, scope='', - duration=self.duration, exclusive=self.exclusive) + yield TimeMetric( + name=f"Datastore/{product}/allWeb", scope="", duration=self.duration, exclusive=self.exclusive + ) else: - yield TimeMetric(name='Datastore/allOther', scope='', - duration=self.duration, exclusive=self.exclusive) + yield TimeMetric(name="Datastore/allOther", scope="", duration=self.duration, exclusive=self.exclusive) - yield TimeMetric(name='Datastore/%s/allOther' % product, scope='', - duration=self.duration, exclusive=self.exclusive) + yield TimeMetric( + name=f"Datastore/{product}/allOther", scope="", duration=self.duration, exclusive=self.exclusive + ) # Unscoped operation metric - yield TimeMetric(name=operation_metric_name, scope='', - duration=self.duration, exclusive=self.exclusive) + yield TimeMetric(name=operation_metric_name, scope="", duration=self.duration, exclusive=self.exclusive) # Unscoped statement metric if target: - yield TimeMetric(name=statement_metric_name, scope='', - duration=self.duration, exclusive=self.exclusive) + yield TimeMetric(name=statement_metric_name, scope="", duration=self.duration, exclusive=self.exclusive) # Unscoped instance Metric if self.instance_hostname and self.port_path_or_id: - instance_metric_name = 'Datastore/instance/%s/%s/%s' % (product, - self.instance_hostname, self.port_path_or_id) + instance_metric_name = f"Datastore/instance/{product}/{self.instance_hostname}/{self.port_path_or_id}" - yield TimeMetric(name=instance_metric_name, scope='', - duration=self.duration, exclusive=self.exclusive) + yield TimeMetric(name=instance_metric_name, scope="", duration=self.duration, exclusive=self.exclusive) def slow_sql_node(self, stats, root): product = self.product - operation = self.operation or 'other' + operation = self.operation or "other" target = self.target if target: - name = 'Datastore/statement/%s/%s/%s' % (product, target, - operation) + name = f"Datastore/statement/{product}/{target}/{operation}" else: - name = 'Datastore/operation/%s/%s' % (product, operation) + name = f"Datastore/operation/{product}/{operation}" - request_uri = '' - if root.type == 'WebTransaction': + request_uri = "" + if root.type == "WebTransaction": request_uri = root.request_uri params = None @@ -182,19 +208,24 @@ def slow_sql_node(self, stats, root): # explain plan. Only limit the length when sending the # formatted SQL up to the data collector. - return SlowSqlNode(duration=self.duration, path=root.path, - request_uri=request_uri, sql=self.sql, - sql_format=self.sql_format, metric=name, - dbapi2_module=self.dbapi2_module, - stack_trace=self.stack_trace, - connect_params=self.connect_params, - cursor_params=self.cursor_params, - sql_parameters=self.sql_parameters, - execute_params=self.execute_params, - host=self.instance_hostname, - port_path_or_id=self.port_path_or_id, - database_name=self.database_name, - params=params) + return SlowSqlNode( + duration=self.duration, + path=root.path, + request_uri=request_uri, + sql=self.sql, + sql_format=self.sql_format, + metric=name, + dbapi2_module=self.dbapi2_module, + stack_trace=self.stack_trace, + connect_params=self.connect_params, + cursor_params=self.cursor_params, + sql_parameters=self.sql_parameters, + execute_params=self.execute_params, + host=self.instance_hostname, + port_path_or_id=self.port_path_or_id, + database_name=self.database_name, + params=params, + ) def trace_node(self, stats, root, connections): name = root.string_table.cache(self.name) @@ -209,31 +240,30 @@ def trace_node(self, stats, root, connections): sql = self.formatted # Agent attributes - self.agent_attributes['db.instance'] = self.db_instance + self.agent_attributes["db.instance"] = self.db_instance if sql: # Limit the length of any SQL that is reported back. limit = root.settings.agent_limits.sql_query_length_maximum - self.agent_attributes['db.statement'] = sql[:limit] + self.agent_attributes["db.statement"] = sql[:limit] params = self.get_trace_segment_params(root.settings) # Only send datastore instance params if not empty. if self.host: - params['host'] = self.instance_hostname + params["host"] = self.instance_hostname if self.port_path_or_id: - params['port_path_or_id'] = self.port_path_or_id + params["port_path_or_id"] = self.port_path_or_id - sql = params.get('db.statement') + sql = params.get("db.statement") if sql: - params['db.statement'] = root.string_table.cache(sql) + params["db.statement"] = root.string_table.cache(sql) if self.stack_trace: - params['backtrace'] = [root.string_table.cache(x) for x in - self.stack_trace] + params["backtrace"] = [root.string_table.cache(x) for x in self.stack_trace] # Only perform an explain plan if this node ended up being # flagged to have an explain plan. This is applied when cap @@ -241,24 +271,23 @@ def trace_node(self, stats, root, connections): # applied across all transaction traces just prior to the # transaction traces being generated. - if getattr(self, 'generate_explain_plan', None): + if getattr(self, "generate_explain_plan", None): explain_plan_data = self.explain_plan(connections) if explain_plan_data: - params['explain_plan'] = explain_plan_data + params["explain_plan"] = explain_plan_data - return newrelic.core.trace_node.TraceNode(start_time=start_time, - end_time=end_time, name=name, params=params, children=children, - label=None) + return newrelic.core.trace_node.TraceNode( + start_time=start_time, end_time=end_time, name=name, params=params, children=children, label=None + ) def span_event(self, *args, **kwargs): sql = self.formatted if sql: # Truncate to 2000 bytes and append ... - _, sql = attribute.process_user_attribute( - 'db.statement', sql, max_length=2000, ending='...') + _, sql = attribute.process_user_attribute("db.statement", sql, max_length=2000, ending="...") - self.agent_attributes['db.statement'] = sql + self.agent_attributes["db.statement"] = sql if self.target: self.agent_attributes["db.collection"] = self.target diff --git a/newrelic/core/database_utils.py b/newrelic/core/database_utils.py index 774fb93490..db734833b2 100644 --- a/newrelic/core/database_utils.py +++ b/newrelic/core/database_utils.py @@ -21,7 +21,6 @@ import re import weakref -import newrelic.packages.six as six from newrelic.core.internal_metrics import internal_metric from newrelic.core.config import global_settings @@ -45,9 +44,9 @@ _dollar_quotes_p = r'(\$(?!\d)[^$]*?\$).*?(?:\1|$)' _oracle_quotes_p = (r"q'\[.*?(?:\]'|$)|q'\{.*?(?:\}'|$)|" r"q'\<.*?(?:\>'|$)|q'\(.*?(?:\)'|$)") -_any_quotes_p = _single_quotes_p + '|' + _double_quotes_p -_single_dollar_p = _single_quotes_p + '|' + _dollar_quotes_p -_single_oracle_p = _single_quotes_p + '|' + _oracle_quotes_p +_any_quotes_p = f"{_single_quotes_p}|{_double_quotes_p}" +_single_dollar_p = f"{_single_quotes_p}|{_dollar_quotes_p}" +_single_oracle_p = f"{_single_quotes_p}|{_oracle_quotes_p}" _single_quotes_re = re.compile(_single_quotes_p) _any_quotes_re = re.compile(_any_quotes_p) @@ -86,7 +85,7 @@ # first to avoid the situation of partial matches on shorter expressions. UUIDs # might be an example. -_all_literals_p = '(' + ')|('.join([_uuid_p, _hex_p, _int_p, _bool_p]) + ')' +_all_literals_p = f"({_uuid_p})|({_hex_p})|({_int_p})|({_bool_p})" _all_literals_re = re.compile(_all_literals_p, re.IGNORECASE) _quotes_table = { @@ -279,10 +278,7 @@ def _parse_default(sql, regex): _parse_identifier_6_p = r'\{\s*(\S+)\s*\}' _parse_identifier_7_p = r'([^\s\(\)\[\],]+)' -_parse_identifier_p = ''.join(('(', _parse_identifier_1_p, '|', - _parse_identifier_2_p, '|', _parse_identifier_3_p, '|', - _parse_identifier_4_p, '|', _parse_identifier_5_p, '|', - _parse_identifier_6_p, '|', _parse_identifier_7_p, ')')) +_parse_identifier_p = f"({_parse_identifier_1_p}|{_parse_identifier_2_p}|{_parse_identifier_3_p}|{_parse_identifier_4_p}|{_parse_identifier_5_p}|{_parse_identifier_6_p}|{_parse_identifier_7_p})" _parse_from_p = r'\s+FROM\s+' + _parse_identifier_p _parse_from_re = re.compile(_parse_from_p, re.IGNORECASE) @@ -522,7 +518,7 @@ def _obfuscate_explain_plan(database, columns, rows): return columns, rows -class SQLConnection(object): +class SQLConnection(): def __init__(self, database, connection): self.database = database @@ -562,7 +558,7 @@ def cleanup(self): self.connection.close() -class SQLConnections(object): +class SQLConnections(): def __init__(self, maximum=4): self.connections = [] @@ -671,7 +667,7 @@ def _explain_plan(connections, sql, database, connect_params, cursor_params, 'semicolons in the query string.', database.client) return None - query = '%s %s' % (database.explain_query, sql) + query = f'{database.explain_query} {sql}' if settings.debug.log_explain_plan_queries: _logger.debug('Executing explain plan for %r on %r.', query, @@ -768,7 +764,7 @@ def explain_plan(connections, sql_statement, connect_params, cursor_params, # Wrapper for information about a specific database. -class SQLDatabase(object): +class SQLDatabase(): def __init__(self, dbapi2_module): self.dbapi2_module = dbapi2_module @@ -812,7 +808,7 @@ def explain_stmts(self): return result -class SQLStatement(object): +class SQLStatement(): def __init__(self, sql, database=None): self._operation = None @@ -822,14 +818,13 @@ def __init__(self, sql, database=None): self._normalized = None self._identifier = None - if isinstance(sql, six.binary_type): + if isinstance(sql, bytes): try: sql = sql.decode('utf-8') except UnicodeError as e: settings = global_settings() if settings.debug.log_explain_plan_queries: - _logger.debug('An error occurred while decoding sql ' - 'statement: %s' % e.reason) + _logger.debug(f'An error occurred while decoding sql statement: {e.reason}') self._operation = '' self._target = '' diff --git a/newrelic/core/datastore_node.py b/newrelic/core/datastore_node.py index 290ec08464..9300b5d94f 100644 --- a/newrelic/core/datastore_node.py +++ b/newrelic/core/datastore_node.py @@ -48,11 +48,9 @@ def time_metrics(self, stats, root, parent): # Determine the scoped metric - statement_metric_name = 'Datastore/statement/%s/%s/%s' % (product, - target, operation) + statement_metric_name = f'Datastore/statement/{product}/{target}/{operation}' - operation_metric_name = 'Datastore/operation/%s/%s' % (product, - operation) + operation_metric_name = f'Datastore/operation/{product}/{operation}' if target: scoped_metric_name = statement_metric_name @@ -67,20 +65,20 @@ def time_metrics(self, stats, root, parent): yield TimeMetric(name='Datastore/all', scope='', duration=self.duration, exclusive=self.exclusive) - yield TimeMetric(name='Datastore/%s/all' % product, scope='', + yield TimeMetric(name=f'Datastore/{product}/all', scope='', duration=self.duration, exclusive=self.exclusive) if root.type == 'WebTransaction': yield TimeMetric(name='Datastore/allWeb', scope='', duration=self.duration, exclusive=self.exclusive) - yield TimeMetric(name='Datastore/%s/allWeb' % product, scope='', + yield TimeMetric(name=f'Datastore/{product}/allWeb', scope='', duration=self.duration, exclusive=self.exclusive) else: yield TimeMetric(name='Datastore/allOther', scope='', duration=self.duration, exclusive=self.exclusive) - yield TimeMetric(name='Datastore/%s/allOther' % product, scope='', + yield TimeMetric(name=f'Datastore/{product}/allOther', scope='', duration=self.duration, exclusive=self.exclusive) # Unscoped operation metric @@ -102,8 +100,7 @@ def time_metrics(self, stats, root, parent): self.port_path_or_id and ds_tracer_settings.instance_reporting.enabled): - instance_metric_name = 'Datastore/instance/%s/%s/%s' % (product, - self.instance_hostname, self.port_path_or_id) + instance_metric_name = f'Datastore/instance/{product}/{self.instance_hostname}/{self.port_path_or_id}' yield TimeMetric(name=instance_metric_name, scope='', duration=self.duration, exclusive=self.exclusive) diff --git a/newrelic/core/environment.py b/newrelic/core/environment.py index 1306816efd..1ac78b3ecb 100644 --- a/newrelic/core/environment.py +++ b/newrelic/core/environment.py @@ -17,10 +17,10 @@ """ +import logging import os import platform import sys -import sysconfig import newrelic from newrelic.common.package_version_utils import get_package_version @@ -29,12 +29,16 @@ physical_processor_count, total_physical_memory, ) +from newrelic.core.config import global_settings +from newrelic.packages.isort import stdlibs as isort_stdlibs try: import newrelic.core._thread_utilization except ImportError: pass +_logger = logging.getLogger(__name__) + def environment_settings(): """Returns an array of arrays of environment settings""" @@ -148,7 +152,7 @@ def environment_settings(): dispatcher.append(("Dispatcher", "gunicorn (gevent)")) elif "gunicorn.workers.geventlet" in sys.modules: dispatcher.append(("Dispatcher", "gunicorn (eventlet)")) - elif "uvicorn.workers" in sys.modules: + elif "uvicorn.workers" in sys.modules or "uvicorn_worker" in sys.modules: dispatcher.append(("Dispatcher", "gunicorn (uvicorn)")) uvicorn = sys.modules.get("uvicorn") if hasattr(uvicorn, "__version__"): @@ -195,42 +199,75 @@ def environment_settings(): env.extend(dispatcher) # Module information. - purelib = sysconfig.get_path("purelib") - platlib = sysconfig.get_path("platlib") + stdlib_builtin_module_names = _get_stdlib_builtin_module_names() plugins = [] - # Using any iterable to create a snapshot of sys.modules can occassionally - # fail in a rare case when modules are imported in parallel by different - # threads. - # - # TL;DR: Do NOT use an iterable on the original sys.modules to generate the - # list - for name, module in sys.modules.copy().items(): - # Exclude lib.sub_paths as independent modules except for newrelic.hooks. - if "." in name and not name.startswith("newrelic.hooks."): - continue - # If the module isn't actually loaded (such as failed relative imports - # in Python 2.7), the module will be None and should not be reported. - if not module: - continue - # Exclude standard library/built-in modules. - # Third-party modules can be installed in either purelib or platlib directories. - # See https://docs.python.org/3/library/sysconfig.html#installation-paths. - if ( - not hasattr(module, "__file__") - or not module.__file__ - or not module.__file__.startswith(purelib) - or not module.__file__.startswith(platlib) - ): - continue - - try: - version = get_package_version(name) - plugins.append("%s (%s)" % (name, version)) - except Exception: - plugins.append(name) + settings = global_settings() + if settings and settings.package_reporting.enabled: + # Using any iterable to create a snapshot of sys.modules can occassionally + # fail in a rare case when modules are imported in parallel by different + # threads. + # + # TL;DR: Do NOT use an iterable on the original sys.modules to generate the + # list + for name, module in sys.modules.copy().items(): + # Exclude lib.sub_paths as independent modules except for newrelic.hooks. + nr_hook = name.startswith("newrelic.hooks.") + if "." in name and not nr_hook or name.startswith("_"): + continue + + # If the module isn't actually loaded (such as failed relative imports + # in Python 2.7), the module will be None and should not be reported. + try: + if not module: + continue + except Exception: + # if the application uses generalimport to manage optional depedencies, + # it's possible that generalimport.MissingOptionalDependency is raised. + # In this case, we should not report the module as it is not actually loaded and + # is not a runtime dependency of the application. + # + continue + + # Exclude standard library/built-in modules. + if name in stdlib_builtin_module_names: + continue + + # Don't attempt to look up version information for our hooks + version = None + if not nr_hook: + try: + version = get_package_version(name) + except Exception: + pass + + # If it has no version it's likely not a real package so don't report it unless + # it's a new relic hook. + if nr_hook or version: + plugins.append(f"{name} ({version})") env.append(("Plugin List", plugins)) return env + + +def _get_stdlib_builtin_module_names(): + builtins = set(sys.builtin_module_names) + # Since sys.stdlib_module_names is not available in versions of python below 3.10, + # use isort's hardcoded stdlibs instead. + python_version = sys.version_info[0:2] + if python_version < (3,): + stdlibs = isort_stdlibs.py27.stdlib + elif (3, 7) <= python_version < (3, 8): + stdlibs = isort_stdlibs.py37.stdlib + elif python_version < (3, 9): + stdlibs = isort_stdlibs.py38.stdlib + elif python_version < (3, 10): + stdlibs = isort_stdlibs.py39.stdlib + elif python_version >= (3, 10): + stdlibs = sys.stdlib_module_names + else: + _logger.warn("Unsupported Python version. Unable to determine stdlibs.") + return builtins + return builtins | stdlibs diff --git a/newrelic/core/error_collector.py b/newrelic/core/error_collector.py index 9a46433840..c8062927ef 100644 --- a/newrelic/core/error_collector.py +++ b/newrelic/core/error_collector.py @@ -14,5 +14,4 @@ from collections import namedtuple -TracedError = namedtuple('TracedError', - ['start_time','path','message','type','parameters']) +TracedError = namedtuple("TracedError", ["start_time", "path", "message", "type", "parameters"]) diff --git a/newrelic/core/external_node.py b/newrelic/core/external_node.py index 20e07e9a5f..ddda33976a 100644 --- a/newrelic/core/external_node.py +++ b/newrelic/core/external_node.py @@ -12,10 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - import urlparse -except ImportError: - import urllib.parse as urlparse +import urllib.parse as urlparse from collections import namedtuple @@ -49,8 +46,7 @@ def details(self): @property def name(self): - return 'External/%s/%s/%s' % ( - self.netloc, self.library, self.method or '') + return f"External/{self.netloc}/{self.library}/{self.method or ''}" @property def url_with_path(self): @@ -84,7 +80,7 @@ def netloc(self): if (scheme, port) in (('http', 80), ('https', 443)): port = None - netloc = port and ('%s:%s' % (hostname, port)) or hostname + netloc = port and (f'{hostname}:{port}') or hostname return netloc def time_metrics(self, stats, root, parent): @@ -116,7 +112,7 @@ def time_metrics(self, stats, root, parent): self.cross_process_id = None self.external_txn_name = None - name = 'External/%s/all' % netloc + name = f'External/{netloc}/all' yield TimeMetric(name=name, scope='', duration=self.duration, exclusive=self.exclusive) @@ -124,7 +120,7 @@ def time_metrics(self, stats, root, parent): if self.cross_process_id is None: method = self.method or '' - name = 'External/%s/%s/%s' % (netloc, self.library, method) + name = f'External/{netloc}/{self.library}/{method}' yield TimeMetric(name=name, scope='', duration=self.duration, exclusive=self.exclusive) @@ -133,8 +129,7 @@ def time_metrics(self, stats, root, parent): duration=self.duration, exclusive=self.exclusive) else: - name = 'ExternalTransaction/%s/%s/%s' % (netloc, - self.cross_process_id, self.external_txn_name) + name = f'ExternalTransaction/{netloc}/{self.cross_process_id}/{self.external_txn_name}' yield TimeMetric(name=name, scope='', duration=self.duration, exclusive=self.exclusive) @@ -142,7 +137,7 @@ def time_metrics(self, stats, root, parent): yield TimeMetric(name=name, scope=root.path, duration=self.duration, exclusive=self.exclusive) - name = 'ExternalApp/%s/%s/all' % (netloc, self.cross_process_id) + name = f'ExternalApp/{netloc}/{self.cross_process_id}/all' yield TimeMetric(name=name, scope='', duration=self.duration, exclusive=self.exclusive) @@ -154,11 +149,9 @@ def trace_node(self, stats, root, connections): method = self.method or '' if self.cross_process_id is None: - name = 'External/%s/%s/%s' % (netloc, self.library, method) + name = f'External/{netloc}/{self.library}/{method}' else: - name = 'ExternalTransaction/%s/%s/%s' % (netloc, - self.cross_process_id, - self.external_txn_name) + name = f'ExternalTransaction/{netloc}/{self.cross_process_id}/{self.external_txn_name}' name = root.string_table.cache(name) diff --git a/newrelic/core/function_node.py b/newrelic/core/function_node.py index 6acf7c81e0..f8df1976ce 100644 --- a/newrelic/core/function_node.py +++ b/newrelic/core/function_node.py @@ -19,7 +19,6 @@ from newrelic.core.node_mixin import GenericNodeMixin from newrelic.core.metric import TimeMetric -from newrelic.packages import six _FunctionNode = namedtuple('_FunctionNode', ['group', 'name', 'children', 'start_time', 'end_time', @@ -35,7 +34,7 @@ def time_metrics(self, stats, root, parent): """ - name = '%s/%s' % (self.group, self.name) + name = f'{self.group}/{self.name}' yield TimeMetric(name=name, scope='', duration=self.duration, exclusive=self.exclusive) @@ -60,7 +59,7 @@ def time_metrics(self, stats, root, parent): # own rollup categories. if self.rollup: - if isinstance(self.rollup, six.string_types): + if isinstance(self.rollup, str): rollups = [self.rollup] else: rollups = self.rollup @@ -89,7 +88,7 @@ def time_metrics(self, stats, root, parent): def trace_node(self, stats, root, connections): - name = '%s/%s' % (self.group, self.name) + name = f'{self.group}/{self.name}' name = root.string_table.cache(name) @@ -116,6 +115,6 @@ def span_event(self, *args, **kwargs): attrs = super(FunctionNode, self).span_event(*args, **kwargs) i_attrs = attrs[0] - i_attrs['name'] = '%s/%s' % (self.group, self.name) + i_attrs['name'] = f'{self.group}/{self.name}' return attrs diff --git a/newrelic/core/graphql_node.py b/newrelic/core/graphql_node.py index a32e185ee9..b767225841 100644 --- a/newrelic/core/graphql_node.py +++ b/newrelic/core/graphql_node.py @@ -60,7 +60,7 @@ def name(self): field_name = self.field_name or "" product = self.product - name = 'GraphQL/resolve/%s/%s' % (product, field_name) + name = f'GraphQL/resolve/{product}/{field_name}' return name @@ -74,7 +74,7 @@ def time_metrics(self, stats, root, parent): # Determine the scoped metric - field_resolver_metric_name = 'GraphQL/resolve/%s/%s' % (product, field_name) + field_resolver_metric_name = f'GraphQL/resolve/{product}/{field_name}' yield TimeMetric(name=field_resolver_metric_name, scope=root.path, duration=self.duration, exclusive=self.exclusive) @@ -97,8 +97,7 @@ def name(self): deepest_path = self.deepest_path product = self.product - name = 'GraphQL/operation/%s/%s/%s/%s' % (product, operation_type, - operation_name, deepest_path) + name = f'GraphQL/operation/{product}/{operation_type}/{operation_name}/{deepest_path}' return name @@ -115,8 +114,7 @@ def time_metrics(self, stats, root, parent): # Determine the scoped metric - operation_metric_name = 'GraphQL/operation/%s/%s/%s/%s' % (product, - operation_type, operation_name, deepest_path) + operation_metric_name = f'GraphQL/operation/{product}/{operation_type}/{operation_name}/{deepest_path}' scoped_metric_name = operation_metric_name @@ -128,20 +126,20 @@ def time_metrics(self, stats, root, parent): yield TimeMetric(name='GraphQL/all', scope='', duration=self.duration, exclusive=self.exclusive) - yield TimeMetric(name='GraphQL/%s/all' % product, scope='', + yield TimeMetric(name=f'GraphQL/{product}/all', scope='', duration=self.duration, exclusive=self.exclusive) if root.type == 'WebTransaction': yield TimeMetric(name='GraphQL/allWeb', scope='', duration=self.duration, exclusive=self.exclusive) - yield TimeMetric(name='GraphQL/%s/allWeb' % product, scope='', + yield TimeMetric(name=f'GraphQL/{product}/allWeb', scope='', duration=self.duration, exclusive=self.exclusive) else: yield TimeMetric(name='GraphQL/allOther', scope='', duration=self.duration, exclusive=self.exclusive) - yield TimeMetric(name='GraphQL/%s/allOther' % product, scope='', + yield TimeMetric(name=f'GraphQL/{product}/allOther', scope='', duration=self.duration, exclusive=self.exclusive) # Unscoped operation metric diff --git a/newrelic/core/graphql_utils.py b/newrelic/core/graphql_utils.py index f76228c5de..8f377cdcc8 100644 --- a/newrelic/core/graphql_utils.py +++ b/newrelic/core/graphql_utils.py @@ -19,7 +19,7 @@ from newrelic.core.database_utils import SQLStatement -class GraphQLStyle(object): +class GraphQLStyle(): """Helper class to initialize SQLStatement instances.""" quoting_style = "single+double" diff --git a/newrelic/core/internal_metrics.py b/newrelic/core/internal_metrics.py index 87452fce4a..0d3a413a49 100644 --- a/newrelic/core/internal_metrics.py +++ b/newrelic/core/internal_metrics.py @@ -17,10 +17,11 @@ import types import time import threading +import newrelic.common.object_wrapper _context = threading.local() -class InternalTrace(object): +class InternalTrace(): def __init__(self, name, metrics=None): self.name = name @@ -38,7 +39,7 @@ def __exit__(self, exc, value, tb): if self.metrics is not None: self.metrics.record_custom_metric(self.name, duration) -class InternalTraceWrapper(object): +class InternalTraceWrapper(): def __init__(self, wrapped, name): if type(wrapped) == type(()): @@ -67,7 +68,7 @@ def __call__(self, *args, **kwargs): with InternalTrace(self.__name, metrics): return self.__wrapped(*args, **kwargs) -class InternalTraceContext(object): +class InternalTraceContext(): def __init__(self, metrics): self.previous = None @@ -88,7 +89,7 @@ def decorator(wrapped): return decorator def wrap_internal_trace(module, object_path, name=None): - newrelic.api.object_wrapper.wrap_object(module, object_path, + newrelic.common.object_wrapper.wrap_object(module, object_path, InternalTraceWrapper, (name,)) def internal_metric(name, value): diff --git a/newrelic/core/loop_node.py b/newrelic/core/loop_node.py index 1bdbc7cf6c..151787ff19 100644 --- a/newrelic/core/loop_node.py +++ b/newrelic/core/loop_node.py @@ -47,7 +47,7 @@ def time_metrics(self, stats, root, parent): """ - name = 'EventLoop/Wait/%s' % self.name + name = f'EventLoop/Wait/{self.name}' yield TimeMetric(name=name, scope='', duration=self.duration, exclusive=self.duration) @@ -62,15 +62,15 @@ def time_metrics(self, stats, root, parent): exclusive=None) if root.type == 'WebTransaction': - yield TimeMetric(name=name + 'Web', scope='', + yield TimeMetric(name=f"{name}Web", scope='', duration=self.duration, exclusive=None) else: - yield TimeMetric(name=name + 'Other', scope='', + yield TimeMetric(name=f"{name}Other", scope='', duration=self.duration, exclusive=None) def trace_node(self, stats, root, connections): - name = 'EventLoop/Wait/%s' % self.name + name = f'EventLoop/Wait/{self.name}' name = root.string_table.cache(name) @@ -94,6 +94,6 @@ def span_event(self, *args, **kwargs): attrs = super(LoopNode, self).span_event(*args, **kwargs) i_attrs = attrs[0] - i_attrs['name'] = 'EventLoop/Wait/%s' % self.name + i_attrs['name'] = f'EventLoop/Wait/{self.name}' return attrs diff --git a/newrelic/core/memcache_node.py b/newrelic/core/memcache_node.py index 2f73609d0d..678a89c5ec 100644 --- a/newrelic/core/memcache_node.py +++ b/newrelic/core/memcache_node.py @@ -28,7 +28,7 @@ class MemcacheNode(_MemcacheNode, GenericNodeMixin): @property def name(self): - return 'Memcache/%s' % self.command + return f'Memcache/{self.command}' def time_metrics(self, stats, root, parent): """Return a generator yielding the timed metrics for this @@ -46,7 +46,7 @@ def time_metrics(self, stats, root, parent): yield TimeMetric(name='Memcache/allOther', scope='', duration=self.duration, exclusive=self.exclusive) - name = 'Memcache/%s' % self.command + name = f'Memcache/{self.command}' yield TimeMetric(name=name, scope='', duration=self.duration, exclusive=self.exclusive) diff --git a/newrelic/core/message_node.py b/newrelic/core/message_node.py index 02e431eb31..aa1fec992e 100644 --- a/newrelic/core/message_node.py +++ b/newrelic/core/message_node.py @@ -30,8 +30,7 @@ class MessageNode(_MessageNode, GenericNodeMixin): @property def name(self): - name = 'MessageBroker/%s/%s/%s/Named/%s' % (self.library, - self.destination_type, self.operation, self.destination_name) + name = f'MessageBroker/{self.library}/{self.destination_type}/{self.operation}/Named/{self.destination_name}' return name def time_metrics(self, stats, root, parent): diff --git a/newrelic/core/node_mixin.py b/newrelic/core/node_mixin.py index b7c5ed69c8..a45e6645d2 100644 --- a/newrelic/core/node_mixin.py +++ b/newrelic/core/node_mixin.py @@ -13,141 +13,118 @@ # limitations under the License. import newrelic.core.attribute as attribute +from newrelic.core.attribute_filter import DST_SPAN_EVENTS, DST_TRANSACTION_SEGMENTS -from newrelic.core.attribute_filter import (DST_SPAN_EVENTS, - DST_TRANSACTION_SEGMENTS) - -class GenericNodeMixin(object): +class GenericNodeMixin(): @property def processed_user_attributes(self): - if hasattr(self, '_processed_user_attributes'): + if hasattr(self, "_processed_user_attributes"): return self._processed_user_attributes self._processed_user_attributes = u_attrs = {} - user_attributes = getattr(self, 'user_attributes', u_attrs) + user_attributes = getattr(self, "user_attributes", u_attrs) for k, v in user_attributes.items(): k, v = attribute.process_user_attribute(k, v) - u_attrs[k] = v + # Only record the attribute if it passes processing. + # Failures return (None, None). + if k: + u_attrs[k] = v return u_attrs def get_trace_segment_params(self, settings, params=None): _params = attribute.resolve_agent_attributes( - self.agent_attributes, - settings.attribute_filter, - DST_TRANSACTION_SEGMENTS) + self.agent_attributes, settings.attribute_filter, DST_TRANSACTION_SEGMENTS + ) if params: _params.update(params) - _params.update(attribute.resolve_user_attributes( - self.processed_user_attributes, - settings.attribute_filter, - DST_TRANSACTION_SEGMENTS)) + _params.update( + attribute.resolve_user_attributes( + self.processed_user_attributes, settings.attribute_filter, DST_TRANSACTION_SEGMENTS + ) + ) - _params['exclusive_duration_millis'] = 1000.0 * self.exclusive + _params["exclusive_duration_millis"] = 1000.0 * self.exclusive return _params - def span_event( - self, - settings, - base_attrs=None, - parent_guid=None, - attr_class=dict): + def span_event(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): i_attrs = base_attrs and base_attrs.copy() or attr_class() - i_attrs['type'] = 'Span' - i_attrs['name'] = self.name - i_attrs['guid'] = self.guid - i_attrs['timestamp'] = int(self.start_time * 1000) - i_attrs['duration'] = self.duration - i_attrs['category'] = 'generic' + i_attrs["type"] = "Span" + i_attrs["name"] = self.name + i_attrs["guid"] = self.guid + i_attrs["timestamp"] = int(self.start_time * 1000) + i_attrs["duration"] = self.duration + i_attrs["category"] = "generic" if parent_guid: - i_attrs['parentId'] = parent_guid + i_attrs["parentId"] = parent_guid a_attrs = attribute.resolve_agent_attributes( - self.agent_attributes, - settings.attribute_filter, - DST_SPAN_EVENTS, - attr_class=attr_class) + self.agent_attributes, settings.attribute_filter, DST_SPAN_EVENTS, attr_class=attr_class + ) u_attrs = attribute.resolve_user_attributes( - self.processed_user_attributes, - settings.attribute_filter, - DST_SPAN_EVENTS, - attr_class=attr_class) + self.processed_user_attributes, settings.attribute_filter, DST_SPAN_EVENTS, attr_class=attr_class + ) # intrinsics, user attrs, agent attrs return [i_attrs, u_attrs, a_attrs] - def span_events(self, - settings, base_attrs=None, parent_guid=None, attr_class=dict): - - yield self.span_event( - settings, - base_attrs=base_attrs, - parent_guid=parent_guid, - attr_class=attr_class) + def span_events(self, settings, base_attrs=None, parent_guid=None, attr_class=dict): + yield self.span_event(settings, base_attrs=base_attrs, parent_guid=parent_guid, attr_class=attr_class) for child in self.children: for event in child.span_events( - settings, - base_attrs=base_attrs, - parent_guid=self.guid, - attr_class=attr_class): + settings, base_attrs=base_attrs, parent_guid=self.guid, attr_class=attr_class + ): yield event class DatastoreNodeMixin(GenericNodeMixin): - @property def name(self): product = self.product target = self.target - operation = self.operation or 'other' + operation = self.operation or "other" if target: - name = 'Datastore/statement/%s/%s/%s' % (product, target, - operation) + name = f"Datastore/statement/{product}/{target}/{operation}" else: - name = 'Datastore/operation/%s/%s' % (product, operation) + name = f"Datastore/operation/{product}/{operation}" return name @property def db_instance(self): - if hasattr(self, '_db_instance'): + if hasattr(self, "_db_instance"): return self._db_instance db_instance_attr = None if self.database_name: - _, db_instance_attr = attribute.process_user_attribute( - 'db.instance', self.database_name) + _, db_instance_attr = attribute.process_user_attribute("db.instance", self.database_name) self._db_instance = db_instance_attr return db_instance_attr def span_event(self, *args, **kwargs): - self.agent_attributes['db.instance'] = self.db_instance + self.agent_attributes["db.instance"] = self.db_instance attrs = super(DatastoreNodeMixin, self).span_event(*args, **kwargs) i_attrs = attrs[0] a_attrs = attrs[2] - i_attrs['category'] = 'datastore' - i_attrs['component'] = self.product - i_attrs['span.kind'] = 'client' + i_attrs["category"] = "datastore" + i_attrs["component"] = self.product + i_attrs["span.kind"] = "client" if self.instance_hostname: - _, a_attrs['peer.hostname'] = attribute.process_user_attribute( - 'peer.hostname', self.instance_hostname) + _, a_attrs["peer.hostname"] = attribute.process_user_attribute("peer.hostname", self.instance_hostname) else: - a_attrs['peer.hostname'] = 'Unknown' + a_attrs["peer.hostname"] = "Unknown" - peer_address = '%s:%s' % ( - self.instance_hostname or 'Unknown', - self.port_path_or_id or 'Unknown') + peer_address = f"{self.instance_hostname or 'Unknown'}:{self.port_path_or_id or 'Unknown'}" - _, a_attrs['peer.address'] = attribute.process_user_attribute( - 'peer.address', peer_address) + _, a_attrs["peer.address"] = attribute.process_user_attribute("peer.address", peer_address) return attrs diff --git a/newrelic/core/otlp_utils.py b/newrelic/core/otlp_utils.py new file mode 100644 index 0000000000..4c9c57f8fb --- /dev/null +++ b/newrelic/core/otlp_utils.py @@ -0,0 +1,268 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +This module provides common utilities for interacting with OTLP protocol buffers. + +The serialization implemented here attempts to use protobuf as an encoding, but falls +back to JSON when encoutering exceptions unless the content type is explicitly set in debug settings. +""" + +import logging + +from newrelic.api.time_trace import get_service_linking_metadata +from newrelic.common.encoding_utils import json_encode +from newrelic.core.config import global_settings +from newrelic.core.stats_engine import CountStats, TimeStats + +_logger = logging.getLogger(__name__) + +_settings = global_settings() +otlp_content_setting = _settings.debug.otlp_content_encoding +if not otlp_content_setting or otlp_content_setting == "protobuf": + try: + from newrelic.packages.opentelemetry_proto.common_pb2 import AnyValue, KeyValue + from newrelic.packages.opentelemetry_proto.logs_pb2 import ( + LogsData, + ResourceLogs, + ScopeLogs, + ) + from newrelic.packages.opentelemetry_proto.metrics_pb2 import ( + AggregationTemporality, + Metric, + MetricsData, + NumberDataPoint, + ResourceMetrics, + ScopeMetrics, + Sum, + Summary, + SummaryDataPoint, + ) + from newrelic.packages.opentelemetry_proto.resource_pb2 import Resource + + ValueAtQuantile = SummaryDataPoint.ValueAtQuantile + AGGREGATION_TEMPORALITY_DELTA = AggregationTemporality.AGGREGATION_TEMPORALITY_DELTA + OTLP_CONTENT_TYPE = "application/x-protobuf" + + otlp_content_setting = "protobuf" # Explicitly set to overwrite None values + except Exception: + if otlp_content_setting == "protobuf": + raise # Reraise exception if content type explicitly set + # Fallback to JSON + otlp_content_setting = "json" + + +if otlp_content_setting == "json": + AnyValue = dict + KeyValue = dict + Metric = dict + MetricsData = dict + NumberDataPoint = dict + Resource = dict + ResourceMetrics = dict + ScopeMetrics = dict + Sum = dict + Summary = dict + SummaryDataPoint = dict + ValueAtQuantile = dict + ResourceLogs = dict + ScopeLogs = dict + LogsData = dict + + AGGREGATION_TEMPORALITY_DELTA = 1 + OTLP_CONTENT_TYPE = "application/json" + + +def otlp_encode(payload): + if type(payload) is dict: # pylint: disable=C0123 + _logger.warning( + "Using OTLP integration while protobuf is not installed. This may result in larger payload sizes and data loss." + ) + return json_encode(payload).encode("utf-8") + return payload.SerializeToString() + + +def create_key_value(key, value): + if isinstance(value, bool): + return KeyValue(key=key, value=AnyValue(bool_value=value)) + elif isinstance(value, int): + return KeyValue(key=key, value=AnyValue(int_value=value)) + elif isinstance(value, float): + return KeyValue(key=key, value=AnyValue(double_value=value)) + elif isinstance(value, str): + return KeyValue(key=key, value=AnyValue(string_value=value)) + # Technically AnyValue accepts array, kvlist, and bytes however, since + # those are not valid custom attribute types according to our api spec, + # we will not bother to support them here either. + else: + _logger.warning(f"Unsupported attribute value type {key}: {value}.") + + +def create_key_values_from_iterable(iterable): + if not iterable: + return None + elif isinstance(iterable, dict): + iterable = iterable.items() + + # The create_key_value list may return None if the value is an unsupported type + # so filter None values out before returning. + return list( + filter( + lambda i: i is not None, + (create_key_value(key, value) for key, value in iterable), + ) + ) + + +def create_resource(attributes=None, attach_apm_entity=True): + attributes = attributes or {"instrumentation.provider": "newrelic-opentelemetry-python-ml"} + if attach_apm_entity: + metadata = get_service_linking_metadata() + attributes.update(metadata) + return Resource(attributes=create_key_values_from_iterable(attributes)) + + +def TimeStats_to_otlp_data_point(self, start_time, end_time, attributes=None): + data = SummaryDataPoint( + time_unix_nano=int(end_time * 1e9), # Time of current harvest + start_time_unix_nano=int(start_time * 1e9), # Time of last harvest + attributes=attributes, + count=int(self[0]), + sum=float(self[1]), + quantile_values=[ + ValueAtQuantile(quantile=0.0, value=float(self[3])), # Min Value + ValueAtQuantile(quantile=1.0, value=float(self[4])), # Max Value + ], + ) + return data + + +def CountStats_to_otlp_data_point(self, start_time, end_time, attributes=None): + data = NumberDataPoint( + time_unix_nano=int(end_time * 1e9), # Time of current harvest + start_time_unix_nano=int(start_time * 1e9), # Time of last harvest + attributes=attributes, + as_int=int(self[0]), + ) + return data + + +def stats_to_otlp_metrics(metric_data, start_time, end_time): + """ + Generator producing protos for Summary and Sum metrics, for CountStats and TimeStats respectively. + + Individual Metric protos must be entirely one type of metric data point. For mixed metric types we have to + separate the types and report multiple metrics, one for each type. + """ + for name, metric_container in metric_data: + # Types are checked here using type() instead of isinstance, as CountStats is a subclass of TimeStats. + # Imporperly checking with isinstance will lead to count metrics being encoded and reported twice. + if any(type(metric) is CountStats for metric in metric_container.values()): # pylint: disable=C0123 + # Metric contains Sum metric data points. + yield Metric( + name=name, + sum=Sum( + aggregation_temporality=AGGREGATION_TEMPORALITY_DELTA, + is_monotonic=True, + data_points=[ + CountStats_to_otlp_data_point( + value, + start_time=start_time, + end_time=end_time, + attributes=create_key_values_from_iterable(tags), + ) + for tags, value in metric_container.items() + if type(value) is CountStats # pylint: disable=C0123 + ], + ), + ) + if any(type(metric) is TimeStats for metric in metric_container.values()): # pylint: disable=C0123 + # Metric contains Summary metric data points. + yield Metric( + name=name, + summary=Summary( + data_points=[ + TimeStats_to_otlp_data_point( + value, + start_time=start_time, + end_time=end_time, + attributes=create_key_values_from_iterable(tags), + ) + for tags, value in metric_container.items() + if type(value) is TimeStats # pylint: disable=C0123 + ] + ), + ) + + +def encode_metric_data(metric_data, start_time, end_time, resource=None, scope=None): + resource = resource or create_resource(attach_apm_entity=False) + return MetricsData( + resource_metrics=[ + ResourceMetrics( + resource=resource, + scope_metrics=[ + ScopeMetrics( + scope=scope, + metrics=list(stats_to_otlp_metrics(metric_data, start_time, end_time)), + ) + ], + ) + ] + ) + + +def encode_ml_event_data(custom_event_data, agent_run_id): + # An InferenceEvent is attached to a separate ML Model entity instead + # of the APM entity. + ml_inference_events = [] + ml_apm_events = [] + for event in custom_event_data: + event_info, event_attrs = event + event_type = event_info["type"] + event_attrs.update( + { + "real_agent_id": agent_run_id, + "event.domain": "newrelic.ml_events", + "event.name": event_type, + } + ) + ml_attrs = create_key_values_from_iterable(event_attrs) + unix_nano_timestamp = event_info["timestamp"] * 1e6 + if event_type == "InferenceEvent": + ml_inference_events.append( + { + "time_unix_nano": int(unix_nano_timestamp), + "attributes": ml_attrs, + } + ) + else: + ml_apm_events.append( + { + "time_unix_nano": int(unix_nano_timestamp), + "attributes": ml_attrs, + } + ) + + resource_logs = [] + if ml_inference_events: + inference_resource = create_resource(attach_apm_entity=False) + resource_logs.append( + ResourceLogs(resource=inference_resource, scope_logs=[ScopeLogs(log_records=ml_inference_events)]) + ) + if ml_apm_events: + apm_resource = create_resource() + resource_logs.append(ResourceLogs(resource=apm_resource, scope_logs=[ScopeLogs(log_records=ml_apm_events)])) + + return LogsData(resource_logs=resource_logs) diff --git a/newrelic/core/profile_sessions.py b/newrelic/core/profile_sessions.py index 663e90fe4e..5d71fde34c 100644 --- a/newrelic/core/profile_sessions.py +++ b/newrelic/core/profile_sessions.py @@ -19,24 +19,20 @@ import time import zlib from collections import defaultdict, deque +from sys import intern import newrelic -import newrelic.packages.six as six from newrelic.common.encoding_utils import json_encode from newrelic.core.config import global_settings from newrelic.core.trace_cache import trace_cache -try: - from sys import intern -except ImportError: - pass _logger = logging.getLogger(__name__) -AGENT_PACKAGE_DIRECTORY = os.path.dirname(newrelic.__file__) + "/" +AGENT_PACKAGE_DIRECTORY = os.path.dirname(newrelic.__file__) + os.sep -class SessionState(object): +class SessionState(): RUNNING = 1 FINISHED = 2 @@ -117,7 +113,7 @@ def collect_stack_traces(include_nr_threads=False): yield thread_category, stack_trace -class ProfileSessionManager(object): +class ProfileSessionManager(): """Singleton class that manages multiple profile sessions. Do NOT instantiate directly from this class. Instead use profile_session_manager() @@ -286,7 +282,7 @@ def shutdown(self, app_name): return True -class ProfileSession(object): +class ProfileSession(): def __init__(self, profile_id, stop_time): self.profile_id = profile_id self.start_time_s = time.time() @@ -393,7 +389,7 @@ def profile_data(self): flat_tree = {} thread_count = 0 - for category, bucket in six.iteritems(self.call_buckets): + for category, bucket in self.call_buckets.items(): # Only flatten buckets that have data in them. No need to send # empty buckets. @@ -414,10 +410,7 @@ def profile_data(self): level = settings.agent_limits.data_compression_level level = level or zlib.Z_DEFAULT_COMPRESSION - encoded_tree = base64.standard_b64encode(zlib.compress(six.b(json_call_tree), level)) - - if six.PY3: - encoded_tree = encoded_tree.decode("Latin-1") + encoded_tree = base64.standard_b64encode(zlib.compress(json_call_tree.encode("latin-1"), level)).decode("Latin-1") profile = [ [ @@ -438,7 +431,7 @@ def profile_data(self): return profile -class CallTree(object): +class CallTree(): def __init__(self, method_data, call_count=0, depth=1): self.method_data = method_data self.call_count = call_count @@ -456,9 +449,9 @@ def flatten(self): # are labeled with an @ sign in the second element of the tuple. if func_line == exec_line: - method_data = (filename, "@%s#%s" % (func_name, func_line), exec_line) + method_data = (filename, f"@{func_name}#{func_line}", exec_line) else: - method_data = (filename, "%s#%s" % (func_name, func_line), exec_line) + method_data = (filename, f"{func_name}#{func_line}", exec_line) return [method_data, self.call_count, 0, [x.flatten() for x in self.children.values() if not x.ignore]] diff --git a/newrelic/core/root_node.py b/newrelic/core/root_node.py index 63acf61742..2cc84cf338 100644 --- a/newrelic/core/root_node.py +++ b/newrelic/core/root_node.py @@ -16,30 +16,39 @@ import newrelic.core.trace_node from newrelic.core.node_mixin import GenericNodeMixin -from newrelic.core.attribute import resolve_user_attributes -from newrelic.packages import six - -_RootNode = namedtuple('_RootNode', - ['name', 'children', 'start_time', 'end_time', 'exclusive', - 'duration', 'guid', 'agent_attributes', 'user_attributes', - 'path', 'trusted_parent_span', 'tracing_vendors',]) +_RootNode = namedtuple( + "_RootNode", + [ + "name", + "children", + "start_time", + "end_time", + "exclusive", + "duration", + "guid", + "agent_attributes", + "user_attributes", + "path", + "trusted_parent_span", + "tracing_vendors", + ], +) class RootNode(_RootNode, GenericNodeMixin): def span_event(self, *args, **kwargs): span = super(RootNode, self).span_event(*args, **kwargs) i_attrs = span[0] - i_attrs['transaction.name'] = self.path - i_attrs['nr.entryPoint'] = True + i_attrs["transaction.name"] = self.path + i_attrs["nr.entryPoint"] = True if self.trusted_parent_span: - i_attrs['trustedParentId'] = self.trusted_parent_span + i_attrs["trustedParentId"] = self.trusted_parent_span if self.tracing_vendors: - i_attrs['tracingVendors'] = self.tracing_vendors + i_attrs["tracingVendors"] = self.tracing_vendors return span def trace_node(self, stats, root, connections): - name = self.path start_time = newrelic.core.trace_node.node_start_time(root, self) @@ -57,9 +66,5 @@ def trace_node(self, stats, root, connections): params = self.get_trace_segment_params(root.settings) return newrelic.core.trace_node.TraceNode( - start_time=start_time, - end_time=end_time, - name=name, - params=params, - children=children, - label=None) + start_time=start_time, end_time=end_time, name=name, params=params, children=children, label=None + ) diff --git a/newrelic/core/rules_engine.py b/newrelic/core/rules_engine.py index fccc5e5e1f..4b40c984b2 100644 --- a/newrelic/core/rules_engine.py +++ b/newrelic/core/rules_engine.py @@ -22,6 +22,27 @@ class NormalizationRule(_NormalizationRule): + def __new__( + cls, + match_expression="", + replacement="", + ignore=False, + eval_order=0, + terminate_chain=False, + each_segment=False, + replace_all=False, + ): + return _NormalizationRule.__new__( + cls, + match_expression=match_expression, + replacement=replacement, + ignore=ignore, + eval_order=eval_order, + terminate_chain=terminate_chain, + each_segment=each_segment, + replace_all=replace_all, + ) + def __init__(self, *args, **kwargs): self.match_expression_re = re.compile(self.match_expression, re.IGNORECASE) @@ -33,7 +54,7 @@ def apply(self, string): return self.match_expression_re.subn(self.replacement, string, count) -class RulesEngine(object): +class RulesEngine(): def __init__(self, rules): self.__rules = [] @@ -105,7 +126,7 @@ def normalize(self, string): return (final_string, ignore) -class SegmentCollapseEngine(object): +class SegmentCollapseEngine(): """Segment names in transaction name are collapsed using the rules from the data collector. The collector sends a prefix and list of allowlist terms associated with that prefix. If a transaction name @@ -153,7 +174,7 @@ def __init__(self, rules): # should always return prefixes and term strings as Unicode. choices = "|".join([re.escape(x) for x in prefixes]) - pattern = "^(%s)/(.+)$" % choices + pattern = f"^({choices})/(.+)$" self.prefixes = re.compile(pattern) @@ -207,4 +228,4 @@ def normalize(self, txn_name): result = [x if x in allowlist_terms else "*" for x in segments] result = self.COLLAPSE_STAR_RE.sub("\\1", "/".join(result)) - return "/".join((prefix, result)), False + return f"{prefix}/{result}", False diff --git a/newrelic/core/solr_node.py b/newrelic/core/solr_node.py index 849ca535af..9ffcd29a69 100644 --- a/newrelic/core/solr_node.py +++ b/newrelic/core/solr_node.py @@ -29,7 +29,7 @@ class SolrNode(_SolrNode, GenericNodeMixin): @property def name(self): - return 'SolrClient/%s/%s' % (self.library, self.command) + return f'SolrClient/{self.library}/{self.command}' def time_metrics(self, stats, root, parent): """Return a generator yielding the timed metrics for this @@ -46,7 +46,7 @@ def time_metrics(self, stats, root, parent): yield TimeMetric(name='Solr/allOther', scope='', duration=self.duration, exclusive=self.exclusive) - name = 'Solr/%s' % self.command + name = f'Solr/{self.command}' yield TimeMetric(name=name, scope='', duration=self.duration, exclusive=self.exclusive) diff --git a/newrelic/core/stack_trace.py b/newrelic/core/stack_trace.py index c7e378e4a2..1ab1f96c4e 100644 --- a/newrelic/core/stack_trace.py +++ b/newrelic/core/stack_trace.py @@ -27,8 +27,7 @@ def _format_stack_trace(frames): result = ['Traceback (most recent call last):'] - result.extend(['File "{source}", line {line}, in {name}'.format(**d) - for d in frames]) + result.extend([f'File "{f["source"]}", line {f["line"]}, in {f["name"]}' for f in frames]) return result def _extract_stack(f, skip, limit): diff --git a/newrelic/core/stats_engine.py b/newrelic/core/stats_engine.py index 203e3e7960..e77647d9d8 100644 --- a/newrelic/core/stats_engine.py +++ b/newrelic/core/stats_engine.py @@ -31,20 +31,21 @@ import zlib from heapq import heapify, heapreplace -import newrelic.packages.six as six from newrelic.api.settings import STRIP_EXCEPTION_MESSAGE from newrelic.api.time_trace import get_linking_metadata from newrelic.common.encoding_utils import json_encode +from newrelic.common.metric_utils import create_metric_identity from newrelic.common.object_names import parse_exc_info from newrelic.common.streaming_utils import StreamBuffer from newrelic.core.attribute import ( MAX_LOG_MESSAGE_LENGTH, create_agent_attributes, - create_user_attributes, + create_attributes, process_user_attribute, + resolve_logging_context_attributes, truncate, ) -from newrelic.core.attribute_filter import DST_ERROR_COLLECTOR +from newrelic.core.attribute_filter import DST_ALL, DST_ERROR_COLLECTOR from newrelic.core.code_level_metrics import extract_code_from_traceback from newrelic.core.config import is_expected_error, should_ignore_error from newrelic.core.database_utils import explain_plan @@ -61,7 +62,7 @@ "reset_synthetics_events", ), "span_event_data": ("reset_span_events",), - "custom_event_data": ("reset_custom_events",), + "custom_event_data": ("reset_custom_events", "reset_ml_events"), "error_event_data": ("reset_error_events",), "log_event_data": ("reset_log_events",), } @@ -180,6 +181,11 @@ def merge_custom_metric(self, value): self.merge_raw_time_metric(value) + def merge_dimensional_metric(self, value): + """Merge data value.""" + + self.merge_raw_time_metric(value) + class CountStats(TimeStats): def merge_stats(self, other): @@ -189,7 +195,7 @@ def merge_raw_time_metric(self, duration, exclusive=None): pass -class CustomMetrics(object): +class CustomMetrics(): """Table for collection a set of value metrics.""" @@ -225,7 +231,7 @@ def metrics(self): """ - return six.iteritems(self.__stats_table) + return self.__stats_table.items() def reset_metric_stats(self): """Resets the accumulated statistics back to initial state for @@ -235,6 +241,99 @@ def reset_metric_stats(self): self.__stats_table = {} +class DimensionalMetrics(): + + """Nested dictionary table for collecting a set of metrics broken down by tags.""" + + def __init__(self): + self.__stats_table = {} + + def __contains__(self, key): + if isinstance(key, tuple): + if not isinstance(key[1], frozenset): + # Convert tags dict to a frozen set for proper comparisons + name, tags = create_metric_identity(*key) + else: + name, tags = key + + # Check that both metric name and tags are already present. + stats_container = self.__stats_table.get(name) + return stats_container and tags in stats_container + else: + # Only look for metric name + return key in self.__stats_table + + def record_dimensional_metric(self, name, value, tags=None): + """Record a single value metric, merging the data with any data + from prior value metrics with the same name and tags. + """ + name, tags = create_metric_identity(name, tags) + + if isinstance(value, dict): + if len(value) == 1 and "count" in value: + new_stats = CountStats(call_count=value["count"]) + else: + new_stats = TimeStats(*c2t(**value)) + else: + new_stats = TimeStats(1, value, value, value, value, value**2) + + stats_container = self.__stats_table.get(name) + if stats_container is None: + # No existing metrics with this name. Set up new stats container. + self.__stats_table[name] = {tags: new_stats} + else: + # Existing metric container found. + stats = stats_container.get(tags) + if stats is None: + # No data points for this set of tags. Add new data. + stats_container[tags] = new_stats + else: + # Existing data points found, merge stats. + stats.merge_stats(new_stats) + + return (name, tags) + + def metrics(self): + """Returns an iterator over the set of value metrics. + The items returned are a dictionary of tags for each metric value. + Metric values are each a tuple consisting of the metric name and accumulated + stats for the metric. + """ + + return self.__stats_table.items() + + def metrics_count(self): + """Returns a count of the number of unique metrics currently + recorded for apdex, time and value metrics. + """ + + return sum(len(metric) for metric in self.__stats_table.values()) + + def reset_metric_stats(self): + """Resets the accumulated statistics back to initial state for + metric data. + """ + self.__stats_table = {} + + def get(self, key, default=None): + return self.__stats_table.get(key, default) + + def __setitem__(self, key, value): + self.__stats_table[key] = value + + def __getitem__(self, key): + return self.__stats_table[key] + + def __str__(self): + return str(self.__stats_table) + + def __repr__(self): + return f"{__class__.__name__}({repr(self.__stats_table)})" + + def items(self): + return self.metrics() + + class SlowSqlStats(list): def __init__(self): super(SlowSqlStats, self).__init__([0, 0, 0, 0, None]) @@ -278,7 +377,7 @@ def merge_slow_sql_node(self, node): self[0] += 1 -class SampledDataSet(object): +class SampledDataSet(): def __init__(self, capacity=100): self.pq = [] self.heap = False @@ -402,7 +501,7 @@ def merge(self, other_data_set): self.num_seen += other_data_set.num_seen - other_data_set.num_samples -class StatsEngine(object): +class StatsEngine(): """The stats engine object holds the accumulated transactions metrics, details of errors and slow transactions. There should be one instance @@ -433,9 +532,11 @@ class StatsEngine(object): def __init__(self): self.__settings = None self.__stats_table = {} + self.__dimensional_stats_table = DimensionalMetrics() self._transaction_events = SampledDataSet() self._error_events = SampledDataSet() self._custom_events = SampledDataSet() + self._ml_events = SampledDataSet() self._span_events = SampledDataSet() self._log_events = SampledDataSet() self._span_stream = None @@ -456,6 +557,10 @@ def settings(self): def stats_table(self): return self.__stats_table + @property + def dimensional_stats_table(self): + return self.__dimensional_stats_table + @property def transaction_events(self): return self._transaction_events @@ -464,6 +569,10 @@ def transaction_events(self): def custom_events(self): return self._custom_events + @property + def ml_events(self): + return self._ml_events + @property def span_events(self): return self._span_events @@ -494,7 +603,7 @@ def metrics_count(self): """ - return len(self.__stats_table) + return len(self.__stats_table) + self.__dimensional_stats_table.metrics_count() def record_apdex_metric(self, metric): """Record a single apdex metric, merging the data with any data @@ -615,6 +724,11 @@ def notice_error(self, error=None, attributes=None, expected=None, ignore=None, module, name, fullnames, message_raw = parse_exc_info(error) fullname = fullnames[0] + # In the case case of JSON formatting for OpenAI models + # this will result in a "cleaner" message format + if getattr(value, "_nr_message", None): + message_raw = value._nr_message + # Check to see if we need to strip the message before recording it. if settings.strip_exception_messages.enabled and fullname not in settings.strip_exception_messages.allowlist: @@ -714,8 +828,7 @@ def notice_error(self, error=None, attributes=None, expected=None, ignore=None, ) custom_attributes = {} - user_attributes = create_user_attributes(custom_attributes, settings.attribute_filter) - + user_attributes = create_attributes(custom_attributes, DST_ALL, settings.attribute_filter) # Extract additional details about the exception as agent attributes agent_attributes = {} @@ -728,28 +841,36 @@ def notice_error(self, error=None, attributes=None, expected=None, ignore=None, error_group_name = None try: # Call callback to obtain error group name - error_group_name_raw = settings.error_collector.error_group_callback(value, { - "traceback": tb, - "error.class": exc, - "error.message": message_raw, - "error.expected": is_expected, - "custom_params": attributes, - # Transaction specific items should be set to None - "transactionName": None, - "response.status": None, - "request.method": None, - "request.uri": None, - }) + error_group_name_raw = settings.error_collector.error_group_callback( + value, + { + "traceback": tb, + "error.class": exc, + "error.message": message_raw, + "error.expected": is_expected, + "custom_params": attributes, + # Transaction specific items should be set to None + "transactionName": None, + "response.status": None, + "request.method": None, + "request.uri": None, + }, + ) if error_group_name_raw: _, error_group_name = process_user_attribute("error.group.name", error_group_name_raw) - if error_group_name is None or not isinstance(error_group_name, six.string_types): - raise ValueError("Invalid attribute value for error.group.name. Expected string, got: %s" % repr(error_group_name_raw)) + if error_group_name is None or not isinstance(error_group_name, str): + raise ValueError( + f"Invalid attribute value for error.group.name. Expected string, got: {repr(error_group_name_raw)}" + ) else: agent_attributes["error.group.name"] = error_group_name except Exception: - _logger.error("Encountered error when calling error group callback:\n%s", "".join(traceback.format_exception(*sys.exc_info()))) - + _logger.error( + "Encountered error when calling error group callback:\n%s", + "".join(traceback.format_exception(*sys.exc_info())), + ) + agent_attributes = create_agent_attributes(agent_attributes, settings.attribute_filter) # Record the exception details. @@ -774,9 +895,13 @@ def notice_error(self, error=None, attributes=None, expected=None, ignore=None, for attr in agent_attributes: if attr.destinations & DST_ERROR_COLLECTOR: attributes["agentAttributes"][attr.name] = attr.value - + error_details = TracedError( - start_time=time.time(), path="Exception", message=message, type=fullname, parameters=attributes + start_time=time.time(), + path="Exception", + message=message, + type=fullname, + parameters=attributes, ) # Save this error as a trace and an event. @@ -829,6 +954,15 @@ def record_custom_event(self, event): if settings.collect_custom_events and settings.custom_insights_events.enabled: self._custom_events.add(event) + def record_ml_event(self, event): + settings = self.__settings + + if not settings: + return + + if settings.ml_insights_events.enabled: + self._ml_events.add(event) + def record_custom_metric(self, name, value): """Record a single value metric, merging the data with any data from prior value metrics with the same name. @@ -865,6 +999,28 @@ def record_custom_metrics(self, metrics): for name, value in metrics: self.record_custom_metric(name, value) + def record_dimensional_metric(self, name, value, tags=None): + """Record a single value metric, merging the data with any data + from prior value metrics with the same name and tags. + """ + return self.__dimensional_stats_table.record_dimensional_metric(name, value, tags) + + def record_dimensional_metrics(self, metrics): + """Record the value metrics supplied by the iterable, merging + the data with any data from prior value metrics with the same + name. + + """ + + if not self.__settings: + return + + for metric in metrics: + name, value = metric[:2] + tags = metric[2] if len(metric) >= 3 else None + + self.record_dimensional_metric(name, value, tags) + def record_slow_sql_node(self, node): """Record a single sql metric, merging the data with any data from prior sql metrics for the same sql key. @@ -975,6 +1131,8 @@ def record_transaction(self, transaction): self.merge_custom_metrics(transaction.custom_metrics.metrics()) + self.merge_dimensional_metrics(transaction.dimensional_metrics.metrics()) + self.record_time_metrics(transaction.time_metrics(self)) # Capture any errors if error collection is enabled. @@ -1042,6 +1200,11 @@ def record_transaction(self, transaction): if settings.collect_custom_events and settings.custom_insights_events.enabled: self.custom_events.merge(transaction.custom_events) + # Merge in machine learning events + + if settings.ml_insights_events.enabled: + self.ml_events.merge(transaction.ml_events) + # Merge in span events if settings.distributed_tracing.enabled and settings.span_events.enabled and settings.collect_span_events: @@ -1063,7 +1226,7 @@ def record_transaction(self, transaction): ): self._log_events.merge(transaction.log_events, priority=transaction.priority) - def record_log_event(self, message, level=None, timestamp=None, priority=None): + def record_log_event(self, message, level=None, timestamp=None, attributes=None, priority=None): settings = self.__settings if not ( settings @@ -1076,18 +1239,62 @@ def record_log_event(self, message, level=None, timestamp=None, priority=None): timestamp = timestamp if timestamp is not None else time.time() level = str(level) if level is not None else "UNKNOWN" + context_attributes = attributes # Name reassigned for clarity - if not message or message.isspace(): - _logger.debug("record_log_event called where message was missing. No log event will be sent.") - return + # Unpack message and attributes from dict inputs + if isinstance(message, dict): + message_attributes = {k: v for k, v in message.items() if k != "message"} + message = message.get("message", "") + else: + message_attributes = None + + if message is not None: + # Coerce message into a string type + if not isinstance(message, str): + try: + message = str(message) + except Exception: + # Exit early for invalid message type after unpacking + _logger.debug( + "record_log_event called where message could not be converted to a string type. No log event will be sent." + ) + return + + # Truncate the now unpacked and string converted message + message = truncate(message, MAX_LOG_MESSAGE_LENGTH) + + # Collect attributes from linking metadata, context data, and message attributes + collected_attributes = {} + if settings and settings.application_logging.forwarding.context_data.enabled: + if context_attributes: + context_attributes = resolve_logging_context_attributes( + context_attributes, settings.attribute_filter, "context." + ) + if context_attributes: + collected_attributes.update(context_attributes) + + if message_attributes: + message_attributes = resolve_logging_context_attributes( + message_attributes, settings.attribute_filter, "message." + ) + if message_attributes: + collected_attributes.update(message_attributes) - message = truncate(message, MAX_LOG_MESSAGE_LENGTH) + # Exit early if no message or attributes found after filtering + if (not message or message.isspace()) and not context_attributes and not message_attributes: + _logger.debug( + "record_log_event called where no message and no attributes were found. No log event will be sent." + ) + return + + # Finally, add in linking attributes after checking that there is a valid message or at least 1 attribute + collected_attributes.update(get_linking_metadata()) event = LogEventNode( timestamp=timestamp, level=level, message=message, - attributes=get_linking_metadata(), + attributes=collected_attributes, ) if priority is None: @@ -1124,12 +1331,16 @@ def metric_data(self, normalizer=None): _logger.info( "Raw metric data for harvest of %r is %r.", self.__settings.app_name, - list(six.iteritems(self.__stats_table)), + list(self.__stats_table.items()), ) if normalizer is not None: - for key, value in six.iteritems(self.__stats_table): - key = (normalizer(key[0])[0], key[1]) + for key, value in self.__stats_table.items(): + normalized_name, ignored = normalizer(key[0]) + if ignored: + continue + + key = (normalized_name, key[1]) stats = normalized_stats.get(key) if stats is None: normalized_stats[key] = copy.copy(value) @@ -1142,10 +1353,10 @@ def metric_data(self, normalizer=None): _logger.info( "Normalized metric data for harvest of %r is %r.", self.__settings.app_name, - list(six.iteritems(normalized_stats)), + list(normalized_stats.items()), ) - for key, value in six.iteritems(normalized_stats): + for key, value in normalized_stats.items(): key = dict(name=key[0], scope=key[1]) result.append((key, value)) @@ -1159,6 +1370,66 @@ def metric_data_count(self): return len(self.__stats_table) + def dimensional_metric_data(self, normalizer=None): + """Returns a list containing the low level metric data for + sending to the core application pertaining to the reporting + period. This consists of tuple pairs where first is dictionary + with name and scope keys with corresponding values, or integer + identifier if metric had an entry in dictionary mapping metric + (name, tags) as supplied from core application. The second is + the list of accumulated metric data, the list always being of + length 6. + + """ + + if not self.__settings: + return [] + + result = [] + normalized_stats = {} + + # Metric Renaming and Re-Aggregation. After applying the metric + # renaming rules, the metrics are re-aggregated to collapse the + # metrics with same names after the renaming. + + if self.__settings.debug.log_raw_metric_data: + _logger.info( + "Raw dimensional metric data for harvest of %r is %r.", + self.__settings.app_name, + list(self.__dimensional_stats_table.metrics()), + ) + + if normalizer is not None: + for key, value in self.__dimensional_stats_table.metrics(): + key = normalizer(key)[0] + stats = normalized_stats.get(key) + if stats is None: + normalized_stats[key] = copy.copy(value) + else: + stats.merge_stats(value) + else: + normalized_stats = self.__dimensional_stats_table + + if self.__settings.debug.log_normalized_metric_data: + _logger.info( + "Normalized metric data for harvest of %r is %r.", + self.__settings.app_name, + list(normalized_stats.metrics()), + ) + + for key, value in normalized_stats.items(): + result.append((key, value)) + + return result + + def dimensional_metric_data_count(self): + """Returns a count of the number of unique metrics.""" + + if not self.__settings: + return 0 + + return self.__dimensional_stats_table.metrics_count() + def error_data(self): """Returns a to a list containing any errors collected during the reporting period. @@ -1184,7 +1455,7 @@ def slow_sql_data(self, connections): maximum = self.__settings.agent_limits.slow_sql_data - slow_sql_nodes = sorted(six.itervalues(self.__sql_stats_table), key=lambda x: x.max_call_time)[-maximum:] + slow_sql_nodes = sorted(self.__sql_stats_table.values(), key=lambda x: x.max_call_time)[-maximum:] result = [] @@ -1225,10 +1496,7 @@ def slow_sql_data(self, connections): level = self.__settings.agent_limits.data_compression_level level = level or zlib.Z_DEFAULT_COMPRESSION - params_data = base64.standard_b64encode(zlib.compress(six.b(json_data), level)) - - if six.PY3: - params_data = params_data.decode("Latin-1") + params_data = base64.standard_b64encode(zlib.compress(json_data.encode("latin-1"), level)).decode("Latin-1") # Limit the length of any SQL that is reported back. @@ -1339,12 +1607,9 @@ def transaction_trace_data(self, connections): level = self.__settings.agent_limits.data_compression_level level = level or zlib.Z_DEFAULT_COMPRESSION - zlib_data = zlib.compress(six.b(json_data), level) - - pack_data = base64.standard_b64encode(zlib_data) + zlib_data = zlib.compress(json_data.encode("latin-1"), level) - if six.PY3: - pack_data = pack_data.decode("Latin-1") + pack_data = base64.standard_b64encode(zlib_data).decode("Latin-1") root = transaction_trace.root @@ -1404,12 +1669,9 @@ def slow_transaction_data(self): level = self.__settings.agent_limits.data_compression_level level = level or zlib.Z_DEFAULT_COMPRESSION - zlib_data = zlib.compress(six.b(json_data), level) - - pack_data = base64.standard_b64encode(zlib_data) + zlib_data = zlib.compress(json_data.encode("latin-1"), level) - if six.PY3: - pack_data = pack_data.decode("Latin-1") + pack_data = base64.standard_b64encode(zlib_data).decode("Latin-1") root = transaction_trace.root @@ -1436,7 +1698,6 @@ def reset_stats(self, settings, reset_stream=False): """ self.__settings = settings - self.__stats_table = {} self.__sql_stats_table = {} self.__slow_transaction = None self.__slow_transaction_map = {} @@ -1444,9 +1705,11 @@ def reset_stats(self, settings, reset_stream=False): self.__transaction_errors = [] self.__synthetics_transactions = [] + self.reset_metric_stats() self.reset_transaction_events() self.reset_error_events() self.reset_custom_events() + self.reset_ml_events() self.reset_span_events() self.reset_log_events() self.reset_synthetics_events() @@ -1463,6 +1726,7 @@ def reset_metric_stats(self): """ self.__stats_table = {} + self.__dimensional_stats_table.reset_metric_stats() def reset_transaction_events(self): """Resets the accumulated statistics back to initial state for @@ -1489,6 +1753,12 @@ def reset_custom_events(self): else: self._custom_events = SampledDataSet() + def reset_ml_events(self): + if self.__settings is not None: + self._ml_events = SampledDataSet(self.__settings.event_harvest_config.harvest_limits.ml_event_data) + else: + self._ml_events = SampledDataSet() + def reset_span_events(self): if self.__settings is not None: self._span_events = SampledDataSet(self.__settings.event_harvest_config.harvest_limits.span_event_data) @@ -1622,6 +1892,7 @@ def merge(self, snapshot): self._merge_error_events(snapshot) self._merge_error_traces(snapshot) self._merge_custom_events(snapshot) + self._merge_ml_events(snapshot) self._merge_span_events(snapshot) self._merge_log_events(snapshot) self._merge_sql(snapshot) @@ -1647,6 +1918,7 @@ def rollback(self, snapshot): self._merge_synthetics_events(snapshot, rollback=True) self._merge_error_events(snapshot) self._merge_custom_events(snapshot, rollback=True) + self._merge_ml_events(snapshot, rollback=True) self._merge_span_events(snapshot, rollback=True) self._merge_log_events(snapshot, rollback=True) @@ -1660,7 +1932,7 @@ def merge_metric_stats(self, snapshot): if not self.__settings: return - for key, other in six.iteritems(snapshot.__stats_table): + for key, other in snapshot.__stats_table.items(): stats = self.__stats_table.get(key) if not stats: self.__stats_table[key] = other @@ -1716,6 +1988,12 @@ def _merge_custom_events(self, snapshot, rollback=False): return self._custom_events.merge(events) + def _merge_ml_events(self, snapshot, rollback=False): + events = snapshot.ml_events + if not events: + return + self._ml_events.merge(events) + def _merge_span_events(self, snapshot, rollback=False): events = snapshot.span_events if not events: @@ -1742,7 +2020,7 @@ def _merge_sql(self, snapshot): # the limit of how many to collect, only merge in if already # seen the specific SQL. - for key, slow_sql_stats in six.iteritems(snapshot.__sql_stats_table): + for key, slow_sql_stats in snapshot.__sql_stats_table.items(): stats = self.__sql_stats_table.get(key) if not stats: maximum = self.__settings.agent_limits.slow_sql_data @@ -1785,6 +2063,29 @@ def merge_custom_metrics(self, metrics): else: stats.merge_stats(other) + def merge_dimensional_metrics(self, metrics): + """ + Merges in a set of dimensional metrics. The metrics should be + provide as an iterable where each item is a tuple of the metric + key and the accumulated stats for the metric. The metric key should + also be a tuple, containing a name and attribute filtered frozenset of tags. + """ + + if not self.__settings: + return + + for key, other in metrics: + stats_container = self.__dimensional_stats_table.get(key) + if not stats_container: + self.__dimensional_stats_table[key] = other + else: + for tags, other_value in other.items(): + stats = stats_container.get(tags) + if not stats: + stats_container[tags] = other_value + else: + stats.merge_stats(other_value) + def _snapshot(self): copy = object.__new__(StatsEngineSnapshot) copy.__dict__.update(self.__dict__) @@ -1798,6 +2099,9 @@ def reset_transaction_events(self): def reset_custom_events(self): self._custom_events = None + def reset_ml_events(self): + self._ml_events = None + def reset_span_events(self): self._span_events = None diff --git a/newrelic/core/string_table.py b/newrelic/core/string_table.py index 5e37729e48..f8d38a1148 100644 --- a/newrelic/core/string_table.py +++ b/newrelic/core/string_table.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -class StringTable(object): +class StringTable(): def __init__(self): self.__values = [] @@ -20,7 +20,7 @@ def __init__(self): def cache(self, value): if not value in self.__mapping: - token = '`%d' % len(self.__values) + token = f'`{len(self.__values)}' self.__mapping[value] = token self.__values.append(value) return self.__mapping[value] diff --git a/newrelic/core/thread_utilization.py b/newrelic/core/thread_utilization.py index fd57ba9f86..dad118ec2c 100644 --- a/newrelic/core/thread_utilization.py +++ b/newrelic/core/thread_utilization.py @@ -26,7 +26,7 @@ def utilization_tracker(application): return _utilization_trackers.get(application) -class ThreadUtilizationDataSource(object): +class ThreadUtilizationDataSource(): def __init__(self, application): self._consumer_name = application diff --git a/newrelic/core/trace_cache.py b/newrelic/core/trace_cache.py index 5f0ddcd3da..c6c648ad91 100644 --- a/newrelic/core/trace_cache.py +++ b/newrelic/core/trace_cache.py @@ -22,17 +22,14 @@ import threading import traceback import weakref + +from collections.abc import MutableMapping try: import thread except ImportError: import _thread as thread -try: - from collections.abc import MutableMapping -except ImportError: - from collections import MutableMapping - from newrelic.core.config import global_settings from newrelic.core.loop_node import LoopNode @@ -74,7 +71,7 @@ def get_event_loop(task): return getattr(task, "_loop", None) -class cached_module(object): +class cached_module(): def __init__(self, module_path, name=None): self.module_path = module_path self.name = name or module_path @@ -105,7 +102,7 @@ def __init__(self): self._cache = weakref.WeakValueDictionary() def __repr__(self): - return "<%s object at 0x%x %s>" % (self.__class__.__name__, id(self), str(dict(self.items()))) + return f"<{self.__class__.__name__} object at 0x{id(self):x} {str(dict(self.items()))}>" def current_thread_id(self): """Returns the thread ID for the caller. @@ -382,7 +379,7 @@ def record_event_loop_wait(self, start_time, end_time): seen = None for root in roots: - guid = "%016x" % random.getrandbits(64) + guid = f"{random.getrandbits(64):016x}" node = LoopNode( fetch_name=fetch_name, start_time=start_time, diff --git a/newrelic/core/transaction_node.py b/newrelic/core/transaction_node.py index 0faae37909..675bdcaf03 100644 --- a/newrelic/core/transaction_node.py +++ b/newrelic/core/transaction_node.py @@ -22,9 +22,11 @@ import newrelic.core.error_collector import newrelic.core.trace_node +from newrelic.common.encoding_utils import camel_case from newrelic.common.streaming_utils import SpanProtoAttrs -from newrelic.core.attribute import create_agent_attributes, create_user_attributes +from newrelic.core.attribute import create_agent_attributes, create_attributes from newrelic.core.attribute_filter import ( + DST_ALL, DST_ERROR_COLLECTOR, DST_TRANSACTION_EVENTS, DST_TRANSACTION_TRACER, @@ -60,10 +62,12 @@ "errors", "slow_sql", "custom_events", + "ml_events", "log_events", "apdex_t", "suppress_apdex", "custom_metrics", + "dimensional_metrics", "guid", "cpu_time", "suppress_transaction_trace", @@ -74,6 +78,10 @@ "synthetics_job_id", "synthetics_monitor_id", "synthetics_header", + "synthetics_type", + "synthetics_initiator", + "synthetics_attributes", + "synthetics_info_header", "is_part_of_cat", "trip_id", "path_hash", @@ -167,8 +175,7 @@ def time_metrics(self, stats): if self.queue_start != 0: queue_wait = self.start_time - self.queue_start - if queue_wait < 0: - queue_wait = 0 + queue_wait = max(queue_wait, 0) yield TimeMetric(name="WebFrontend/QueueTime", scope="", duration=queue_wait, exclusive=None) @@ -179,7 +186,7 @@ def time_metrics(self, stats): # Generate the rollup metric. if self.type != "WebTransaction": - rollup = "%s/all" % self.type + rollup = f"{self.type}/all" else: rollup = self.type @@ -195,7 +202,7 @@ def time_metrics(self, stats): metric_suffix = "Other" yield TimeMetric( - name="%s/%s" % (metric_prefix, self.name_for_metric), + name=f"{metric_prefix}/{self.name_for_metric}", scope="", duration=self.total_time, exclusive=self.total_time, @@ -206,16 +213,11 @@ def time_metrics(self, stats): # Generate Distributed Tracing metrics if self.settings.distributed_tracing.enabled: - dt_tag = "%s/%s/%s/%s/all" % ( - self.parent_type or "Unknown", - self.parent_account or "Unknown", - self.parent_app or "Unknown", - self.parent_transport_type or "Unknown", - ) + dt_tag = f"{self.parent_type or 'Unknown'}/{self.parent_account or 'Unknown'}/{self.parent_app or 'Unknown'}/{self.parent_transport_type or 'Unknown'}/all" for bonus_tag in ("", metric_suffix): yield TimeMetric( - name="DurationByCaller/%s%s" % (dt_tag, bonus_tag), + name=f"DurationByCaller/{dt_tag}{bonus_tag}", scope="", duration=self.duration, exclusive=self.duration, @@ -223,7 +225,7 @@ def time_metrics(self, stats): if self.parent_transport_duration is not None: yield TimeMetric( - name="TransportDuration/%s%s" % (dt_tag, bonus_tag), + name=f"TransportDuration/{dt_tag}{bonus_tag}", scope="", duration=self.parent_transport_duration, exclusive=self.parent_transport_duration, @@ -231,7 +233,7 @@ def time_metrics(self, stats): if self.errors: yield TimeMetric( - name="ErrorsByCaller/%s%s" % (dt_tag, bonus_tag), scope="", duration=0.0, exclusive=None + name=f"ErrorsByCaller/{dt_tag}{bonus_tag}", scope="", duration=0.0, exclusive=None ) # Generate Error metrics @@ -242,10 +244,10 @@ def time_metrics(self, stats): yield TimeMetric(name="Errors/all", scope="", duration=0.0, exclusive=None) # Generate individual error metric for transaction. - yield TimeMetric(name="Errors/%s" % self.path, scope="", duration=0.0, exclusive=None) + yield TimeMetric(name=f"Errors/{self.path}", scope="", duration=0.0, exclusive=None) # Generate rollup metric for WebTransaction errors. - yield TimeMetric(name="Errors/all%s" % metric_suffix, scope="", duration=0.0, exclusive=None) + yield TimeMetric(name=f"Errors/all{metric_suffix}", scope="", duration=0.0, exclusive=None) else: yield TimeMetric(name="ErrorsExpected/all", scope="", duration=0.0, exclusive=None) @@ -290,7 +292,7 @@ def apdex_metrics(self, stats): # Generate the full apdex metric. yield ApdexMetric( - name="Apdex/%s" % self.name_for_metric, + name=f"Apdex/{self.name_for_metric}", satisfying=satisfying, tolerating=tolerating, frustrating=frustrating, @@ -361,17 +363,20 @@ def error_details(self): if attr.destinations & DST_ERROR_COLLECTOR: params["agentAttributes"][attr.name] = attr.value - err_attrs = create_user_attributes(error.custom_params, self.settings.attribute_filter) + err_attrs = create_attributes(error.custom_params, DST_ALL, self.settings.attribute_filter) for attr in err_attrs: if attr.destinations & DST_ERROR_COLLECTOR: params["userAttributes"][attr.name] = attr.value yield newrelic.core.error_collector.TracedError( - start_time=error.timestamp, path=self.path, message=error.message, type=error.type, parameters=params + start_time=error.timestamp, + path=self.path, + message=error.message, + type=error.type, + parameters=params, ) def transaction_trace(self, stats, limit, connections): - self.trace_node_count = 0 self.trace_node_limit = limit @@ -498,10 +503,8 @@ def _add_if_not_empty(key, value): return intrinsics def error_events(self, stats_table): - errors = [] for error in self.errors: - intrinsics = self.error_event_intrinsics(error, stats_table) # Add user and agent attributes to event @@ -530,7 +533,7 @@ def error_events(self, stats_table): # add error specific custom params to this error's userAttributes - err_attrs = create_user_attributes(error.custom_params, self.settings.attribute_filter) + err_attrs = create_attributes(error.custom_params, DST_ALL, self.settings.attribute_filter) for attr in err_attrs: if attr.destinations & DST_ERROR_COLLECTOR: user_attributes[attr.name] = attr.value @@ -541,7 +544,6 @@ def error_events(self, stats_table): return errors def error_event_intrinsics(self, error, stats_table): - intrinsics = self._event_intrinsics(stats_table) intrinsics["type"] = "TransactionError" @@ -563,7 +565,6 @@ def _event_intrinsics(self, stats_table): cache = getattr(self, "_event_intrinsics_cache", None) if cache is not None: - # We don't want to execute this function more than once, since # it should always yield the same data per transaction @@ -571,6 +572,7 @@ def _event_intrinsics(self, stats_table): intrinsics = self.distributed_trace_intrinsics.copy() + intrinsics["guid"] = self.guid intrinsics["timestamp"] = int(1000.0 * self.start_time) intrinsics["duration"] = self.response_time @@ -584,6 +586,15 @@ def _event_intrinsics(self, stats_table): intrinsics["nr.syntheticsJobId"] = self.synthetics_job_id intrinsics["nr.syntheticsMonitorId"] = self.synthetics_monitor_id + if self.synthetics_type: + intrinsics["nr.syntheticsType"] = self.synthetics_type + intrinsics["nr.syntheticsInitiator"] = self.synthetics_initiator + if self.synthetics_attributes: + # Add all synthetics attributes + for k, v in self.synthetics_attributes.items(): + if k: + intrinsics[f"nr.synthetics{camel_case(k, upper=True)}"] = v + def _add_call_time(source, target): # include time for keys previously added to stats table via # stats_engine.record_transaction diff --git a/newrelic/hooks/adapter_asgiref.py b/newrelic/hooks/adapter_asgiref.py index bcacd098ce..64d0b2fd91 100644 --- a/newrelic/hooks/adapter_asgiref.py +++ b/newrelic/hooks/adapter_asgiref.py @@ -14,16 +14,23 @@ from newrelic.api.time_trace import current_trace from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.signature import bind_args from newrelic.core.context import ContextOf, context_wrapper_async -def _bind_thread_handler(loop, source_task, *args, **kwargs): - return source_task +async def wrap_SyncToAsync__call__(wrapped, instance, args, kwargs): + kwargs["_nr_current_trace"] = current_trace() + return await wrapped(*args, **kwargs) def thread_handler_wrapper(wrapped, instance, args, kwargs): - task = _bind_thread_handler(*args, **kwargs) - with ContextOf(trace_cache_id=id(task), strict=False): + try: + bound_args = bind_args(wrapped, args, kwargs) + trace = bound_args["args"][0].keywords.pop("_nr_current_trace", None) + except Exception: + trace = None + + with ContextOf(trace=trace, strict=False): return wrapped(*args, **kwargs) @@ -34,4 +41,5 @@ def main_wrap_wrapper(wrapped, instance, args, kwargs): def instrument_asgiref_sync(module): wrap_function_wrapper(module, "SyncToAsync.thread_handler", thread_handler_wrapper) + wrap_function_wrapper(module, "SyncToAsync.__call__", wrap_SyncToAsync__call__) wrap_function_wrapper(module, "AsyncToSync.main_wrap", main_wrap_wrapper) diff --git a/newrelic/hooks/application_celery.py b/newrelic/hooks/application_celery.py index 12f41d8d0d..9fff9c818b 100644 --- a/newrelic/hooks/application_celery.py +++ b/newrelic/hooks/application_celery.py @@ -25,45 +25,50 @@ from newrelic.api.application import application_instance from newrelic.api.background_task import BackgroundTask from newrelic.api.function_trace import FunctionTrace +from newrelic.api.message_trace import MessageTrace from newrelic.api.pre_function import wrap_pre_function -from newrelic.api.object_wrapper import callable_name, ObjectWrapper from newrelic.api.transaction import current_transaction +from newrelic.common.object_wrapper import FunctionWrapper, wrap_function_wrapper, _NRBoundFunctionWrapper from newrelic.core.agent import shutdown_agent +UNKNOWN_TASK_NAME = "" +MAPPING_TASK_NAMES = {"celery.starmap", "celery.map"} -def CeleryTaskWrapper(wrapped, application=None, name=None): - def wrapper(wrapped, instance, args, kwargs): - transaction = current_transaction(active_only=False) +def task_info(instance, *args, **kwargs): + # Grab the current task, which can be located in either place + if instance: + task = instance + elif args: + task = args[0] + elif "task" in kwargs: + task = kwargs["task"] + else: + return UNKNOWN_TASK_NAME # Failsafe - if callable(name): - # Start Hotfix v2.2.1. - # if instance and inspect.ismethod(wrapped): - # _name = name(instance, *args, **kwargs) - # else: - # _name = name(*args, **kwargs) + # Task can be either a task instance or a signature, which subclasses dict, or an actual dict in some cases. + task_name = getattr(task, "name", None) or task.get("task", UNKNOWN_TASK_NAME) + task_source = task - if instance is not None: - _name = name(instance, *args, **kwargs) - else: - _name = name(*args, **kwargs) - # End Hotfix v2.2.1. + # Under mapping tasks, the root task name isn't descriptive enough so we append the + # subtask name to differentiate between different mapping tasks + if task_name in MAPPING_TASK_NAMES: + try: + subtask = kwargs["task"]["task"] + task_name = f"{task_name}/{subtask}" + task_source = task.app._tasks[subtask] + except Exception: + pass - elif name is None: - _name = callable_name(wrapped) + return task_name, task_source - else: - _name = name - # Helper for obtaining the appropriate application object. If - # has an activate() method assume it is a valid application - # object. Don't check by type so se can easily mock it for - # testing if need be. +def CeleryTaskWrapper(wrapped): + def wrapper(wrapped, instance, args, kwargs): + transaction = current_transaction(active_only=False) - def _application(): - if hasattr(application, 'activate'): - return application - return application_instance(application) + # Grab task name and source + _name, _source = task_info(instance, *args, **kwargs) # A Celery Task can be called either outside of a transaction, or # within the context of an existing transaction. There are 3 @@ -86,21 +91,36 @@ def _application(): # running inside of an existing transaction, we want to create # a new background transaction for it. - if transaction and (transaction.ignore_transaction or - transaction.stopped): + if transaction and (transaction.ignore_transaction or transaction.stopped): return wrapped(*args, **kwargs) elif transaction: - with FunctionTrace(_name, source=instance): + with FunctionTrace(_name, source=_source): return wrapped(*args, **kwargs) else: - with BackgroundTask(_application(), _name, 'Celery', source=instance): - return wrapped(*args, **kwargs) + with BackgroundTask(application_instance(), _name, "Celery", source=_source) as transaction: + # Attempt to grab distributed tracing headers + try: + # Headers on earlier versions of Celery may end up as attributes + # on the request context instead of as custom headers. Handler this + # by defaulting to using vars() if headers is not available + request = instance.request + headers = getattr(request, "headers", None) or vars(request) + + settings = transaction.settings + if headers is not None and settings is not None: + if settings.distributed_tracing.enabled: + transaction.accept_distributed_trace_headers(headers, transport_type="AMQP") + elif transaction.settings.cross_application_tracer.enabled: + transaction._process_incoming_cat_headers( + headers.get(MessageTrace.cat_id_key, None), + headers.get(MessageTrace.cat_transaction_key, None), + ) + except Exception: + pass - # Start Hotfix v2.2.1. - # obj = ObjectWrapper(wrapped, None, wrapper) - # End Hotfix v2.2.1. + return wrapped(*args, **kwargs) # Celery tasks that inherit from celery.app.task must implement a run() # method. @@ -110,37 +130,39 @@ def _application(): # task. But celery does a micro-optimization where if the __call__ method # was not overridden by an inherited task, then it will directly execute # the run() method without going through the __call__ method. Our - # instrumentation via ObjectWrapper() relies on __call__ being called which + # instrumentation via FunctionWrapper() relies on __call__ being called which # in turn executes the wrapper() function defined above. Since the micro # optimization bypasses __call__ method it breaks our instrumentation of - # celery. To circumvent this problem, we added a run() attribute to our - # ObjectWrapper which points to our __call__ method. This causes Celery + # celery. + # + # For versions of celery 2.5.3 to 2.5.5+ + # Celery has included a monkey-patching provision which did not perform this + # optimization on functions that were monkey-patched. Unfortunately, our + # wrappers are too transparent for celery to detect that they've even been + # monky-patched. To circumvent this, we set the __module__ of our wrapped task + # to this file which causes celery to properly detect that it has been patched. + # + # For versions of celery 2.5.3 to 2.5.5 + # To circumvent this problem, we added a run() attribute to our + # FunctionWrapper which points to our __call__ method. This causes Celery # to execute our __call__ method which in turn applies the wrapper # correctly before executing the task. - # - # This is only a problem in Celery versions 2.5.3 to 2.5.5. The later - # versions included a monkey-patching provision which did not perform this - # optimization on functions that were monkey-patched. - - # Start Hotfix v2.2.1. - # obj.__dict__['run'] = obj.__call__ - class _ObjectWrapper(ObjectWrapper): + class TaskWrapper(FunctionWrapper): def run(self, *args, **kwargs): return self.__call__(*args, **kwargs) - obj = _ObjectWrapper(wrapped, None, wrapper) - # End Hotfix v2.2.1. + wrapped_task = TaskWrapper(wrapped, wrapper) + # Reset __module__ to be less transparent so celery detects our monkey-patching + wrapped_task.__module__ = CeleryTaskWrapper.__module__ - return obj + return wrapped_task def instrument_celery_app_task(module): - # Triggered for both 'celery.app.task' and 'celery.task.base'. - if hasattr(module, 'BaseTask'): - + if hasattr(module, "BaseTask"): # Need to add a wrapper for background task entry point. # In Celery 2.2 the 'BaseTask' class actually resided in the @@ -155,44 +177,60 @@ def instrument_celery_app_task(module): # the task doesn't pass through it. For Celery 2.5+ need to wrap # the tracer instead. - def task_name(task, *args, **kwargs): - return task.name - if module.BaseTask.__module__ == module.__name__: - module.BaseTask.__call__ = CeleryTaskWrapper( - module.BaseTask.__call__, name=task_name) + module.BaseTask.__call__ = CeleryTaskWrapper(module.BaseTask.__call__) -def instrument_celery_execute_trace(module): +def wrap_Celery_send_task(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) - # Triggered for 'celery.execute_trace'. + # Merge distributed tracing headers into outgoing task headers + try: + dt_headers = MessageTrace.generate_request_headers(transaction) + original_headers = kwargs.get("headers", None) + if dt_headers: + if not original_headers: + kwargs["headers"] = dict(dt_headers) + else: + kwargs["headers"] = dt_headers = dict(dt_headers) + dt_headers.update(dict(original_headers)) + except Exception: + pass - if hasattr(module, 'build_tracer'): + return wrapped(*args, **kwargs) - # Need to add a wrapper for background task entry point. - # In Celery 2.5+ we need to wrap the task when tracer is being - # created. Note that in Celery 2.5 the 'build_tracer' function - # actually resided in the module 'celery.execute.task'. In - # Celery 3.0 the 'build_tracer' function moved to - # 'celery.task.trace'. +def wrap_worker_optimizations(wrapped, instance, args, kwargs): + # Attempt to uninstrument BaseTask before stack protection is installed or uninstalled + try: + from celery.app.task import BaseTask - _build_tracer = module.build_tracer + if isinstance(BaseTask.__call__, _NRBoundFunctionWrapper): + BaseTask.__call__ = BaseTask.__call__.__wrapped__ + except Exception: + BaseTask = None - def build_tracer(name, task, *args, **kwargs): - task = task or module.tasks[name] - task = CeleryTaskWrapper(task, name=name) - return _build_tracer(name, task, *args, **kwargs) + # Allow metaprogramming to run + result = wrapped(*args, **kwargs) - module.build_tracer = build_tracer + # Rewrap finalized BaseTask + if BaseTask: # Ensure imports succeeded + BaseTask.__call__ = CeleryTaskWrapper(BaseTask.__call__) + + return result -def instrument_celery_worker(module): +def instrument_celery_app_base(module): + if hasattr(module, "Celery") and hasattr(module.Celery, "send_task"): + wrap_function_wrapper(module, "Celery.send_task", wrap_Celery_send_task) - # Triggered for 'celery.worker' and 'celery.concurrency.processes'. - if hasattr(module, 'process_initializer'): +def instrument_celery_worker(module): + # Triggered for 'celery.worker' and 'celery.concurrency.processes'. + if hasattr(module, "process_initializer"): # We try and force registration of default application after # fork of worker process rather than lazily on first request. @@ -211,18 +249,24 @@ def process_initializer(*args, **kwargs): def instrument_celery_loaders_base(module): - def force_application_activation(*args, **kwargs): application_instance().activate() - wrap_pre_function(module, 'BaseLoader.init_worker', - force_application_activation) + wrap_pre_function(module, "BaseLoader.init_worker", force_application_activation) def instrument_billiard_pool(module): - def force_agent_shutdown(*args, **kwargs): shutdown_agent() - if hasattr(module, 'Worker'): - wrap_pre_function(module, 'Worker._do_exit', force_agent_shutdown) + if hasattr(module, "Worker"): + wrap_pre_function(module, "Worker._do_exit", force_agent_shutdown) + + +def instrument_celery_app_trace(module): + # Uses same wrapper for setup and reset worker optimizations to prevent patching and unpatching from removing wrappers + if hasattr(module, "setup_worker_optimizations"): + wrap_function_wrapper(module, "setup_worker_optimizations", wrap_worker_optimizations) + + if hasattr(module, "reset_worker_optimizations"): + wrap_function_wrapper(module, "reset_worker_optimizations", wrap_worker_optimizations) diff --git a/newrelic/hooks/application_gearman.py b/newrelic/hooks/application_gearman.py index 12646c3d2c..dbe86e3584 100644 --- a/newrelic/hooks/application_gearman.py +++ b/newrelic/hooks/application_gearman.py @@ -69,8 +69,7 @@ def _bind_params(submitted_connections, *args, **kwargs): first_connection = list(submitted_connections)[0] - url = 'gearman://%s:%s' % (first_connection.gearman_host, - first_connection.gearman_port) + url = f'gearman://{first_connection.gearman_host}:{first_connection.gearman_port}' with ExternalTrace('gearman', url): return wrapped(*args, **kwargs) @@ -105,8 +104,7 @@ def _bind_params(current_connection, *args, **kwargs): current_connection = _bind_params(*args, **kwargs) - tracer.url = 'gearman://%s:%s' % (current_connection.gearman_host, - current_connection.gearman_port) + tracer.url = f'gearman://{current_connection.gearman_host}:{current_connection.gearman_port}' return wrapped(*args, **kwargs) diff --git a/newrelic/hooks/component_djangorestframework.py b/newrelic/hooks/component_djangorestframework.py index f2d9c31c81..b7e132f797 100644 --- a/newrelic/hooks/component_djangorestframework.py +++ b/newrelic/hooks/component_djangorestframework.py @@ -44,10 +44,9 @@ def _args(request, *args, **kwargs): None) if view_func_callable_name: if handler == view.http_method_not_allowed: - name = '%s.%s' % (view_func_callable_name, - 'http_method_not_allowed') + name = f'{view_func_callable_name}.http_method_not_allowed' else: - name = '%s.%s' % (view_func_callable_name, request_method) + name = f'{view_func_callable_name}.{request_method}' else: name = callable_name(handler) diff --git a/newrelic/hooks/component_graphqlserver.py b/newrelic/hooks/component_graphqlserver.py index 29004c11fa..ebc62a34df 100644 --- a/newrelic/hooks/component_graphqlserver.py +++ b/newrelic/hooks/component_graphqlserver.py @@ -1,19 +1,18 @@ -from newrelic.api.asgi_application import wrap_asgi_application from newrelic.api.error_trace import ErrorTrace from newrelic.api.graphql_trace import GraphQLOperationTrace from newrelic.api.transaction import current_transaction -from newrelic.api.transaction_name import TransactionNameWrapper from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version from newrelic.core.graphql_utils import graphql_statement from newrelic.hooks.framework_graphql import ( - framework_version as graphql_framework_version, + GRAPHQL_VERSION, + ignore_graphql_duplicate_exception, ) -from newrelic.hooks.framework_graphql import ignore_graphql_duplicate_exception -def framework_details(): - import graphql_server - return ("GraphQLServer", getattr(graphql_server, "__version__", None)) +GRAPHQL_SERVER_VERSION = get_package_version("graphql-server") +graphql_server_major_version = int(GRAPHQL_SERVER_VERSION.split(".")[0]) + def bind_query(schema, params, *args, **kwargs): return getattr(params, "query", None) @@ -30,9 +29,8 @@ def wrap_get_response(wrapped, instance, args, kwargs): except TypeError: return wrapped(*args, **kwargs) - framework = framework_details() - transaction.add_framework_info(name=framework[0], version=framework[1]) - transaction.add_framework_info(name="GraphQL", version=graphql_framework_version()) + transaction.add_framework_info(name="GraphQLServer", version=GRAPHQL_SERVER_VERSION) + transaction.add_framework_info(name="GraphQL", version=GRAPHQL_VERSION) if hasattr(query, "body"): query = query.body @@ -45,5 +43,8 @@ def wrap_get_response(wrapped, instance, args, kwargs): with ErrorTrace(ignore=ignore_graphql_duplicate_exception): return wrapped(*args, **kwargs) + def instrument_graphqlserver(module): - wrap_function_wrapper(module, "get_response", wrap_get_response) + if graphql_server_major_version <= 2: + return + wrap_function_wrapper(module, "get_response", wrap_get_response) diff --git a/newrelic/hooks/component_piston.py b/newrelic/hooks/component_piston.py index 78b975ed53..54dc170ad3 100644 --- a/newrelic/hooks/component_piston.py +++ b/newrelic/hooks/component_piston.py @@ -12,18 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import newrelic.packages.six as six import newrelic.api.transaction import newrelic.api.function_trace -import newrelic.api.object_wrapper +import newrelic.common.object_wrapper +from newrelic.common.object_names import callable_name import newrelic.api.in_function -class MethodWrapper(object): +class MethodWrapper(): def __init__(self, wrapped, priority=None): - self._nr_name = newrelic.api.object_wrapper.callable_name(wrapped) + self._nr_name = callable_name(wrapped) self._nr_wrapped = wrapped self._nr_priority = priority @@ -46,7 +46,7 @@ def __call__(self, *args, **kwargs): return self._nr_wrapped(*args, **kwargs) -class ResourceInitWrapper(object): +class ResourceInitWrapper(): def __init__(self, wrapped): if isinstance(wrapped, tuple): @@ -68,7 +68,7 @@ def __getattr__(self, name): def __call__(self, *args, **kwargs): self._nr_wrapped(*args, **kwargs) handler = self.__instance.handler - for name in six.itervalues(self.__instance.callmap): + for name in self.__instance.callmap.values(): if hasattr(handler, name): setattr(handler, name, MethodWrapper( getattr(handler, name), priority=6)) @@ -76,7 +76,7 @@ def __call__(self, *args, **kwargs): def instrument_piston_resource(module): - newrelic.api.object_wrapper.wrap_object(module, + newrelic.common.object_wrapper.wrap_object(module, 'Resource.__init__', ResourceInitWrapper) diff --git a/newrelic/hooks/component_tastypie.py b/newrelic/hooks/component_tastypie.py index 8cc251916c..ed67a2be56 100644 --- a/newrelic/hooks/component_tastypie.py +++ b/newrelic/hooks/component_tastypie.py @@ -12,13 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - from newrelic.api.function_trace import FunctionTraceWrapper -from newrelic.api.object_wrapper import ObjectWrapper, callable_name +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import wrap_function_wrapper, function_wrapper from newrelic.api.transaction import current_transaction from newrelic.api.time_trace import notice_error -from newrelic.common.object_wrapper import wrap_function_wrapper def _nr_wrap_handle_exception(wrapped, instance, args, kwargs): @@ -41,11 +39,11 @@ def outer_fn_wrapper(outer_fn, instance, args, kwargs): callback = getattr(instance, 'top_level', None) elif meta.api_name is not None: group = 'Python/TastyPie/Api' - name = '%s/%s/%s' % (meta.api_name, meta.resource_name, view_name) + name = f'{meta.api_name}/{meta.resource_name}/{view_name}' callback = getattr(instance, view_name, None) else: group = 'Python/TastyPie/Resource' - name = '%s/%s' % (meta.resource_name, view_name) + name = f'{meta.resource_name}/{view_name}' callback = getattr(instance, view_name, None) # Give preference to naming web transaction and trace node after @@ -56,6 +54,7 @@ def outer_fn_wrapper(outer_fn, instance, args, kwargs): name = callable_name(callback) group = None + @function_wrapper def inner_fn_wrapper(inner_fn, instance, args, kwargs): transaction = current_transaction() @@ -69,18 +68,14 @@ def inner_fn_wrapper(inner_fn, instance, args, kwargs): result = outer_fn(*args, **kwargs) - return ObjectWrapper(result, None, inner_fn_wrapper) + return inner_fn_wrapper(result) def instrument_tastypie_resources(module): - _wrap_view = module.Resource.wrap_view - module.Resource.wrap_view = ObjectWrapper( - _wrap_view, None, outer_fn_wrapper) + wrap_function_wrapper(module, "Resource.wrap_view", outer_fn_wrapper) - wrap_function_wrapper(module, 'Resource._handle_500', - _nr_wrap_handle_exception) + wrap_function_wrapper(module, 'Resource._handle_500', _nr_wrap_handle_exception) def instrument_tastypie_api(module): - _wrap_view = module.Api.wrap_view - module.Api.wrap_view = ObjectWrapper(_wrap_view, None, outer_fn_wrapper) + wrap_function_wrapper(module, "Api.wrap_view", outer_fn_wrapper) diff --git a/newrelic/hooks/database_asyncpg.py b/newrelic/hooks/database_asyncpg.py index 0d03e9139a..0a00514a59 100644 --- a/newrelic/hooks/database_asyncpg.py +++ b/newrelic/hooks/database_asyncpg.py @@ -12,16 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -from newrelic.api.database_trace import ( - DatabaseTrace, - enable_datastore_instance_feature, - register_database_client, -) +from newrelic.api.database_trace import DatabaseTrace, register_database_client from newrelic.api.datastore_trace import DatastoreTrace from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper -class PostgresApi(object): +class PostgresApi(): @staticmethod def _instance_info(addr, connected_fut, con_params, *args, **kwargs): if isinstance(addr, str): @@ -43,7 +39,6 @@ def instance_info(cls, args, kwargs): quoting_style="single+dollar", instance_info=PostgresApi.instance_info, ) -enable_datastore_instance_feature(PostgresApi) class ProtocolProxy(ObjectProxy): @@ -94,9 +89,7 @@ async def query(self, query, *args, **kwargs): async def prepare(self, stmt_name, query, *args, **kwargs): with DatabaseTrace( - "PREPARE {stmt_name} FROM '{query}'".format( - stmt_name=stmt_name, query=query - ), + f"PREPARE {stmt_name} FROM '{query}'", dbapi2_module=PostgresApi, connect_params=getattr(self, "_nr_connect_params", None), source=self.__wrapped__.prepare, @@ -131,9 +124,7 @@ def proxy_protocol(wrapped, instance, args, kwargs): def wrap_connect(wrapped, instance, args, kwargs): host = port = database_name = None if "addr" in kwargs: - host, port, database_name = PostgresApi._instance_info( - kwargs["addr"], None, kwargs.get("params") - ) + host, port, database_name = PostgresApi._instance_info(kwargs["addr"], None, kwargs.get("params")) with DatastoreTrace( PostgresApi._nr_database_product, diff --git a/newrelic/hooks/database_dbapi2.py b/newrelic/hooks/database_dbapi2.py index 9506b0be49..ce6e933175 100644 --- a/newrelic/hooks/database_dbapi2.py +++ b/newrelic/hooks/database_dbapi2.py @@ -14,13 +14,12 @@ from newrelic.api.database_trace import DatabaseTrace, register_database_client from newrelic.api.function_trace import FunctionTrace -from newrelic.api.transaction import current_transaction from newrelic.common.object_names import callable_name -from newrelic.common.object_wrapper import wrap_object, ObjectProxy -from newrelic.core.config import global_settings +from newrelic.common.object_wrapper import ObjectProxy, wrap_object DEFAULT = object() + class CursorWrapper(ObjectProxy): def __init__(self, cursor, dbapi2_module, connect_params, cursor_params): @@ -31,41 +30,67 @@ def __init__(self, cursor, dbapi2_module, connect_params, cursor_params): def execute(self, sql, parameters=DEFAULT, *args, **kwargs): if parameters is not DEFAULT: - with DatabaseTrace(sql, self._nr_dbapi2_module, - self._nr_connect_params, self._nr_cursor_params, - parameters, (args, kwargs), source=self.__wrapped__.execute): - return self.__wrapped__.execute(sql, parameters, - *args, **kwargs) + with DatabaseTrace( + sql=sql, + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + cursor_params=self._nr_cursor_params, + sql_parameters=parameters, + execute_params=(args, kwargs), + source=self.__wrapped__.execute, + ): + return self.__wrapped__.execute(sql, parameters, *args, **kwargs) else: - with DatabaseTrace(sql, self._nr_dbapi2_module, - self._nr_connect_params, self._nr_cursor_params, - None, (args, kwargs), source=self.__wrapped__.execute): + with DatabaseTrace( + sql=sql, + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + cursor_params=self._nr_cursor_params, + execute_params=(args, kwargs), + source=self.__wrapped__.execute, + ): return self.__wrapped__.execute(sql, **kwargs) - def executemany(self, sql, seq_of_parameters): + def executemany(self, sql, seq_of_parameters, *args, **kwargs): try: seq_of_parameters = list(seq_of_parameters) parameters = seq_of_parameters[0] except (TypeError, IndexError): parameters = DEFAULT + if parameters is not DEFAULT: - with DatabaseTrace(sql, self._nr_dbapi2_module, - self._nr_connect_params, self._nr_cursor_params, - parameters, source=self.__wrapped__.executemany): - return self.__wrapped__.executemany(sql, seq_of_parameters) + with DatabaseTrace( + sql=sql, + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + cursor_params=self._nr_cursor_params, + sql_parameters=parameters, + source=self.__wrapped__.executemany, + ): + return self.__wrapped__.executemany(sql, seq_of_parameters, *args, **kwargs) else: - with DatabaseTrace(sql, self._nr_dbapi2_module, - self._nr_connect_params, self._nr_cursor_params, source=self.__wrapped__.executemany): - return self.__wrapped__.executemany(sql, seq_of_parameters) + with DatabaseTrace( + sql=sql, + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + cursor_params=self._nr_cursor_params, + source=self.__wrapped__.executemany, + ): + return self.__wrapped__.executemany(sql, seq_of_parameters, *args, **kwargs) def callproc(self, procname, parameters=DEFAULT): - with DatabaseTrace('CALL %s' % procname, - self._nr_dbapi2_module, self._nr_connect_params, source=self.__wrapped__.callproc): + with DatabaseTrace( + sql=f"CALL {procname}", + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + source=self.__wrapped__.callproc, + ): if parameters is not DEFAULT: return self.__wrapped__.callproc(procname, parameters) else: return self.__wrapped__.callproc(procname) + class ConnectionWrapper(ObjectProxy): __cursor_wrapper__ = CursorWrapper @@ -76,20 +101,29 @@ def __init__(self, connection, dbapi2_module, connect_params): self._nr_connect_params = connect_params def cursor(self, *args, **kwargs): - return self.__cursor_wrapper__(self.__wrapped__.cursor( - *args, **kwargs), self._nr_dbapi2_module, - self._nr_connect_params, (args, kwargs)) + return self.__cursor_wrapper__( + self.__wrapped__.cursor(*args, **kwargs), self._nr_dbapi2_module, self._nr_connect_params, (args, kwargs) + ) def commit(self): - with DatabaseTrace('COMMIT', self._nr_dbapi2_module, - self._nr_connect_params, source=self.__wrapped__.commit): + with DatabaseTrace( + sql="COMMIT", + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + source=self.__wrapped__.commit, + ): return self.__wrapped__.commit() def rollback(self): - with DatabaseTrace('ROLLBACK', self._nr_dbapi2_module, - self._nr_connect_params, source=self.__wrapped__.rollback): + with DatabaseTrace( + sql="ROLLBACK", + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + source=self.__wrapped__.rollback, + ): return self.__wrapped__.rollback() + class ConnectionFactory(ObjectProxy): __connection_wrapper__ = ConnectionWrapper @@ -99,17 +133,15 @@ def __init__(self, connect, dbapi2_module): self._nr_dbapi2_module = dbapi2_module def __call__(self, *args, **kwargs): - rollup = [] - rollup.append('Datastore/all') - rollup.append('Datastore/%s/all' % - self._nr_dbapi2_module._nr_database_product) + rollup = ["Datastore/all", f"Datastore/{self._nr_dbapi2_module._nr_database_product}/all"] + + with FunctionTrace(name=callable_name(self.__wrapped__), terminal=True, rollup=rollup, source=self.__wrapped__): + return self.__connection_wrapper__( + self.__wrapped__(*args, **kwargs), self._nr_dbapi2_module, (args, kwargs) + ) - with FunctionTrace(callable_name(self.__wrapped__), - terminal=True, rollup=rollup, source=self.__wrapped__): - return self.__connection_wrapper__(self.__wrapped__( - *args, **kwargs), self._nr_dbapi2_module, (args, kwargs)) def instrument(module): - register_database_client(module, 'DBAPI2', 'single') + register_database_client(module, "DBAPI2", "single") - wrap_object(module, 'connect', ConnectionFactory, (module,)) + wrap_object(module, "connect", ConnectionFactory, (module,)) diff --git a/newrelic/hooks/database_dbapi2_async.py b/newrelic/hooks/database_dbapi2_async.py new file mode 100644 index 0000000000..88f988f906 --- /dev/null +++ b/newrelic/hooks/database_dbapi2_async.py @@ -0,0 +1,161 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.api.database_trace import DatabaseTrace, register_database_client +from newrelic.api.function_trace import FunctionTrace +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import ObjectProxy, wrap_object +from newrelic.hooks.database_dbapi2 import DEFAULT + + +class AsyncCursorWrapper(ObjectProxy): + def __init__(self, cursor, dbapi2_module, connect_params, cursor_params): + super(AsyncCursorWrapper, self).__init__(cursor) + self._nr_dbapi2_module = dbapi2_module + self._nr_connect_params = connect_params + self._nr_cursor_params = cursor_params + + async def execute(self, sql, parameters=DEFAULT, *args, **kwargs): + if parameters is not DEFAULT: + with DatabaseTrace( + sql=sql, + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + cursor_params=self._nr_cursor_params, + sql_parameters=parameters, + execute_params=(args, kwargs), + source=self.__wrapped__.execute, + ): + return await self.__wrapped__.execute(sql, parameters, *args, **kwargs) + else: + with DatabaseTrace( + sql=sql, + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + cursor_params=self._nr_cursor_params, + execute_params=(args, kwargs), + source=self.__wrapped__.execute, + ): + return await self.__wrapped__.execute(sql, **kwargs) + + async def executemany(self, sql, seq_of_parameters, *args, **kwargs): + try: + seq_of_parameters = list(seq_of_parameters) + parameters = seq_of_parameters[0] + except (TypeError, IndexError): + parameters = DEFAULT + + if parameters is not DEFAULT: + with DatabaseTrace( + sql=sql, + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + cursor_params=self._nr_cursor_params, + sql_parameters=parameters, + source=self.__wrapped__.executemany, + ): + return await self.__wrapped__.executemany(sql, seq_of_parameters, *args, **kwargs) + else: + with DatabaseTrace( + sql=sql, + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + cursor_params=self._nr_cursor_params, + source=self.__wrapped__.executemany, + ): + return await self.__wrapped__.executemany(sql, seq_of_parameters, *args, **kwargs) + + async def callproc(self, procname, parameters=DEFAULT): + with DatabaseTrace( + sql=f"CALL {procname}", + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + source=self.__wrapped__.callproc, + ): + if parameters is not DEFAULT: + return await self.__wrapped__.callproc(procname, parameters) + else: + return await self.__wrapped__.callproc(procname) + + def __aiter__(self): + return self.__wrapped__.__aiter__() + + async def __aenter__(self): + await self.__wrapped__.__aenter__() + return self + + async def __aexit__(self, exc=None, val=None, tb=None): + return await self.__wrapped__.__aexit__(exc, val, tb) + + +class AsyncConnectionWrapper(ObjectProxy): + + __cursor_wrapper__ = AsyncCursorWrapper + + def __init__(self, connection, dbapi2_module, connect_params): + super(AsyncConnectionWrapper, self).__init__(connection) + self._nr_dbapi2_module = dbapi2_module + self._nr_connect_params = connect_params + + def cursor(self, *args, **kwargs): + return self.__cursor_wrapper__( + self.__wrapped__.cursor(*args, **kwargs), self._nr_dbapi2_module, self._nr_connect_params, (args, kwargs) + ) + + async def commit(self): + with DatabaseTrace( + sql="COMMIT", + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + source=self.__wrapped__.commit, + ): + return await self.__wrapped__.commit() + + async def rollback(self): + with DatabaseTrace( + sql="ROLLBACK", + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + source=self.__wrapped__.rollback, + ): + return await self.__wrapped__.rollback() + + async def __aenter__(self): + await self.__wrapped__.__aenter__() + return self + + async def __aexit__(self, exc=None, val=None, tb=None): + return await self.__wrapped__.__aexit__(exc, val, tb) + + +class AsyncConnectionFactory(ObjectProxy): + + __connection_wrapper__ = AsyncConnectionWrapper + + def __init__(self, connect, dbapi2_module): + super(AsyncConnectionFactory, self).__init__(connect) + self._nr_dbapi2_module = dbapi2_module + + async def __call__(self, *args, **kwargs): + rollup = ["Datastore/all", f"Datastore/{self._nr_dbapi2_module._nr_database_product}/all"] + + with FunctionTrace(name=callable_name(self.__wrapped__), terminal=True, rollup=rollup, source=self.__wrapped__): + connection = await self.__wrapped__(*args, **kwargs) + return self.__connection_wrapper__(connection, self._nr_dbapi2_module, (args, kwargs)) + + +def instrument(module): + register_database_client(module, "DBAPI2", "single") + + wrap_object(module, "connect", AsyncConnectionFactory, (module,)) diff --git a/newrelic/hooks/database_mysqldb.py b/newrelic/hooks/database_mysqldb.py index 31dd6bc19f..c36d91d409 100644 --- a/newrelic/hooks/database_mysqldb.py +++ b/newrelic/hooks/database_mysqldb.py @@ -14,54 +14,69 @@ import os -from newrelic.api.database_trace import (enable_datastore_instance_feature, - DatabaseTrace, register_database_client) +from newrelic.api.database_trace import DatabaseTrace, register_database_client from newrelic.api.function_trace import FunctionTrace from newrelic.api.transaction import current_transaction from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import wrap_object +from newrelic.hooks.database_dbapi2 import ConnectionFactory as DBAPI2ConnectionFactory +from newrelic.hooks.database_dbapi2 import ConnectionWrapper as DBAPI2ConnectionWrapper -from newrelic.hooks.database_dbapi2 import (ConnectionWrapper as - DBAPI2ConnectionWrapper, ConnectionFactory as DBAPI2ConnectionFactory) class ConnectionWrapper(DBAPI2ConnectionWrapper): - def __enter__(self): transaction = current_transaction() name = callable_name(self.__wrapped__.__enter__) with FunctionTrace(name, source=self.__wrapped__.__enter__): - cursor = self.__wrapped__.__enter__() + cursor = self.__wrapped__.__enter__() # The __enter__() method of original connection object returns # a new cursor instance for use with 'as' assignment. We need # to wrap that in a cursor wrapper otherwise we will not track # any queries done via it. - return self.__cursor_wrapper__(cursor, self._nr_dbapi2_module, - self._nr_connect_params, None) + return self.__cursor_wrapper__(cursor, self._nr_dbapi2_module, self._nr_connect_params, None) def __exit__(self, exc, value, tb): transaction = current_transaction() name = callable_name(self.__wrapped__.__exit__) with FunctionTrace(name, source=self.__wrapped__.__exit__): if exc is None: - with DatabaseTrace('COMMIT', self._nr_dbapi2_module, self._nr_connect_params, source=self.__wrapped__.__exit__): + with DatabaseTrace( + "COMMIT", self._nr_dbapi2_module, self._nr_connect_params, source=self.__wrapped__.__exit__ + ): return self.__wrapped__.__exit__(exc, value, tb) else: - with DatabaseTrace('ROLLBACK', self._nr_dbapi2_module, self._nr_connect_params, source=self.__wrapped__.__exit__): + with DatabaseTrace( + "ROLLBACK", self._nr_dbapi2_module, self._nr_connect_params, source=self.__wrapped__.__exit__ + ): return self.__wrapped__.__exit__(exc, value, tb) + class ConnectionFactory(DBAPI2ConnectionFactory): __connection_wrapper__ = ConnectionWrapper + def instance_info(args, kwargs): - def _bind_params(host=None, user=None, passwd=None, db=None, port=None, - unix_socket=None, conv=None, connect_timeout=None, compress=None, - named_pipe=None, init_command=None, read_default_file=None, - read_default_group=None, *args, **kwargs): - return (host, port, db, unix_socket, - read_default_file, read_default_group) + def _bind_params( + host=None, + user=None, + passwd=None, + db=None, + port=None, + unix_socket=None, + conv=None, + connect_timeout=None, + compress=None, + named_pipe=None, + init_command=None, + read_default_file=None, + read_default_group=None, + *args, + **kwargs + ): + return (host, port, db, unix_socket, read_default_file, read_default_group) params = _bind_params(*args, **kwargs) host, port, db, unix_socket, read_default_file, read_default_group = params @@ -69,38 +84,38 @@ def _bind_params(host=None, user=None, passwd=None, db=None, port=None, port_path_or_id = None if read_default_file or read_default_group: - host = host or 'default' - port_path_or_id = 'unknown' + host = host or "default" + port_path_or_id = "unknown" elif not host: - host = 'localhost' + host = "localhost" - if host == 'localhost': + if host == "localhost": # precedence: explicit -> cnf (if used) -> env -> 'default' - port_path_or_id = (unix_socket or - port_path_or_id or - os.getenv('MYSQL_UNIX_PORT', 'default')) + port_path_or_id = unix_socket or port_path_or_id or os.getenv("MYSQL_UNIX_PORT", "default") elif explicit_host: # only reach here if host is explicitly passed in port = port and str(port) # precedence: explicit -> cnf (if used) -> env -> '3306' - port_path_or_id = (port or - port_path_or_id or - os.getenv('MYSQL_TCP_PORT', '3306')) + port_path_or_id = port or port_path_or_id or os.getenv("MYSQL_TCP_PORT", "3306") # There is no default database if omitted from the connect params # In this case, we should report unknown - db = db or 'unknown' + db = db or "unknown" return (host, port_path_or_id, db) -def instrument_mysqldb(module): - register_database_client(module, database_product='MySQL', - quoting_style='single+double', explain_query='explain', - explain_stmts=('select',), instance_info=instance_info) - enable_datastore_instance_feature(module) +def instrument_mysqldb(module): + register_database_client( + module, + database_product="MySQL", + quoting_style="single+double", + explain_query="explain", + explain_stmts=("select",), + instance_info=instance_info, + ) - wrap_object(module, 'connect', ConnectionFactory, (module,)) + wrap_object(module, "connect", ConnectionFactory, (module,)) # The connect() function is actually aliased with Connect() and # Connection, the later actually being the Connection type object. @@ -108,5 +123,5 @@ def instrument_mysqldb(module): # interferes with direct type usage. If people are using the # Connection object directly, they should really be using connect(). - if hasattr(module, 'Connect'): - wrap_object(module, 'Connect', ConnectionFactory, (module,)) + if hasattr(module, "Connect"): + wrap_object(module, "Connect", ConnectionFactory, (module,)) diff --git a/newrelic/hooks/database_oursql.py b/newrelic/hooks/database_oursql.py deleted file mode 100644 index e8c5811538..0000000000 --- a/newrelic/hooks/database_oursql.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from newrelic.common.object_wrapper import wrap_object -from newrelic.api.database_trace import register_database_client - -from newrelic.hooks.database_mysqldb import ConnectionFactory - -def instance_info(args, kwargs): - def _bind_params(host=None, user=None, passwd=None, db=None, - port=None, *args, **kwargs): - return host, port, db - - host, port, db = _bind_params(*args, **kwargs) - - return (host, port, db) - -def instrument_oursql(module): - register_database_client(module, database_product='MySQL', - quoting_style='single+double', explain_query='explain', - explain_stmts=('select',), instance_info=instance_info) - - wrap_object(module, 'connect', ConnectionFactory, (module,)) - - # The connect() function is actually aliased with Connect() and - # Connection, the later actually being the Connection type object. - # Instrument Connect(), but don't instrument Connection in case that - # interferes with direct type usage. If people are using the - # Connection object directly, they should really be using connect(). - - if hasattr(module, 'Connect'): - wrap_object(module, 'Connect', ConnectionFactory, (module,)) diff --git a/newrelic/hooks/database_psycopg.py b/newrelic/hooks/database_psycopg.py new file mode 100644 index 0000000000..04fec9b714 --- /dev/null +++ b/newrelic/hooks/database_psycopg.py @@ -0,0 +1,507 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import os + +from urllib.parse import unquote, parse_qsl + +from newrelic.api.database_trace import DatabaseTrace, register_database_client +from newrelic.api.function_trace import FunctionTrace +from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import ( + ObjectProxy, + wrap_function_wrapper, + wrap_object, +) +from newrelic.hooks.database_dbapi2 import DEFAULT +from newrelic.hooks.database_dbapi2 import ConnectionFactory as DBAPI2ConnectionFactory +from newrelic.hooks.database_dbapi2 import ConnectionWrapper as DBAPI2ConnectionWrapper +from newrelic.hooks.database_dbapi2 import CursorWrapper as DBAPI2CursorWrapper +from newrelic.hooks.database_dbapi2_async import ( + AsyncConnectionFactory as DBAPI2AsyncConnectionFactory, +) +from newrelic.hooks.database_dbapi2_async import ( + AsyncConnectionWrapper as DBAPI2AsyncConnectionWrapper, +) +from newrelic.hooks.database_dbapi2_async import ( + AsyncCursorWrapper as DBAPI2AsyncCursorWrapper, +) + +from newrelic.packages.urllib3 import util as ul3_util + +# These functions return True if a non-default connection or cursor class is +# used. If the default connection and cursor are used without any unknown +# arguments, we can safely drop all cursor parameters to generate explain +# plans. Explain plans do not work with named cursors, so dropping the name +# allows explain plans to continue to function. +PsycopgConnection = None +PsycopgAsyncConnection = None + + +def should_preserve_connection_args(self, conninfo="", cursor_factory=None, **kwargs): + try: + if cursor_factory: + return True + + return self._nr_last_object.__self__ not in (PsycopgConnection, PsycopgAsyncConnection) + except Exception: + pass + + return False + + +def should_preserve_cursor_args( + name=None, binary=False, row_factory=None, scrollable=None, withhold=False, *args, **kwargs +): + return bool(args or kwargs) + + +class CursorWrapper(DBAPI2CursorWrapper): + def __enter__(self): + self.__wrapped__.__enter__() + + # Must return a reference to self as otherwise will be + # returning the inner cursor object. If 'as' is used + # with the 'with' statement this will mean no longer + # using the wrapped cursor object and nothing will be + # tracked. + + return self + + def execute(self, query, params=DEFAULT, *args, **kwargs): + if hasattr(query, "as_string"): + query = query.as_string(self) + + return super(CursorWrapper, self).execute(query, params, *args, **kwargs) + + def executemany(self, query, params_seq, *args, **kwargs): + if hasattr(query, "as_string"): + query = query.as_string(self) + + return super(CursorWrapper, self).executemany(query, params_seq, *args, **kwargs) + + +class ConnectionSaveParamsWrapper(DBAPI2ConnectionWrapper): + + __cursor_wrapper__ = CursorWrapper + + def execute(self, query, params=DEFAULT, *args, **kwargs): + if hasattr(query, "as_string"): + query = query.as_string(self) + + if params is not DEFAULT: + with DatabaseTrace( + sql=query, + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + cursor_params=None, + sql_parameters=params, + execute_params=(args, kwargs), + source=self.__wrapped__.execute, + ): + cursor = self.__wrapped__.execute(query, params, *args, **kwargs) + else: + with DatabaseTrace( + sql=query, + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + cursor_params=None, + sql_parameters=None, + execute_params=(args, kwargs), + source=self.__wrapped__.execute, + ): + cursor = self.__wrapped__.execute(query, **kwargs) + + return self.__cursor_wrapper__(cursor, self._nr_dbapi2_module, self._nr_connect_params, (args, kwargs)) + + def __enter__(self): + name = callable_name(self.__wrapped__.__enter__) + with FunctionTrace(name, source=self.__wrapped__.__enter__): + self.__wrapped__.__enter__() + + # Must return a reference to self as otherwise will be + # returning the inner connection object. If 'as' is used + # with the 'with' statement this will mean no longer + # using the wrapped connection object and nothing will be + # tracked. + + return self + + def __exit__(self, exc, value, tb): + name = callable_name(self.__wrapped__.__exit__) + with FunctionTrace(name, source=self.__wrapped__.__exit__): + if exc is None: + with DatabaseTrace( + sql="COMMIT", + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + source=self.__wrapped__.__exit__, + ): + return self.__wrapped__.__exit__(exc, value, tb) + else: + with DatabaseTrace( + sql="ROLLBACK", + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + source=self.__wrapped__.__exit__, + ): + return self.__wrapped__.__exit__(exc, value, tb) + + +# This connection wrapper does not save cursor parameters for explain plans. It +# is only used for the default connection class. +class ConnectionWrapper(ConnectionSaveParamsWrapper): + def cursor(self, *args, **kwargs): + # If any unknown cursor params are detected or a cursor factory is + # used, store params for explain plans later. + if should_preserve_cursor_args(*args, **kwargs): + cursor_params = (args, kwargs) + else: + cursor_params = None + + return self.__cursor_wrapper__( + self.__wrapped__.cursor(*args, **kwargs), self._nr_dbapi2_module, self._nr_connect_params, cursor_params + ) + + +class ConnectionFactory(DBAPI2ConnectionFactory): + + __connection_wrapper__ = ConnectionWrapper + + def __call__(self, *args, **kwargs): + if should_preserve_connection_args(self, *args, **kwargs): + self.__connection_wrapper__ = ConnectionSaveParamsWrapper + + return super(ConnectionFactory, self).__call__(*args, **kwargs) + + +# Due to our explain plan feature requiring the use of synchronous DBAPI2 compliant modules, we can't support the use +# of AsyncConnection or any derivative to retrieve explain plans. There's also no longer a connection_factory argument +# on psycopg.connect() that was present in psycopg2, which affects the logic that attempted to use the same Connection +# class for explain plans. This is only relevant for users subclassing psycopg.Connection or psycopg.AsyncConnection. +# With no easy way to preserve or use the same class, and given that using the AsyncConnection class would never +# function with our explain plan feature, we always attempt to use the DBAPI2 compliant method of instantiating a new +# Connection, that being psycopg.connect(). That function is an alias of psycopg.Connection.connect(), and returns an +# instance of psycopg.Connection. +# +# Additionally, care is taken to preserve the cursor_factory argument and to use custom cursor classes. However, with +# AsyncConnection the compatible cursors will be async, which will not be compatible with the explain plan's +# synchronous Connection instance. To avoid this issue, we refuse to preserve the cursor_factory arument for +# AsyncConnection and instead fall back to using the default psycopg.Cursor class. +# +# This should allow the largest number of users to still have explain plans function for their applications, whether or +# not they are using AsyncConnection or custom classes. The issue of using a synchronous Connection object in an async +# application should be somewhat mitigated by the fact that our explain plan feature functions on the harvest thread. + + +class AsyncCursorWrapper(DBAPI2AsyncCursorWrapper): + def __init__(self, cursor, dbapi2_module, connect_params, cursor_params): + # Remove async cursor_factory so it doesn't interfere with sync Connections for explain plans + args, kwargs = connect_params + kwargs = dict(kwargs) + kwargs.pop("cursor_factory", None) + + super().__init__(cursor, dbapi2_module, (args, kwargs), cursor_params) + + async def __aenter__(self): + await self.__wrapped__.__aenter__() + + # Must return a reference to self as otherwise will be + # returning the inner cursor object. If 'as' is used + # with the 'with' statement this will mean no longer + # using the wrapped cursor object and nothing will be + # tracked. + + return self + + async def execute(self, query, params=DEFAULT, *args, **kwargs): + if hasattr(query, "as_string"): + query = query.as_string(self) + + return await super(AsyncCursorWrapper, self).execute(query, params, *args, **kwargs) + + async def executemany(self, query, params_seq, *args, **kwargs): + if hasattr(query, "as_string"): + query = query.as_string(self) + + return await super(AsyncCursorWrapper, self).executemany(query, params_seq, *args, **kwargs) + + +class AsyncConnectionSaveParamsWrapper(DBAPI2AsyncConnectionWrapper): + + __cursor_wrapper__ = AsyncCursorWrapper + + async def execute(self, query, params=DEFAULT, *args, **kwargs): + if hasattr(query, "as_string"): + query = query.as_string(self) + + if params is not DEFAULT: + with DatabaseTrace( + sql=query, + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + cursor_params=None, + sql_parameters=params, + execute_params=(args, kwargs), + source=self.__wrapped__.execute, + ): + cursor = await self.__wrapped__.execute(query, params, *args, **kwargs) + else: + with DatabaseTrace( + sql=query, + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + cursor_params=None, + sql_parameters=None, + execute_params=(args, kwargs), + source=self.__wrapped__.execute, + ): + cursor = await self.__wrapped__.execute(query, **kwargs) + + return self.__cursor_wrapper__(cursor, self._nr_dbapi2_module, self._nr_connect_params, (args, kwargs)) + + async def __aenter__(self): + name = callable_name(self.__wrapped__.__aenter__) + with FunctionTrace(name, source=self.__wrapped__.__aenter__): + await self.__wrapped__.__aenter__() + + # Must return a reference to self as otherwise will be + # returning the inner connection object. If 'as' is used + # with the 'with' statement this will mean no longer + # using the wrapped connection object and nothing will be + # tracked. + + return self + + async def __aexit__(self, exc, value, tb): + name = callable_name(self.__wrapped__.__aexit__) + with FunctionTrace(name, source=self.__wrapped__.__aexit__): + if exc is None: + with DatabaseTrace( + sql="COMMIT", + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + source=self.__wrapped__.__aexit__, + ): + return await self.__wrapped__.__aexit__(exc, value, tb) + else: + with DatabaseTrace( + sql="ROLLBACK", + dbapi2_module=self._nr_dbapi2_module, + connect_params=self._nr_connect_params, + source=self.__wrapped__.__aexit__, + ): + return await self.__wrapped__.__aexit__(exc, value, tb) + + +# This connection wrapper does not save cursor parameters for explain plans. It +# is only used for the default connection class. +class AsyncConnectionWrapper(AsyncConnectionSaveParamsWrapper): + def cursor(self, *args, **kwargs): + # If any unknown cursor params are detected or a cursor factory is + # used, store params for explain plans later. + if should_preserve_cursor_args(*args, **kwargs): + cursor_params = (args, kwargs) + else: + cursor_params = None + + return self.__cursor_wrapper__( + self.__wrapped__.cursor(*args, **kwargs), self._nr_dbapi2_module, self._nr_connect_params, cursor_params + ) + + +class AsyncConnectionFactory(DBAPI2AsyncConnectionFactory): + + __connection_wrapper__ = AsyncConnectionWrapper + + async def __call__(self, *args, **kwargs): + if should_preserve_connection_args(self, *args, **kwargs): + self.__connection_wrapper__ = AsyncConnectionSaveParamsWrapper + + return await super(AsyncConnectionFactory, self).__call__(*args, **kwargs) + + +def instance_info(args, kwargs): + + p_host, p_hostaddr, p_port, p_dbname = _parse_connect_params(args, kwargs) + host, port, db_name = _add_defaults(p_host, p_hostaddr, p_port, p_dbname) + + return (host, port, db_name) + + +def _parse_connect_params(args, kwargs): + def _bind_params(conninfo=None, *args, **kwargs): + return conninfo + + dsn = _bind_params(*args, **kwargs) + + try: + if dsn and (dsn.startswith("postgres://") or dsn.startswith("postgresql://")): + + # Parse dsn as URI + # + # According to PGSQL, connect URIs are in the format of RFC 3896 + # https://www.postgresql.org/docs/9.5/static/libpq-connect.html + + parsed_uri = ul3_util.parse_url(dsn) + + host = parsed_uri.hostname or None + host = host and unquote(host) + + # ipv6 brackets [] are contained in the URI hostname + # and should be removed + host = host and host.strip("[]") + + port = parsed_uri.port + + db_name = parsed_uri.path + db_name = db_name and db_name.lstrip("/") + db_name = db_name or None + + query = parsed_uri.query or "" + qp = dict(parse_qsl(query)) + + # Query parameters override hierarchical values in URI. + + host = qp.get("host") or host or None + hostaddr = qp.get("hostaddr") + port = qp.get("port") or port + db_name = qp.get("dbname") or db_name + + elif dsn: + + # Parse dsn as a key-value connection string + + kv = dict([pair.split("=", 2) for pair in dsn.split()]) + host = kv.get("host") + hostaddr = kv.get("hostaddr") + port = kv.get("port") + db_name = kv.get("dbname") + + else: + + # No dsn, so get the instance info from keyword arguments. + + host = kwargs.get("host") + hostaddr = kwargs.get("hostaddr") + port = kwargs.get("port") + db_name = kwargs.get("dbname") + + # Ensure non-None values are strings. + + (host, hostaddr, port, db_name) = [str(s) if s is not None else s for s in (host, hostaddr, port, db_name)] + + except Exception: + host = "unknown" + hostaddr = "unknown" + port = "unknown" + db_name = "unknown" + + return (host, hostaddr, port, db_name) + + +def _add_defaults(parsed_host, parsed_hostaddr, parsed_port, parsed_database): + + # ENV variables set the default values + + parsed_host = parsed_host or os.environ.get("PGHOST") + parsed_hostaddr = parsed_hostaddr or os.environ.get("PGHOSTADDR") + parsed_port = parsed_port or os.environ.get("PGPORT") + database = parsed_database or os.environ.get("PGDATABASE") or "default" + + # If hostaddr is present, we use that, since host is used for auth only. + + parsed_host = parsed_hostaddr or parsed_host + + if parsed_host is None: + host = "localhost" + port = "default" + elif parsed_host.startswith("/"): + host = "localhost" + port = f"{parsed_host}/.s.PGSQL.{parsed_port or '5432'}" + else: + host = parsed_host + port = parsed_port or "5432" + + return (host, port, database) + + +def wrapper_psycopg_as_string(wrapped, instance, args, kwargs): + def _bind_params(context, *args, **kwargs): + return context, args, kwargs + + context, _args, _kwargs = _bind_params(*args, **kwargs) + + # Unwrap the context for string conversion since psycopg uses duck typing + # and a TypeError will be raised if a wrapper is used. + if hasattr(context, "__wrapped__"): + context = context.__wrapped__ + + return wrapped(context, *_args, **_kwargs) + + +def wrap_Connection_connect(module): + def _wrap_Connection_connect(wrapped, instance, args, kwargs): + return ConnectionFactory(wrapped, module)(*args, **kwargs) + + return _wrap_Connection_connect + + +def wrap_AsyncConnection_connect(module): + async def _wrap_AsyncConnection_connect(wrapped, instance, args, kwargs): + return await AsyncConnectionFactory(wrapped, module)(*args, **kwargs) + + return _wrap_AsyncConnection_connect + + +def instrument_psycopg(module): + global PsycopgConnection, PsycopgAsyncConnection + + PsycopgConnection = module.Connection + PsycopgAsyncConnection = module.AsyncConnection + + register_database_client( + module, + database_product="Postgres", + quoting_style="single+dollar", + explain_query="explain", + explain_stmts=("select", "insert", "update", "delete"), + instance_info=instance_info, + ) + + if hasattr(module, "Connection"): + if hasattr(module.Connection, "connect"): + if not isinstance(module.Connection.connect, ObjectProxy): + wrap_function_wrapper(module, "Connection.connect", wrap_Connection_connect(module)) + + if hasattr(module, "connect"): + if not isinstance(module.connect, ObjectProxy): + wrap_object(module, "connect", ConnectionFactory, (module,)) + + if hasattr(module, "AsyncConnection") and hasattr(module.AsyncConnection, "connect"): + if not isinstance(module.AsyncConnection.connect, ObjectProxy): + wrap_function_wrapper(module, "AsyncConnection.connect", wrap_AsyncConnection_connect(module)) + + +def instrument_psycopg_sql(module): + if hasattr(module, "Composable") and hasattr(module.Composable, "as_string"): + for name, cls in inspect.getmembers(module): + if not inspect.isclass(cls): + continue + + if not issubclass(cls, module.Composable): + continue + + wrap_function_wrapper(module, f"{name}.as_string", wrapper_psycopg_as_string) diff --git a/newrelic/hooks/database_psycopg2.py b/newrelic/hooks/database_psycopg2.py index 970909a33d..ec1d96ce06 100644 --- a/newrelic/hooks/database_psycopg2.py +++ b/newrelic/hooks/database_psycopg2.py @@ -15,26 +15,21 @@ import inspect import os -from newrelic.api.database_trace import (enable_datastore_instance_feature, - register_database_client, DatabaseTrace) +from urllib.parse import unquote, parse_qsl + +from newrelic.api.database_trace import DatabaseTrace, register_database_client from newrelic.api.function_trace import FunctionTrace from newrelic.api.transaction import current_transaction from newrelic.common.object_names import callable_name -from newrelic.common.object_wrapper import (wrap_object, ObjectProxy, - wrap_function_wrapper) - -from newrelic.hooks.database_dbapi2 import (ConnectionWrapper as - DBAPI2ConnectionWrapper, ConnectionFactory as DBAPI2ConnectionFactory, - CursorWrapper as DBAPI2CursorWrapper, DEFAULT) - -try: - from urllib import unquote -except ImportError: - from urllib.parse import unquote -try: - from urlparse import parse_qsl -except ImportError: - from urllib.parse import parse_qsl +from newrelic.common.object_wrapper import ( + ObjectProxy, + wrap_function_wrapper, + wrap_object, +) +from newrelic.hooks.database_dbapi2 import DEFAULT +from newrelic.hooks.database_dbapi2 import ConnectionFactory as DBAPI2ConnectionFactory +from newrelic.hooks.database_dbapi2 import ConnectionWrapper as DBAPI2ConnectionWrapper +from newrelic.hooks.database_dbapi2 import CursorWrapper as DBAPI2CursorWrapper from newrelic.packages.urllib3 import util as ul3_util @@ -43,33 +38,27 @@ # used. If the default connection and cursor are used without any unknown # arguments, we can safely drop all cursor parameters to generate explain # plans. Explain plans do not work with named cursors. -def _bind_connect( - dsn=None, connection_factory=None, cursor_factory=None, - *args, **kwargs): +def _bind_connect(dsn=None, connection_factory=None, cursor_factory=None, *args, **kwargs): return bool(connection_factory or cursor_factory) -def _bind_cursor( - name=None, cursor_factory=None, scrollable=None, - withhold=False, *args, **kwargs): +def _bind_cursor(name=None, cursor_factory=None, scrollable=None, withhold=False, *args, **kwargs): return bool(cursor_factory or args or kwargs) class CursorWrapper(DBAPI2CursorWrapper): - def execute(self, sql, parameters=DEFAULT, *args, **kwargs): - if hasattr(sql, 'as_string'): + if hasattr(sql, "as_string"): sql = sql.as_string(self) - return super(CursorWrapper, self).execute(sql, parameters, *args, - **kwargs) + return super(CursorWrapper, self).execute(sql, parameters, *args, **kwargs) def __enter__(self): self.__wrapped__.__enter__() return self def executemany(self, sql, seq_of_parameters): - if hasattr(sql, 'as_string'): + if hasattr(sql, "as_string"): sql = sql.as_string(self) return super(CursorWrapper, self).executemany(sql, seq_of_parameters) @@ -83,7 +72,7 @@ def __enter__(self): transaction = current_transaction() name = callable_name(self.__wrapped__.__enter__) with FunctionTrace(name, source=self.__wrapped__.__enter__): - self.__wrapped__.__enter__() + self.__wrapped__.__enter__() # Must return a reference to self as otherwise will be # returning the inner connection object. If 'as' is used @@ -98,19 +87,20 @@ def __exit__(self, exc, value, tb): name = callable_name(self.__wrapped__.__exit__) with FunctionTrace(name, source=self.__wrapped__.__exit__): if exc is None: - with DatabaseTrace('COMMIT', - self._nr_dbapi2_module, self._nr_connect_params, source=self.__wrapped__.__exit__): + with DatabaseTrace( + "COMMIT", self._nr_dbapi2_module, self._nr_connect_params, source=self.__wrapped__.__exit__ + ): return self.__wrapped__.__exit__(exc, value, tb) else: - with DatabaseTrace('ROLLBACK', - self._nr_dbapi2_module, self._nr_connect_params, source=self.__wrapped__.__exit__): + with DatabaseTrace( + "ROLLBACK", self._nr_dbapi2_module, self._nr_connect_params, source=self.__wrapped__.__exit__ + ): return self.__wrapped__.__exit__(exc, value, tb) # This connection wrapper does not save cursor parameters for explain plans. It # is only used for the default connection class. class ConnectionWrapper(ConnectionSaveParamsWrapper): - def cursor(self, *args, **kwargs): # If any unknown cursor params are detected or a cursor factory is # used, store params for explain plans later. @@ -119,9 +109,9 @@ def cursor(self, *args, **kwargs): else: cursor_params = None - return self.__cursor_wrapper__(self.__wrapped__.cursor( - *args, **kwargs), self._nr_dbapi2_module, - self._nr_connect_params, cursor_params) + return self.__cursor_wrapper__( + self.__wrapped__.cursor(*args, **kwargs), self._nr_dbapi2_module, self._nr_connect_params, cursor_params + ) class ConnectionFactory(DBAPI2ConnectionFactory): @@ -144,15 +134,13 @@ def instance_info(args, kwargs): def _parse_connect_params(args, kwargs): - def _bind_params(dsn=None, *args, **kwargs): return dsn dsn = _bind_params(*args, **kwargs) try: - if dsn and (dsn.startswith('postgres://') or - dsn.startswith('postgresql://')): + if dsn and (dsn.startswith("postgres://") or dsn.startswith("postgresql://")): # Parse dsn as URI # @@ -166,53 +154,52 @@ def _bind_params(dsn=None, *args, **kwargs): # ipv6 brackets [] are contained in the URI hostname # and should be removed - host = host and host.strip('[]') + host = host and host.strip("[]") port = parsed_uri.port db_name = parsed_uri.path - db_name = db_name and db_name.lstrip('/') + db_name = db_name and db_name.lstrip("/") db_name = db_name or None - query = parsed_uri.query or '' + query = parsed_uri.query or "" qp = dict(parse_qsl(query)) # Query parameters override hierarchical values in URI. - host = qp.get('host') or host or None - hostaddr = qp.get('hostaddr') - port = qp.get('port') or port - db_name = qp.get('dbname') or db_name + host = qp.get("host") or host or None + hostaddr = qp.get("hostaddr") + port = qp.get("port") or port + db_name = qp.get("dbname") or db_name elif dsn: # Parse dsn as a key-value connection string - kv = dict([pair.split('=', 2) for pair in dsn.split()]) - host = kv.get('host') - hostaddr = kv.get('hostaddr') - port = kv.get('port') - db_name = kv.get('dbname') + kv = dict([pair.split("=", 2) for pair in dsn.split()]) + host = kv.get("host") + hostaddr = kv.get("hostaddr") + port = kv.get("port") + db_name = kv.get("dbname") else: # No dsn, so get the instance info from keyword arguments. - host = kwargs.get('host') - hostaddr = kwargs.get('hostaddr') - port = kwargs.get('port') - db_name = kwargs.get('database') + host = kwargs.get("host") + hostaddr = kwargs.get("hostaddr") + port = kwargs.get("port") + db_name = kwargs.get("database") # Ensure non-None values are strings. - (host, hostaddr, port, db_name) = [str(s) if s is not None else s - for s in (host, hostaddr, port, db_name)] + (host, hostaddr, port, db_name) = [str(s) if s is not None else s for s in (host, hostaddr, port, db_name)] except Exception: - host = 'unknown' - hostaddr = 'unknown' - port = 'unknown' - db_name = 'unknown' + host = "unknown" + hostaddr = "unknown" + port = "unknown" + db_name = "unknown" return (host, hostaddr, port, db_name) @@ -221,37 +208,39 @@ def _add_defaults(parsed_host, parsed_hostaddr, parsed_port, parsed_database): # ENV variables set the default values - parsed_host = parsed_host or os.environ.get('PGHOST') - parsed_hostaddr = parsed_hostaddr or os.environ.get('PGHOSTADDR') - parsed_port = parsed_port or os.environ.get('PGPORT') - database = parsed_database or os.environ.get('PGDATABASE') or 'default' + parsed_host = parsed_host or os.environ.get("PGHOST") + parsed_hostaddr = parsed_hostaddr or os.environ.get("PGHOSTADDR") + parsed_port = parsed_port or os.environ.get("PGPORT") + database = parsed_database or os.environ.get("PGDATABASE") or "default" # If hostaddr is present, we use that, since host is used for auth only. parsed_host = parsed_hostaddr or parsed_host if parsed_host is None: - host = 'localhost' - port = 'default' - elif parsed_host.startswith('/'): - host = 'localhost' - port = '%s/.s.PGSQL.%s' % (parsed_host, parsed_port or '5432') + host = "localhost" + port = "default" + elif parsed_host.startswith("/"): + host = "localhost" + port = f"{parsed_host}/.s.PGSQL.{parsed_port or '5432'}" else: host = parsed_host - port = parsed_port or '5432' + port = parsed_port or "5432" return (host, port, database) def instrument_psycopg2(module): - register_database_client(module, database_product='Postgres', - quoting_style='single+dollar', explain_query='explain', - explain_stmts=('select', 'insert', 'update', 'delete'), - instance_info=instance_info) - - enable_datastore_instance_feature(module) + register_database_client( + module, + database_product="Postgres", + quoting_style="single+dollar", + explain_query="explain", + explain_stmts=("select", "insert", "update", "delete"), + instance_info=instance_info, + ) - wrap_object(module, 'connect', ConnectionFactory, (module,)) + wrap_object(module, "connect", ConnectionFactory, (module,)) def wrapper_psycopg2_register_type(wrapped, instance, args, kwargs): @@ -277,7 +266,7 @@ def _bind_params(context, *args, **kwargs): # Unwrap the context for string conversion since psycopg2 uses duck typing # and a TypeError will be raised if a wrapper is used. - if hasattr(context, '__wrapped__'): + if hasattr(context, "__wrapped__"): context = context.__wrapped__ return wrapped(context, *_args, **_kwargs) @@ -289,36 +278,31 @@ def _bind_params(context, *args, **kwargs): # In doing that we need to make sure it has not already been monkey # patched by checking to see if it is already an ObjectProxy. def instrument_psycopg2__psycopg2(module): - if hasattr(module, 'register_type'): + if hasattr(module, "register_type"): if not isinstance(module.register_type, ObjectProxy): - wrap_function_wrapper(module, 'register_type', - wrapper_psycopg2_register_type) + wrap_function_wrapper(module, "register_type", wrapper_psycopg2_register_type) def instrument_psycopg2_extensions(module): - if hasattr(module, 'register_type'): + if hasattr(module, "register_type"): if not isinstance(module.register_type, ObjectProxy): - wrap_function_wrapper(module, 'register_type', - wrapper_psycopg2_register_type) + wrap_function_wrapper(module, "register_type", wrapper_psycopg2_register_type) def instrument_psycopg2__json(module): - if hasattr(module, 'register_type'): + if hasattr(module, "register_type"): if not isinstance(module.register_type, ObjectProxy): - wrap_function_wrapper(module, 'register_type', - wrapper_psycopg2_register_type) + wrap_function_wrapper(module, "register_type", wrapper_psycopg2_register_type) def instrument_psycopg2__range(module): - if hasattr(module, 'register_type'): + if hasattr(module, "register_type"): if not isinstance(module.register_type, ObjectProxy): - wrap_function_wrapper(module, 'register_type', - wrapper_psycopg2_register_type) + wrap_function_wrapper(module, "register_type", wrapper_psycopg2_register_type) def instrument_psycopg2_sql(module): - if (hasattr(module, 'Composable') and - hasattr(module.Composable, 'as_string')): + if hasattr(module, "Composable") and hasattr(module.Composable, "as_string"): for name, cls in inspect.getmembers(module): if not inspect.isclass(cls): continue @@ -326,5 +310,4 @@ def instrument_psycopg2_sql(module): if not issubclass(cls, module.Composable): continue - wrap_function_wrapper(module, name + '.as_string', - wrapper_psycopg2_as_string) + wrap_function_wrapper(module, f"{name}.as_string", wrapper_psycopg2_as_string) diff --git a/newrelic/hooks/datastore_aiomcache.py b/newrelic/hooks/datastore_aiomcache.py new file mode 100644 index 0000000000..7afb76faeb --- /dev/null +++ b/newrelic/hooks/datastore_aiomcache.py @@ -0,0 +1,42 @@ +from newrelic.api.datastore_trace import wrap_datastore_trace + +_memcache_client_methods = ( + "get", + "gets", + "get_multi", + "set", + "cas", + "set_multi", + "add", + "replace", + "delete", + "delete_multi", + "incr", + "decr", + "flush_all", + "stats", +) + + +def capture_host(self, *args, **kwargs): + if hasattr(self, "_pool") and hasattr(self._pool, "_host"): + return self._pool._host + + +def capture_port(self, *args, **kwargs): + if hasattr(self, "_pool") and hasattr(self._pool, "_port"): + return self._pool._port + + +def instrument_aiomcache_client(module): + for name in _memcache_client_methods: + if hasattr(module.Client, name): + wrap_datastore_trace( + module, + f"Client.{name}", + product="Memcached", + target=None, + operation=name, + host=capture_host, + port_path_or_id=capture_port, + ) diff --git a/newrelic/hooks/datastore_aioredis.py b/newrelic/hooks/datastore_aioredis.py index 9bd5b17b0e..19c9c41996 100644 --- a/newrelic/hooks/datastore_aioredis.py +++ b/newrelic/hooks/datastore_aioredis.py @@ -11,17 +11,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - from newrelic.api.datastore_trace import DatastoreTrace from newrelic.api.time_trace import current_trace from newrelic.api.transaction import current_transaction -from newrelic.common.object_wrapper import wrap_function_wrapper, function_wrapper +from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version_tuple from newrelic.hooks.datastore_redis import ( _redis_client_methods, _redis_multipart_commands, _redis_operation_re, ) -from newrelic.common.package_version_utils import get_package_version_tuple + +AIOREDIS_VERSION = get_package_version_tuple("aioredis") def _conn_attrs_to_dict(connection): @@ -39,14 +40,13 @@ def _conn_attrs_to_dict(connection): def _instance_info(kwargs): host = kwargs.get("host") or "localhost" - port_path_or_id = str(kwargs.get("port") or kwargs.get("path", 6379)) + port_path_or_id = str(kwargs.get("path") or kwargs.get("port", 6379)) db = str(kwargs.get("db") or 0) return (host, port_path_or_id, db) def _wrap_AioRedis_method_wrapper(module, instance_class_name, operation): - @function_wrapper async def _nr_wrapper_AioRedis_async_method_(wrapped, instance, args, kwargs): transaction = current_transaction() @@ -55,33 +55,36 @@ async def _nr_wrapper_AioRedis_async_method_(wrapped, instance, args, kwargs): with DatastoreTrace(product="Redis", target=None, operation=operation): return await wrapped(*args, **kwargs) - + def _nr_wrapper_AioRedis_method_(wrapped, instance, args, kwargs): # Check for transaction and return early if found. # Method will return synchronously without executing, # it will be added to the command stack and run later. - aioredis_version = get_package_version_tuple("aioredis") - if aioredis_version and aioredis_version < (2,): + + # This conditional is for versions of aioredis that are outside + # New Relic's supportability window but will still work. New + # Relic does not provide testing/support for this. In order to + # keep functionality without affecting coverage metrics, this + # segment is excluded from coverage analysis. + if AIOREDIS_VERSION and AIOREDIS_VERSION < (2,): # pragma: no cover # AioRedis v1 uses a RedisBuffer instead of a real connection for queueing up pipeline commands from aioredis.commands.transaction import _RedisBuffer + if isinstance(instance._pool_or_conn, _RedisBuffer): # Method will return synchronously without executing, # it will be added to the command stack and run later. return wrapped(*args, **kwargs) else: # AioRedis v2 uses a Pipeline object for a client and internally queues up pipeline commands - if aioredis_version: + if AIOREDIS_VERSION: from aioredis.client import Pipeline - else: - from redis.asyncio.client import Pipeline if isinstance(instance, Pipeline): return wrapped(*args, **kwargs) # Method should be run when awaited, therefore we wrap in an async wrapper. return _nr_wrapper_AioRedis_async_method_(wrapped)(*args, **kwargs) - - name = "%s.%s" % (instance_class_name, operation) + name = f"{instance_class_name}.{operation}" wrap_function_wrapper(module, name, _nr_wrapper_AioRedis_method_) @@ -109,7 +112,9 @@ async def wrap_Connection_send_command(wrapped, instance, args, kwargs): # If it's not a multi part command, there's no need to trace it, so # we can return early. - if operation.split()[0] not in _redis_multipart_commands: # Set the datastore info on the DatastoreTrace containing this function call. + if ( + operation.split()[0] not in _redis_multipart_commands + ): # Set the datastore info on the DatastoreTrace containing this function call. trace = current_trace() # Find DatastoreTrace no matter how many other traces are inbetween @@ -126,7 +131,7 @@ async def wrap_Connection_send_command(wrapped, instance, args, kwargs): # Convert multi args to single arg string if operation in _redis_multipart_commands and len(args) > 1: - operation = "%s %s" % (operation, args[1].strip().lower()) + operation = f"{operation} {args[1].strip().lower()}" operation = _redis_operation_re.sub("_", operation) @@ -136,7 +141,12 @@ async def wrap_Connection_send_command(wrapped, instance, args, kwargs): return await wrapped(*args, **kwargs) -def wrap_RedisConnection_execute(wrapped, instance, args, kwargs): +# This wrapper is for versions of aioredis that are outside +# New Relic's supportability window but will still work. New +# Relic does not provide testing/support for this. In order to +# keep functionality without affecting coverage metrics, this +# segment is excluded from coverage analysis. +def wrap_RedisConnection_execute(wrapped, instance, args, kwargs): # pragma: no cover # RedisConnection in aioredis v1 returns a future instead of using coroutines transaction = current_transaction() if not transaction: @@ -161,7 +171,9 @@ def wrap_RedisConnection_execute(wrapped, instance, args, kwargs): # If it's not a multi part command, there's no need to trace it, so # we can return early. - if operation.split()[0] not in _redis_multipart_commands: # Set the datastore info on the DatastoreTrace containing this function call. + if ( + operation.split()[0] not in _redis_multipart_commands + ): # Set the datastore info on the DatastoreTrace containing this function call. trace = current_trace() # Find DatastoreTrace no matter how many other traces are inbetween @@ -178,7 +190,7 @@ def wrap_RedisConnection_execute(wrapped, instance, args, kwargs): # Convert multi args to single arg string if operation in _redis_multipart_commands and len(args) > 1: - operation = "%s %s" % (operation, args[1].strip().lower()) + operation = f"{operation} {args[1].strip().lower()}" operation = _redis_operation_re.sub("_", operation) @@ -202,6 +214,11 @@ def instrument_aioredis_connection(module): if hasattr(module.Connection, "send_command"): wrap_function_wrapper(module, "Connection.send_command", wrap_Connection_send_command) - if hasattr(module, "RedisConnection"): + # This conditional is for versions of aioredis that are outside + # New Relic's supportability window but will still work. New + # Relic does not provide testing/support for this. In order to + # keep functionality without affecting coverage metrics, this + # segment is excluded from coverage analysis. + if hasattr(module, "RedisConnection"): # pragma: no cover if hasattr(module.RedisConnection, "execute"): wrap_function_wrapper(module, "RedisConnection.execute", wrap_RedisConnection_execute) diff --git a/newrelic/hooks/datastore_aredis.py b/newrelic/hooks/datastore_aredis.py index 236cbf3f8c..4eeb4a230e 100644 --- a/newrelic/hooks/datastore_aredis.py +++ b/newrelic/hooks/datastore_aredis.py @@ -34,7 +34,7 @@ async def _nr_wrapper_Aredis_method_(wrapped, instance, args, kwargs): with DatastoreTrace(product="Redis", target=None, operation=operation): return await wrapped(*args, **kwargs) - name = "%s.%s" % (instance_class_name, operation) + name = f"{instance_class_name}.{operation}" wrap_function_wrapper(module, name, _nr_wrapper_Aredis_method_) @@ -80,7 +80,7 @@ async def wrap_Connection_send_command(wrapped, instance, args, kwargs): # Convert multi args to single arg string if operation in _redis_multipart_commands and len(args) > 1: - operation = "%s %s" % (operation, args[1].strip().lower()) + operation = f"{operation} {args[1].strip().lower()}" operation = _redis_operation_re.sub("_", operation) diff --git a/newrelic/hooks/datastore_bmemcached.py b/newrelic/hooks/datastore_bmemcached.py index 3091f0992b..3bf2a1cb3b 100644 --- a/newrelic/hooks/datastore_bmemcached.py +++ b/newrelic/hooks/datastore_bmemcached.py @@ -32,7 +32,29 @@ ) +def capture_host(self, *args, **kwargs): + if hasattr(self, "servers"): + for s in self.servers: + if hasattr(s, "host"): + return s.host + + +def capture_port(self, *args, **kwargs): + if hasattr(self, "servers"): + for s in self.servers: + if hasattr(s, "port"): + return s.port + + def instrument_bmemcached_client(module): for name in _memcache_client_methods: if hasattr(module.Client, name): - wrap_datastore_trace(module, "Client.%s" % name, product="Memcached", target=None, operation=name) + wrap_datastore_trace( + module, + f"Client.{name}", + product="Memcached", + target=None, + operation=name, + host=capture_host, + port_path_or_id=capture_port, + ) diff --git a/newrelic/hooks/datastore_elasticsearch.py b/newrelic/hooks/datastore_elasticsearch.py index 2417aabfe5..980385a1b3 100644 --- a/newrelic/hooks/datastore_elasticsearch.py +++ b/newrelic/hooks/datastore_elasticsearch.py @@ -16,7 +16,6 @@ from newrelic.api.transaction import current_transaction from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper from newrelic.common.package_version_utils import get_package_version_tuple -from newrelic.packages import six # An index name can be a string, None or a sequence. In the case of None # an empty string or '*', it is the same as using '_all'. When a string @@ -30,7 +29,7 @@ def _index_name(index): if not index or index == "*": return "_all" - if not isinstance(index, six.string_types) or "," in index: + if not isinstance(index, str) or "," in index: return "other" return index @@ -124,7 +123,7 @@ def _nr_wrapper_Elasticsearch_method_(wrapped, instance, args, kwargs): index = arg_extractor(*args, **kwargs) if prefix: - operation = "%s.%s" % (prefix, method_name) + operation = f"{prefix}.{method_name}" else: operation = method_name @@ -143,7 +142,7 @@ def _nr_wrapper_Elasticsearch_method_(wrapped, instance, args, kwargs): return result - wrap_function_wrapper(module, "%s.%s" % (class_name, method_name), _nr_wrapper_Elasticsearch_method_) + wrap_function_wrapper(module, f"{class_name}.{method_name}", _nr_wrapper_Elasticsearch_method_) _elasticsearch_client_methods_below_v8 = ( diff --git a/newrelic/hooks/datastore_firestore.py b/newrelic/hooks/datastore_firestore.py new file mode 100644 index 0000000000..ba7fdfb507 --- /dev/null +++ b/newrelic/hooks/datastore_firestore.py @@ -0,0 +1,473 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.api.datastore_trace import wrap_datastore_trace +from newrelic.api.function_trace import wrap_function_trace +from newrelic.common.async_wrapper import generator_wrapper, async_generator_wrapper + + +def _conn_str_to_host(getter): + """Safely transform a getter that can retrieve a connection string into the resulting host.""" + + def closure(obj, *args, **kwargs): + try: + return getter(obj, *args, **kwargs).split(":")[0] + except Exception: + return None + + return closure + + +def _conn_str_to_port(getter): + """Safely transform a getter that can retrieve a connection string into the resulting port.""" + + def closure(obj, *args, **kwargs): + try: + return getter(obj, *args, **kwargs).split(":")[1] + except Exception: + return None + + return closure + + +# Default Target ID and Instance Info +_get_object_id = lambda obj, *args, **kwargs: getattr(obj, "id", None) +_get_client_database_string = lambda obj, *args, **kwargs: getattr( + getattr(obj, "_client", None), "_database_string", None +) +_get_client_target = lambda obj, *args, **kwargs: obj._client._target +_get_client_target_host = _conn_str_to_host(_get_client_target) +_get_client_target_port = _conn_str_to_port(_get_client_target) + +# Client Instance Info +_get_database_string = lambda obj, *args, **kwargs: getattr(obj, "_database_string", None) +_get_target = lambda obj, *args, **kwargs: obj._target +_get_target_host = _conn_str_to_host(_get_target) +_get_target_port = _conn_str_to_port(_get_target) + +# Query Target ID +_get_parent_id = lambda obj, *args, **kwargs: getattr(getattr(obj, "_parent", None), "id", None) + +# AggregationQuery Target ID +_get_collection_ref_id = lambda obj, *args, **kwargs: getattr(getattr(obj, "_collection_ref", None), "id", None) + + +def instrument_google_cloud_firestore_v1_base_client(module): + rollup = ("Datastore/all", "Datastore/Firestore/all") + wrap_function_trace( + module, "BaseClient.__init__", name=f"{module.__name__}:BaseClient.__init__", terminal=True, rollup=rollup + ) + + +def instrument_google_cloud_firestore_v1_client(module): + if hasattr(module, "Client"): + class_ = module.Client + for method in ("collections", "get_all"): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"Client.{method}", + operation=method, + product="Firestore", + target=None, + host=_get_target_host, + port_path_or_id=_get_target_port, + database_name=_get_database_string, + async_wrapper=generator_wrapper, + ) + + +def instrument_google_cloud_firestore_v1_async_client(module): + if hasattr(module, "AsyncClient"): + class_ = module.AsyncClient + for method in ("collections", "get_all"): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"AsyncClient.{method}", + operation=method, + product="Firestore", + target=None, + host=_get_target_host, + port_path_or_id=_get_target_port, + database_name=_get_database_string, + async_wrapper=async_generator_wrapper, + ) + + +def instrument_google_cloud_firestore_v1_collection(module): + if hasattr(module, "CollectionReference"): + class_ = module.CollectionReference + for method in ("add", "get"): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"CollectionReference.{method}", + product="Firestore", + target=_get_object_id, + operation=method, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + ) + + for method in ("stream", "list_documents"): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"CollectionReference.{method}", + operation=method, + product="Firestore", + target=_get_object_id, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + async_wrapper=generator_wrapper, + ) + + +def instrument_google_cloud_firestore_v1_async_collection(module): + if hasattr(module, "AsyncCollectionReference"): + class_ = module.AsyncCollectionReference + for method in ("add", "get"): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"AsyncCollectionReference.{method}", + product="Firestore", + target=_get_object_id, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + operation=method, + ) + + for method in ("stream", "list_documents"): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"AsyncCollectionReference.{method}", + operation=method, + product="Firestore", + target=_get_object_id, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + async_wrapper=async_generator_wrapper, + ) + + +def instrument_google_cloud_firestore_v1_document(module): + if hasattr(module, "DocumentReference"): + class_ = module.DocumentReference + for method in ("create", "delete", "get", "set", "update"): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"DocumentReference.{method}", + product="Firestore", + target=_get_object_id, + operation=method, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + ) + + for method in ("collections",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"DocumentReference.{method}", + operation=method, + product="Firestore", + target=_get_object_id, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + async_wrapper=generator_wrapper, + ) + + +def instrument_google_cloud_firestore_v1_async_document(module): + if hasattr(module, "AsyncDocumentReference"): + class_ = module.AsyncDocumentReference + for method in ("create", "delete", "get", "set", "update"): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"AsyncDocumentReference.{method}", + product="Firestore", + target=_get_object_id, + operation=method, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + ) + + for method in ("collections",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"AsyncDocumentReference.{method}", + operation=method, + product="Firestore", + target=_get_object_id, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + async_wrapper=async_generator_wrapper, + ) + + +def instrument_google_cloud_firestore_v1_query(module): + if hasattr(module, "Query"): + class_ = module.Query + for method in ("get",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"Query.{method}", + product="Firestore", + target=_get_parent_id, + operation=method, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + ) + + for method in ("stream",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"Query.{method}", + operation=method, + product="Firestore", + target=_get_parent_id, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + async_wrapper=generator_wrapper, + ) + + if hasattr(module, "CollectionGroup"): + class_ = module.CollectionGroup + for method in ("get_partitions",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"CollectionGroup.{method}", + operation=method, + product="Firestore", + target=_get_parent_id, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + async_wrapper=generator_wrapper, + ) + + +def instrument_google_cloud_firestore_v1_async_query(module): + if hasattr(module, "AsyncQuery"): + class_ = module.AsyncQuery + for method in ("get",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"AsyncQuery.{method}", + product="Firestore", + target=_get_parent_id, + operation=method, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + ) + + for method in ("stream",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"AsyncQuery.{method}", + operation=method, + product="Firestore", + target=_get_parent_id, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + async_wrapper=async_generator_wrapper, + ) + + if hasattr(module, "AsyncCollectionGroup"): + class_ = module.AsyncCollectionGroup + for method in ("get_partitions",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"AsyncCollectionGroup.{method}", + operation=method, + product="Firestore", + target=_get_parent_id, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + async_wrapper=async_generator_wrapper, + ) + + +def instrument_google_cloud_firestore_v1_aggregation(module): + if hasattr(module, "AggregationQuery"): + class_ = module.AggregationQuery + for method in ("get",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"AggregationQuery.{method}", + product="Firestore", + target=_get_collection_ref_id, + operation=method, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + ) + + for method in ("stream",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"AggregationQuery.{method}", + operation=method, + product="Firestore", + target=_get_collection_ref_id, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + async_wrapper=generator_wrapper, + ) + + +def instrument_google_cloud_firestore_v1_async_aggregation(module): + if hasattr(module, "AsyncAggregationQuery"): + class_ = module.AsyncAggregationQuery + for method in ("get",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"AsyncAggregationQuery.{method}", + product="Firestore", + target=_get_collection_ref_id, + operation=method, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + ) + + for method in ("stream",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"AsyncAggregationQuery.{method}", + operation=method, + product="Firestore", + target=_get_collection_ref_id, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + async_wrapper=async_generator_wrapper, + ) + + +def instrument_google_cloud_firestore_v1_batch(module): + if hasattr(module, "WriteBatch"): + class_ = module.WriteBatch + for method in ("commit",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"WriteBatch.{method}", + product="Firestore", + target=None, + operation=method, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + ) + + +def instrument_google_cloud_firestore_v1_async_batch(module): + if hasattr(module, "AsyncWriteBatch"): + class_ = module.AsyncWriteBatch + for method in ("commit",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"AsyncWriteBatch.{method}", + product="Firestore", + target=None, + operation=method, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + ) + + +def instrument_google_cloud_firestore_v1_bulk_batch(module): + if hasattr(module, "BulkWriteBatch"): + class_ = module.BulkWriteBatch + for method in ("commit",): + if hasattr(class_, method): + wrap_datastore_trace( + module, + f"BulkWriteBatch.{method}", + product="Firestore", + target=None, + operation=method, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + ) + + +def instrument_google_cloud_firestore_v1_transaction(module): + if hasattr(module, "Transaction"): + class_ = module.Transaction + for method in ("_commit", "_rollback"): + if hasattr(class_, method): + operation = method[1:] # Trim leading underscore + wrap_datastore_trace( + module, + f"Transaction.{method}", + product="Firestore", + target=None, + operation=operation, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + ) + + +def instrument_google_cloud_firestore_v1_async_transaction(module): + if hasattr(module, "AsyncTransaction"): + class_ = module.AsyncTransaction + for method in ("_commit", "_rollback"): + if hasattr(class_, method): + operation = method[1:] # Trim leading underscore + wrap_datastore_trace( + module, + f"AsyncTransaction.{method}", + product="Firestore", + target=None, + operation=operation, + host=_get_client_target_host, + port_path_or_id=_get_client_target_port, + database_name=_get_client_database_string, + ) diff --git a/newrelic/hooks/datastore_memcache.py b/newrelic/hooks/datastore_memcache.py index 90b2d43dcf..23f6d2d73b 100644 --- a/newrelic/hooks/datastore_memcache.py +++ b/newrelic/hooks/datastore_memcache.py @@ -83,7 +83,7 @@ def _nr_datastore_trace_wrapper_(wrapped, instance, args, kwargs): def wrap_memcache_single(module, object_path, product, target, operation): - wrap_object(module, "Client.%s" % object_path, MemcacheSingleWrapper, (product, target, operation, module)) + wrap_object(module, f"Client.{object_path}", MemcacheSingleWrapper, (product, target, operation, module)) _memcache_client_methods = ( @@ -112,4 +112,4 @@ def instrument_memcache(module): for name in _memcache_multi_methods: if hasattr(module.Client, name): - wrap_datastore_trace(module, "Client.%s" % name, product="Memcached", target=None, operation=name) + wrap_datastore_trace(module, f"Client.{name}", product="Memcached", target=None, operation=name) diff --git a/newrelic/hooks/datastore_motor.py b/newrelic/hooks/datastore_motor.py index 1c000285ea..3cea77ac42 100644 --- a/newrelic/hooks/datastore_motor.py +++ b/newrelic/hooks/datastore_motor.py @@ -28,9 +28,7 @@ def _bind_params(name, *args, **kwargs): name = _bind_params(*args, **kwargs) if name.startswith('__') or name.startswith('_nr_'): - raise AttributeError('%s class has no attribute %s. To access ' - 'use object[%r].' % (instance.__class__.__name__, - name, name)) + raise AttributeError(f'{instance.__class__.__name__} class has no attribute {name}. To access use object[{name!r}].') return wrapped(*args, **kwargs) @@ -44,5 +42,5 @@ def patch_motor(module): 'MotorCollection'] for patched_class in patched_classes: if hasattr(module, patched_class): - wrap_function_wrapper(module, patched_class + '.__getattr__', + wrap_function_wrapper(module, f"{patched_class}.__getattr__", _nr_wrapper_Motor_getattr_) diff --git a/newrelic/hooks/datastore_pyelasticsearch.py b/newrelic/hooks/datastore_pyelasticsearch.py index 63e33a9bb7..2cfeeeacf6 100644 --- a/newrelic/hooks/datastore_pyelasticsearch.py +++ b/newrelic/hooks/datastore_pyelasticsearch.py @@ -15,7 +15,6 @@ from newrelic.api.datastore_trace import DatastoreTraceWrapper from newrelic.api.transaction import current_transaction from newrelic.common.object_wrapper import wrap_function_wrapper -from newrelic.packages import six # An index name can be a string, None or a sequence. In the case of None # an empty string or '*', it is the same as using '_all'. When a string @@ -27,7 +26,7 @@ def _index_name(index): if not index or index == "*": return "_all" - if not isinstance(index, six.string_types) or "," in index: + if not isinstance(index, str) or "," in index: return "other" return index @@ -102,7 +101,7 @@ def _nr_wrapper_ElasticSearch_method_(wrapped, instance, args, kwargs): return DatastoreTraceWrapper(wrapped, product="Elasticsearch", target=index, operation=name)(*args, **kwargs) if hasattr(module.ElasticSearch, name): - wrap_function_wrapper(module, "ElasticSearch.%s" % name, _nr_wrapper_ElasticSearch_method_) + wrap_function_wrapper(module, f"ElasticSearch.{name}", _nr_wrapper_ElasticSearch_method_) def instrument_pyelasticsearch_client(module): diff --git a/newrelic/hooks/datastore_pylibmc.py b/newrelic/hooks/datastore_pylibmc.py index 3d42a70fb6..81a3b582a9 100644 --- a/newrelic/hooks/datastore_pylibmc.py +++ b/newrelic/hooks/datastore_pylibmc.py @@ -40,4 +40,4 @@ def instrument_pylibmc_client(module): for name in _memcache_client_methods: if hasattr(module.Client, name): - wrap_datastore_trace(module, "Client.%s" % name, product="Memcached", target=None, operation=name) + wrap_datastore_trace(module, f"Client.{name}", product="Memcached", target=None, operation=name) diff --git a/newrelic/hooks/datastore_pymemcache.py b/newrelic/hooks/datastore_pymemcache.py index 690e95d616..1c754c0039 100644 --- a/newrelic/hooks/datastore_pymemcache.py +++ b/newrelic/hooks/datastore_pymemcache.py @@ -37,7 +37,25 @@ ) +def capture_host(self, *args, **kwargs): + if hasattr(self, "server") and self.server and len(self.server) >= 2: + return self.server[0] + + +def capture_port(self, *args, **kwargs): + if hasattr(self, "server") and self.server and len(self.server) >= 2: + return self.server[1] + + def instrument_pymemcache_client(module): for name in _memcache_client_methods: if hasattr(module.Client, name): - wrap_datastore_trace(module, "Client.%s" % name, product="Memcached", target=None, operation=name) + wrap_datastore_trace( + module, + f"Client.{name}", + product="Memcached", + target=None, + operation=name, + host=capture_host, + port_path_or_id=capture_port, + ) diff --git a/newrelic/hooks/datastore_pymongo.py b/newrelic/hooks/datastore_pymongo.py index c9c34b1fc3..80d2ab8c5c 100644 --- a/newrelic/hooks/datastore_pymongo.py +++ b/newrelic/hooks/datastore_pymongo.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys + from newrelic.api.datastore_trace import wrap_datastore_trace from newrelic.api.function_trace import wrap_function_trace @@ -59,34 +61,49 @@ ) -def instrument_pymongo_connection(module): - # Must name function explicitly as pymongo overrides the - # __getattr__() method in a way that breaks introspection. +def instrument_pymongo_pool(module): + # Exit early if this is a reimport of code from the newer module location + moved_module = "pymongo.synchronous.pool" + if module.__name__ != moved_module and moved_module in sys.modules: + return rollup = ("Datastore/all", "Datastore/MongoDB/all") + # Must name function explicitly as pymongo overrides the + # __getattr__() method in a way that breaks introspection. + wrap_function_trace( - module, "Connection.__init__", name="%s:Connection.__init__" % module.__name__, terminal=True, rollup=rollup + module, "Connection.__init__", name=f"{module.__name__}:Connection.__init__", terminal=True, rollup=rollup ) def instrument_pymongo_mongo_client(module): - # Must name function explicitly as pymongo overrides the - # __getattr__() method in a way that breaks introspection. + # Exit early if this is a reimport of code from the newer module location + moved_module = "pymongo.synchronous.mongo_client" + if module.__name__ != moved_module and moved_module in sys.modules: + return rollup = ("Datastore/all", "Datastore/MongoDB/all") + # Must name function explicitly as pymongo overrides the + # __getattr__() method in a way that breaks introspection. + wrap_function_trace( - module, "MongoClient.__init__", name="%s:MongoClient.__init__" % module.__name__, terminal=True, rollup=rollup + module, "MongoClient.__init__", name=f"{module.__name__}:MongoClient.__init__", terminal=True, rollup=rollup ) def instrument_pymongo_collection(module): + # Exit early if this is a reimport of code from the newer module location + moved_module = "pymongo.synchronous.collection" + if module.__name__ != moved_module and moved_module in sys.modules: + return + def _collection_name(collection, *args, **kwargs): return collection.name for name in _pymongo_client_methods: if hasattr(module.Collection, name): wrap_datastore_trace( - module, "Collection.%s" % name, product="MongoDB", target=_collection_name, operation=name + module, f"Collection.{name}", product="MongoDB", target=_collection_name, operation=name ) diff --git a/newrelic/hooks/datastore_pysolr.py b/newrelic/hooks/datastore_pysolr.py index 7d4e8697d8..301d87ae48 100644 --- a/newrelic/hooks/datastore_pysolr.py +++ b/newrelic/hooks/datastore_pysolr.py @@ -22,11 +22,11 @@ def instrument_pysolr(module): for name in _pysolr_client_methods: if hasattr(module.Solr, name): - wrap_datastore_trace(module, "Solr.%s" % name, product="Solr", target=None, operation=name) + wrap_datastore_trace(module, f"Solr.{name}", product="Solr", target=None, operation=name) if hasattr(module, "SolrCoreAdmin"): for name in _pysolr_admin_methods: if hasattr(module.SolrCoreAdmin, name): wrap_datastore_trace( - module, "SolrCoreAdmin.%s" % name, product="Solr", target=None, operation="admin.%s" % name + module, f"SolrCoreAdmin.{name}", product="Solr", target=None, operation=f"admin.{name}" ) diff --git a/newrelic/hooks/datastore_redis.py b/newrelic/hooks/datastore_redis.py index b32c848b35..ad14824195 100644 --- a/newrelic/hooks/datastore_redis.py +++ b/newrelic/hooks/datastore_redis.py @@ -14,14 +14,88 @@ import re -from newrelic.api.datastore_trace import DatastoreTrace +from newrelic.api.datastore_trace import ( + DatastoreTrace, + DatastoreTraceWrapper, + wrap_datastore_trace, +) +from newrelic.api.time_trace import current_trace from newrelic.api.transaction import current_transaction +from newrelic.common.async_wrapper import ( + async_generator_wrapper, + coroutine_wrapper, + generator_wrapper, +) from newrelic.common.object_wrapper import wrap_function_wrapper -_redis_client_methods = { +_redis_client_sync_methods = { + "acl_dryrun", + "auth", + "bgrewriteaof", + "bitfield", + "blmpop", + "bzmpop", + "client", + "command", + "command_docs", + "command_getkeysandflags", + "command_info", + "debug_segfault", + "expiretime", + "failover", + "hello", + "hexpire", + "hexpireat", + "hexpiretime", + "hpersist", + "hpexpire", + "hpexpireat", + "hpexpiretime", + "hpttl", + "httl", + "latency_doctor", + "latency_graph", + "latency_histogram", + "lcs", + "lpop", + "lpos", + "memory_doctor", + "memory_help", + "monitor", + "pexpiretime", + "psetex", + "psync", + "pubsub", + "renamenx", + "rpop", + "script_debug", + "sentinel_ckquorum", + "sentinel_failover", + "sentinel_flushconfig", + "sentinel_get_master_addr_by_name", + "sentinel_master", + "sentinel_masters", + "sentinel_monitor", + "sentinel_remove", + "sentinel_reset", + "sentinel_sentinels", + "sentinel_set", + "sentinel_slaves", + "shutdown", + "sort", + "sort_ro", + "spop", + "srandmember", + "unwatch", + "watch", + "zlexcount", + "zrevrangebyscore", +} + + +_redis_client_async_methods = { "acl_cat", "acl_deluser", - "acl_dryrun", "acl_genpass", "acl_getuser", "acl_help", @@ -50,11 +124,8 @@ "arrlen", "arrpop", "arrtrim", - "auth", - "bgrewriteaof", "bgsave", "bitcount", - "bitfield", "bitfield_ro", "bitop_and", "bitop_not", @@ -63,13 +134,11 @@ "bitop", "bitpos", "blmove", - "blmpop", "blpop", "brpop", "brpoplpush", "byrank", "byrevrank", - "bzmpop", "bzpopmax", "bzpopmin", "card", @@ -85,12 +154,12 @@ "client_no_evict", "client_pause", "client_reply", + "client_setinfo", "client_setname", "client_tracking", "client_trackinginfo", "client_unblock", "client_unpause", - "client", "cluster_add_slots", "cluster_addslots", "cluster_count_failure_report", @@ -117,10 +186,7 @@ "cluster_slots", "cluster", "command_count", - "command_docs", "command_getkeys", - "command_getkeysandflags", - "command_info", "command_list", "command", "commit", @@ -136,7 +202,6 @@ "createrule", "dbsize", "debug_object", - "debug_segfault", "debug_sleep", "debug", "decr", @@ -159,10 +224,8 @@ "exists", "expire", "expireat", - "expiretime", "explain_cli", "explain", - "failover", "fcall_ro", "fcall", "flushall", @@ -176,6 +239,7 @@ "function_load", "function_restore", "function_stats", + "gears_refresh_cluster", "geoadd", "geodist", "geohash", @@ -191,7 +255,6 @@ "getrange", "getset", "hdel", - "hello", "hexists", "hget", "hgetall", @@ -203,7 +266,7 @@ "hmset_dict", "hmset", "hrandfield", - "hscan_inter", + "hscan_iter", "hscan", "hset", "hsetnx", @@ -219,13 +282,9 @@ "insertnx", "keys", "lastsave", - "latency_doctor", - "latency_graph", - "latency_histogram", "latency_history", "latency_latest", "latency_reset", - "lcs", "lindex", "linsert", "list", @@ -234,8 +293,6 @@ "lmpop", "loadchunk", "lolwut", - "lpop", - "lpos", "lpush", "lpushx", "lrange", @@ -244,8 +301,6 @@ "ltrim", "madd", "max", - "memory_doctor", - "memory_help", "memory_malloc_stats", "memory_purge", "memory_stats", @@ -260,7 +315,6 @@ "module_load", "module_loadex", "module_unload", - "monitor", "move", "mrange", "mrevrange", @@ -276,21 +330,19 @@ "persist", "pexpire", "pexpireat", - "pexpiretime", "pfadd", "pfcount", "pfmerge", "ping", "profile", - "psetex", "psubscribe", - "psync", "pttl", "publish", "pubsub_channels", "pubsub_numpat", "pubsub_numsub", - "pubsub", + "pubsub_shardchannels", + "pubsub_shardnumsub", "punsubscribe", "quantile", "query", @@ -302,7 +354,6 @@ "readonly", "readwrite", "rename", - "renamenx", "replicaof", "reserve", "reset", @@ -311,7 +362,6 @@ "revrange", "revrank", "role", - "rpop", "rpoplpush", "rpush", "rpushx", @@ -321,7 +371,6 @@ "scan", "scandump", "scard", - "script_debug", "script_exists", "script_flush", "script_kill", @@ -330,24 +379,11 @@ "sdiffstore", "search", "select", - "sentinel_ckquorum", - "sentinel_failover", - "sentinel_flushconfig", - "sentinel_get_master_addr_by_name", - "sentinel_master", - "sentinel_masters", - "sentinel_monitor", - "sentinel_remove", - "sentinel_reset", - "sentinel_sentinels", - "sentinel_set", - "sentinel_slaves", "set", "setbit", "setex", "setnx", "setrange", - "shutdown", "sinter", "sintercard", "sinterstore", @@ -360,11 +396,8 @@ "smembers", "smismember", "smove", - "sort_ro", - "sort", "spellcheck", - "spop", - "srandmember", + "spublish", "srem", "sscan_iter", "sscan", @@ -384,6 +417,11 @@ "syndump", "synupdate", "tagvals", + "tfcall_async", + "tfcall", + "tfunction_delete", + "tfunction_list", + "tfunction_load", "time", "toggle", "touch", @@ -392,9 +430,8 @@ "type", "unlink", "unsubscribe", - "unwatch", "wait", - "watch", + "waitaof", "xack", "xadd", "xautoclaim", @@ -430,7 +467,6 @@ "zinter", "zintercard", "zinterstore", - "zlexcount", "zmpop", "zmscore", "zpopmax", @@ -447,7 +483,6 @@ "zremrangebyscore", "zrevrange", "zrevrangebylex", - "zrevrangebyscore", "zrevrank", "zscan_iter", "zscan", @@ -456,6 +491,15 @@ "zunionstore", } +_redis_client_gen_methods = { + "scan_iter", + "hscan_iter", + "sscan_iter", + "zscan_iter", +} + +_redis_client_methods = _redis_client_sync_methods.union(_redis_client_async_methods) + _redis_multipart_commands = set(["client", "cluster", "command", "config", "debug", "sentinel", "slowlog", "script"]) _redis_operation_re = re.compile(r"[-\s]+") @@ -479,28 +523,87 @@ def _instance_info(kwargs): def _wrap_Redis_method_wrapper_(module, instance_class_name, operation): - def _nr_wrapper_Redis_method_(wrapped, instance, args, kwargs): - transaction = current_transaction() + name = f"{instance_class_name}.{operation}" + if operation in _redis_client_gen_methods: + async_wrapper = generator_wrapper + else: + async_wrapper = None - if transaction is None: + wrap_datastore_trace(module, name, product="Redis", target=None, operation=operation, async_wrapper=async_wrapper) + + +def _wrap_asyncio_Redis_method_wrapper(module, instance_class_name, operation): + def _nr_wrapper_asyncio_Redis_method_(wrapped, instance, args, kwargs): + from redis.asyncio.client import Pipeline + + if isinstance(instance, Pipeline): return wrapped(*args, **kwargs) - dt = DatastoreTrace(product="Redis", target=None, operation=operation, source=wrapped) + # Method should be run when awaited or iterated, therefore we wrap in an async wrapper. + return DatastoreTraceWrapper( + wrapped, product="Redis", target=None, operation=operation, async_wrapper=async_wrapper + )(*args, **kwargs) - transaction._nr_datastore_instance_info = (None, None, None) + name = f"{instance_class_name}.{operation}" + if operation in _redis_client_gen_methods: + async_wrapper = async_generator_wrapper + else: + async_wrapper = coroutine_wrapper - with dt: - result = wrapped(*args, **kwargs) + wrap_function_wrapper(module, name, _nr_wrapper_asyncio_Redis_method_) - host, port_path_or_id, db = transaction._nr_datastore_instance_info - dt.host = host - dt.port_path_or_id = port_path_or_id - dt.database_name = db - return result +async def wrap_async_Connection_send_command(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) - name = "%s.%s" % (instance_class_name, operation) - wrap_function_wrapper(module, name, _nr_wrapper_Redis_method_) + host, port_path_or_id, db = (None, None, None) + + try: + dt = transaction.settings.datastore_tracer + if dt.instance_reporting.enabled or dt.database_name_reporting.enabled: + conn_kwargs = _conn_attrs_to_dict(instance) + host, port_path_or_id, db = _instance_info(conn_kwargs) + except Exception: + pass + + # Older Redis clients would when sending multi part commands pass + # them in as separate arguments to send_command(). Need to therefore + # detect those and grab the next argument from the set of arguments. + + operation = args[0].strip().lower() + + # If it's not a multi part command, there's no need to trace it, so + # we can return early. + + if ( + operation.split()[0] not in _redis_multipart_commands + ): # Set the datastore info on the DatastoreTrace containing this function call. + trace = current_trace() + + # Find DatastoreTrace no matter how many other traces are inbetween + while trace is not None and not isinstance(trace, DatastoreTrace): + trace = getattr(trace, "parent", None) + + if trace is not None: + trace.host = host + trace.port_path_or_id = port_path_or_id + trace.database_name = db + + return await wrapped(*args, **kwargs) + + # Convert multi args to single arg string + + if operation in _redis_multipart_commands and len(args) > 1: + operation = f"{operation} {args[1].strip().lower()}" + + operation = _redis_operation_re.sub("_", operation) + + with DatastoreTrace( + product="Redis", target=None, operation=operation, host=host, port_path_or_id=port_path_or_id, database_name=db + ): + return await wrapped(*args, **kwargs) def _nr_Connection_send_command_wrapper_(wrapped, instance, args, kwargs): @@ -519,7 +622,15 @@ def _nr_Connection_send_command_wrapper_(wrapped, instance, args, kwargs): except: pass - transaction._nr_datastore_instance_info = (host, port_path_or_id, db) + # Find DatastoreTrace no matter how many other traces are inbetween + trace = current_trace() + while trace is not None and not isinstance(trace, DatastoreTrace): + trace = getattr(trace, "parent", None) + + if trace is not None: + trace.host = host + trace.port_path_or_id = port_path_or_id + trace.database_name = db # Older Redis clients would when sending multi part commands pass # them in as separate arguments to send_command(). Need to therefore @@ -536,7 +647,7 @@ def _nr_Connection_send_command_wrapper_(wrapped, instance, args, kwargs): # Convert multi args to single arg string if operation in _redis_multipart_commands and len(args) > 1: - operation = "%s %s" % (operation, args[1].strip().lower()) + operation = f"{operation} {args[1].strip().lower()}" operation = _redis_operation_re.sub("_", operation) @@ -564,6 +675,14 @@ def instrument_redis_client(module): _wrap_Redis_method_wrapper_(module, "Redis", name) +def instrument_asyncio_redis_client(module): + if hasattr(module, "Redis"): + class_ = getattr(module, "Redis") + for operation in _redis_client_async_methods: + if hasattr(class_, operation): + _wrap_asyncio_Redis_method_wrapper(module, "Redis", operation) + + def instrument_redis_commands_core(module): _instrument_redis_commands_module(module, "CoreCommands") @@ -596,6 +715,10 @@ def instrument_redis_commands_bf_commands(module): _instrument_redis_commands_module(module, "TOPKCommands") +def instrument_redis_commands_cluster(module): + _instrument_redis_commands_module(module, "RedisClusterCommands") + + def _instrument_redis_commands_module(module, class_name): for name in _redis_client_methods: if hasattr(module, class_name): @@ -605,4 +728,12 @@ def _instrument_redis_commands_module(module, class_name): def instrument_redis_connection(module): - wrap_function_wrapper(module, "Connection.send_command", _nr_Connection_send_command_wrapper_) + if hasattr(module, "Connection"): + if hasattr(module.Connection, "send_command"): + wrap_function_wrapper(module, "Connection.send_command", _nr_Connection_send_command_wrapper_) + + +def instrument_asyncio_redis_connection(module): + if hasattr(module, "Connection"): + if hasattr(module.Connection, "send_command"): + wrap_function_wrapper(module, "Connection.send_command", wrap_async_Connection_send_command) diff --git a/newrelic/hooks/datastore_solrpy.py b/newrelic/hooks/datastore_solrpy.py index 74e808ae50..2ba0a72507 100644 --- a/newrelic/hooks/datastore_solrpy.py +++ b/newrelic/hooks/datastore_solrpy.py @@ -30,4 +30,4 @@ def instrument_solrpy(module): for name in _solrpy_client_methods: if hasattr(module.SolrConnection, name): - wrap_datastore_trace(module, "SolrConnection.%s" % name, product="Solr", target=None, operation=name) + wrap_datastore_trace(module, f"SolrConnection.{name}", product="Solr", target=None, operation=name) diff --git a/newrelic/hooks/datastore_umemcache.py b/newrelic/hooks/datastore_umemcache.py deleted file mode 100644 index 02ff073d2f..0000000000 --- a/newrelic/hooks/datastore_umemcache.py +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from newrelic.api.datastore_trace import datastore_trace -from newrelic.common.object_wrapper import wrap_function_wrapper, ObjectProxy - -class _nr_umemcache_Client_proxy_(ObjectProxy): - - @datastore_trace('Memcached', None, 'set') - def set(self, *args, **kwargs): - return self.__wrapped__.set(*args, **kwargs) - - @datastore_trace('Memcached', None, 'get') - def get(self, *args, **kwargs): - return self.__wrapped__.get(*args, **kwargs) - - @datastore_trace('Memcached', None, 'gets') - def gets(self, *args, **kwargs): - return self.__wrapped__.gets(*args, **kwargs) - - @datastore_trace('Memcached', None, 'get_multi') - def get_multi(self, *args, **kwargs): - return self.__wrapped__.get_multi(*args, **kwargs) - - @datastore_trace('Memcached', None, 'gets_multi') - def gets_multi(self, *args, **kwargs): - return self.__wrapped__.gets_multi(*args, **kwargs) - - @datastore_trace('Memcached', None, 'add') - def add(self, *args, **kwargs): - return self.__wrapped__.add(*args, **kwargs) - - @datastore_trace('Memcached', None, 'replace') - def replace(self, *args, **kwargs): - return self.__wrapped__.replace(*args, **kwargs) - - @datastore_trace('Memcached', None, 'append') - def append(self, *args, **kwargs): - return self.__wrapped__.append(*args, **kwargs) - - @datastore_trace('Memcached', None, 'prepend') - def prepend(self, *args, **kwargs): - return self.__wrapped__.prepend(*args, **kwargs) - - @datastore_trace('Memcached', None, 'delete') - def delete(self, *args, **kwargs): - return self.__wrapped__.delete(*args, **kwargs) - - @datastore_trace('Memcached', None, 'cas') - def cas(self, *args, **kwargs): - return self.__wrapped__.cas(*args, **kwargs) - - @datastore_trace('Memcached', None, 'incr') - def incr(self, *args, **kwargs): - return self.__wrapped__.incr(*args, **kwargs) - - @datastore_trace('Memcached', None, 'decr') - def decr(self, *args, **kwargs): - return self.__wrapped__.decr(*args, **kwargs) - - @datastore_trace('Memcached', None, 'stats') - def stats(self, *args, **kwargs): - return self.__wrapped__.stats(*args, **kwargs) - -def _nr_umemcache_Client_wrapper_(wrapped, instance, args, kwargs): - return _nr_umemcache_Client_proxy_(wrapped(*args, **kwargs)) - -def instrument_umemcache(module): - wrap_function_wrapper(module, 'Client', _nr_umemcache_Client_wrapper_) diff --git a/newrelic/hooks/external_aiobotocore.py b/newrelic/hooks/external_aiobotocore.py new file mode 100644 index 0000000000..7c8be883b0 --- /dev/null +++ b/newrelic/hooks/external_aiobotocore.py @@ -0,0 +1,48 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.api.external_trace import ExternalTrace +from newrelic.common.object_wrapper import wrap_function_wrapper + + +def _bind_make_request_params(operation_model, request_dict, *args, **kwargs): + return operation_model, request_dict + + +def bind__send_request(request_dict, operation_model, *args, **kwargs): + return operation_model, request_dict + + +async def wrap_endpoint_make_request(wrapped, instance, args, kwargs): + operation_model, request_dict = _bind_make_request_params(*args, **kwargs) + url = request_dict.get("url") + method = request_dict.get("method") + + with ExternalTrace(library="aiobotocore", url=url, method=method, source=wrapped) as trace: + try: + trace._add_agent_attribute("aws.operation", operation_model.name) + except: + pass + + result = await wrapped(*args, **kwargs) + try: + request_id = result[1]["ResponseMetadata"]["RequestId"] + trace._add_agent_attribute("aws.requestId", request_id) + except: + pass + return result + + +def instrument_aiobotocore_endpoint(module): + wrap_function_wrapper(module, "AioEndpoint.make_request", wrap_endpoint_make_request) diff --git a/newrelic/hooks/external_botocore.py b/newrelic/hooks/external_botocore.py index 7d49fbd031..dc25d63181 100644 --- a/newrelic/hooks/external_botocore.py +++ b/newrelic/hooks/external_botocore.py @@ -12,15 +12,66 @@ # See the License for the specific language governing permissions and # limitations under the License. -from newrelic.api.message_trace import message_trace +import json +import logging +import re +import sys +import traceback +import uuid +from io import BytesIO + +from botocore.response import StreamingBody + from newrelic.api.datastore_trace import datastore_trace from newrelic.api.external_trace import ExternalTrace -from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.message_trace import MessageTrace, message_trace +from newrelic.api.time_trace import current_trace, get_trace_linking_metadata +from newrelic.api.transaction import current_transaction +from newrelic.common.async_wrapper import async_wrapper as get_async_wrapper +from newrelic.common.object_wrapper import ( + ObjectProxy, + function_wrapper, + wrap_function_wrapper, +) +from newrelic.common.package_version_utils import get_package_version +from newrelic.core.config import global_settings + +QUEUE_URL_PATTERN = re.compile(r"https://sqs.([\w\d-]+).amazonaws.com/(\d+)/([^/]+)") +BOTOCORE_VERSION = get_package_version("botocore") + + +_logger = logging.getLogger(__name__) + +EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE = "Exception occurred in botocore instrumentation for AWS Bedrock: While reporting an exception in botocore, another exception occurred. Report this issue to New Relic Support.\n%s" +REQUEST_EXTACTOR_FAILURE_LOG_MESSAGE = "Exception occurred in botocore instrumentation for AWS Bedrock: Failed to extract request information. Report this issue to New Relic Support.\n%s" +RESPONSE_EXTRACTOR_FAILURE_LOG_MESSAGE = "Exception occurred in botocore instrumentation for AWS Bedrock: Failed to extract response information. If the issue persists, report this issue to New Relic support.\n%s" +RESPONSE_PROCESSING_FAILURE_LOG_MESSAGE = "Exception occurred in botocore instrumentation for AWS Bedrock: Failed to report response data. Report this issue to New Relic Support.\n%s" +EMBEDDING_STREAMING_UNSUPPORTED_LOG_MESSAGE = "Response streaming with embedding models is unsupported in botocore instrumentation for AWS Bedrock. If this feature is now supported by AWS and botocore, report this issue to New Relic Support." + +UNSUPPORTED_MODEL_WARNING_SENT = False def extract_sqs(*args, **kwargs): - queue_value = kwargs.get('QueueUrl', 'Unknown') - return queue_value.rsplit('/', 1)[-1] + queue_value = kwargs.get("QueueUrl", "Unknown") + return queue_value.rsplit("/", 1)[-1] + + +def extract_sqs_agent_attrs(*args, **kwargs): + # Try to capture AWS SQS info as agent attributes. Log any exception to debug. + agent_attrs = {} + try: + queue_url = kwargs.get("QueueUrl") + if queue_url: + m = QUEUE_URL_PATTERN.match(queue_url) + if m: + agent_attrs["messaging.system"] = "aws_sqs" + agent_attrs["cloud.region"] = m.group(1) + agent_attrs["cloud.account.id"] = m.group(2) + agent_attrs["messaging.destination.name"] = m.group(3) + except Exception as e: + _logger.debug("Failed to capture AWS SQS info.", exc_info=True) + return agent_attrs def extract(argument_names, default=None): @@ -40,43 +91,836 @@ def extractor_string(*args, **kwargs): return extractor_list +def bedrock_error_attributes(exception, bedrock_attrs): + response = getattr(exception, "response", None) + if not response: + return bedrock_attrs + + response_metadata = response.get("ResponseMetadata", {}) + response_error = response.get("Error", {}) + bedrock_attrs.update( + { + "request_id": response_metadata.get("RequestId"), + "http.statusCode": response_metadata.get("HTTPStatusCode"), + "error.message": response_error.get("Message"), + "error.code": response_error.get("Code"), + "error": True, + } + ) + return bedrock_attrs + + +def create_chat_completion_message_event( + transaction, + input_message_list, + output_message_list, + chat_completion_id, + span_id, + trace_id, + request_model, + request_id, + llm_metadata_dict, + response_id=None, +): + if not transaction: + return + + settings = transaction.settings if transaction.settings is not None else global_settings() + + for index, message in enumerate(input_message_list): + content = message.get("content", "") + + if response_id: + id_ = f"{response_id}-{int(index)}" # Response ID was set, append message index to it. + else: + id_ = str(uuid.uuid4()) # No response IDs, use random UUID + + chat_completion_message_dict = { + "id": id_, + "request_id": request_id, + "span_id": span_id, + "trace_id": trace_id, + "token_count": ( + settings.ai_monitoring.llm_token_count_callback(request_model, content) + if settings.ai_monitoring.llm_token_count_callback + else None + ), + "role": message.get("role"), + "completion_id": chat_completion_id, + "sequence": index, + "response.model": request_model, + "vendor": "bedrock", + "ingest_source": "Python", + } + + if settings.ai_monitoring.record_content.enabled: + chat_completion_message_dict["content"] = content + + chat_completion_message_dict.update(llm_metadata_dict) + + transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_message_dict) + + for index, message in enumerate(output_message_list): + index += len(input_message_list) + content = message.get("content", "") + # For anthropic models run via langchain, a list is returned with a dictionary of content inside + # We only want to report the raw dictionary in the LLM message event + if isinstance(content, list) and len(content) == 1: + content = content[0] + + if response_id: + id_ = f"{response_id}-{int(index)}" # Response ID was set, append message index to it. + else: + id_ = str(uuid.uuid4()) # No response IDs, use random UUID + + chat_completion_message_dict = { + "id": id_, + "request_id": request_id, + "span_id": span_id, + "trace_id": trace_id, + "token_count": ( + settings.ai_monitoring.llm_token_count_callback(request_model, content) + if settings.ai_monitoring.llm_token_count_callback + else None + ), + "role": message.get("role"), + "completion_id": chat_completion_id, + "sequence": index, + "response.model": request_model, + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + } + + if settings.ai_monitoring.record_content.enabled: + chat_completion_message_dict["content"] = content + + chat_completion_message_dict.update(llm_metadata_dict) + + transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_message_dict) + + +def extract_bedrock_titan_text_model_request(request_body, bedrock_attrs): + request_body = json.loads(request_body) + request_config = request_body.get("textGenerationConfig", {}) + + input_message_list = [{"role": "user", "content": request_body.get("inputText")}] + + bedrock_attrs["input_message_list"] = input_message_list + bedrock_attrs["request.max_tokens"] = request_config.get("maxTokenCount") + bedrock_attrs["request.temperature"] = request_config.get("temperature") + + return bedrock_attrs + + +def extract_bedrock_mistral_text_model_request(request_body, bedrock_attrs): + request_body = json.loads(request_body) + bedrock_attrs["input_message_list"] = [{"role": "user", "content": request_body.get("prompt")}] + bedrock_attrs["request.max_tokens"] = request_body.get("max_tokens") + bedrock_attrs["request.temperature"] = request_body.get("temperature") + return bedrock_attrs + + +def extract_bedrock_titan_text_model_response(response_body, bedrock_attrs): + if response_body: + response_body = json.loads(response_body) + + output_message_list = [ + {"role": "assistant", "content": result["outputText"]} for result in response_body.get("results", []) + ] + + bedrock_attrs["response.choices.finish_reason"] = response_body["results"][0]["completionReason"] + bedrock_attrs["output_message_list"] = output_message_list + + return bedrock_attrs + + +def extract_bedrock_mistral_text_model_response(response_body, bedrock_attrs): + if response_body: + response_body = json.loads(response_body) + outputs = response_body.get("outputs") + if outputs: + bedrock_attrs["response.choices.finish_reason"] = outputs[0]["stop_reason"] + bedrock_attrs["output_message_list"] = [ + {"role": "assistant", "content": result["text"]} for result in outputs + ] + return bedrock_attrs + + +def extract_bedrock_titan_text_model_streaming_response(response_body, bedrock_attrs): + if response_body: + if "outputText" in response_body: + bedrock_attrs["output_message_list"] = messages = bedrock_attrs.get("output_message_list", []) + messages.append({"role": "assistant", "content": response_body["outputText"]}) + + bedrock_attrs["response.choices.finish_reason"] = response_body.get("completionReason", None) + + return bedrock_attrs + + +def extract_bedrock_mistral_text_model_streaming_response(response_body, bedrock_attrs): + if response_body: + outputs = response_body.get("outputs") + if outputs: + bedrock_attrs["output_message_list"] = bedrock_attrs.get( + "output_message_list", [{"role": "assistant", "content": ""}] + ) + bedrock_attrs["output_message_list"][0]["content"] += outputs[0].get("text", "") + bedrock_attrs["response.choices.finish_reason"] = outputs[0].get("stop_reason", None) + return bedrock_attrs + + +def extract_bedrock_titan_embedding_model_request(request_body, bedrock_attrs): + request_body = json.loads(request_body) + + bedrock_attrs["input"] = request_body.get("inputText") + + return bedrock_attrs + + +def extract_bedrock_cohere_embedding_model_request(request_body, bedrock_attrs): + request_body = json.loads(request_body) + + bedrock_attrs["input"] = request_body.get("texts") + + return bedrock_attrs + + +def extract_bedrock_ai21_j2_model_request(request_body, bedrock_attrs): + request_body = json.loads(request_body) + + input_message_list = [{"role": "user", "content": request_body.get("prompt")}] + + bedrock_attrs["request.max_tokens"] = request_body.get("maxTokens") + bedrock_attrs["request.temperature"] = request_body.get("temperature") + bedrock_attrs["input_message_list"] = input_message_list + + return bedrock_attrs + + +def extract_bedrock_ai21_j2_model_response(response_body, bedrock_attrs): + if response_body: + response_body = json.loads(response_body) + output_message_list = [ + {"role": "assistant", "content": result["data"]["text"]} for result in response_body.get("completions", []) + ] + + bedrock_attrs["response.choices.finish_reason"] = response_body["completions"][0]["finishReason"]["reason"] + bedrock_attrs["output_message_list"] = output_message_list + bedrock_attrs["response_id"] = str(response_body.get("id")) + + return bedrock_attrs + + +def extract_bedrock_claude_model_request(request_body, bedrock_attrs): + request_body = json.loads(request_body) + + if "messages" in request_body: + input_message_list = [ + {"role": message.get("role", "user"), "content": message.get("content")} + for message in request_body.get("messages") + ] + else: + input_message_list = [{"role": "user", "content": request_body.get("prompt")}] + bedrock_attrs["request.max_tokens"] = request_body.get("max_tokens_to_sample") + bedrock_attrs["request.temperature"] = request_body.get("temperature") + bedrock_attrs["input_message_list"] = input_message_list + + return bedrock_attrs + + +def extract_bedrock_claude_model_response(response_body, bedrock_attrs): + if response_body: + response_body = json.loads(response_body) + role = response_body.get("role", "assistant") + content = response_body.get("content") or response_body.get("completion") + output_message_list = [{"role": role, "content": content}] + bedrock_attrs["response.choices.finish_reason"] = response_body.get("stop_reason") + bedrock_attrs["output_message_list"] = output_message_list + + return bedrock_attrs + + +def extract_bedrock_claude_model_streaming_response(response_body, bedrock_attrs): + if response_body: + content = response_body.get("completion", "") or (response_body.get("delta") or {}).get("text", "") + if "output_message_list" not in bedrock_attrs: + bedrock_attrs["output_message_list"] = [{"role": "assistant", "content": ""}] + bedrock_attrs["output_message_list"][0]["content"] += content + bedrock_attrs["response.choices.finish_reason"] = response_body.get("stop_reason") + return bedrock_attrs + + +def extract_bedrock_llama_model_request(request_body, bedrock_attrs): + request_body = json.loads(request_body) + + input_message_list = [{"role": "user", "content": request_body.get("prompt")}] + + bedrock_attrs["request.max_tokens"] = request_body.get("max_gen_len") + bedrock_attrs["request.temperature"] = request_body.get("temperature") + bedrock_attrs["input_message_list"] = input_message_list + + return bedrock_attrs + + +def extract_bedrock_llama_model_response(response_body, bedrock_attrs): + if response_body: + response_body = json.loads(response_body) + + output_message_list = [{"role": "assistant", "content": response_body.get("generation")}] + bedrock_attrs["response.choices.finish_reason"] = response_body.get("stop_reason") + bedrock_attrs["output_message_list"] = output_message_list + + return bedrock_attrs + + +def extract_bedrock_llama_model_streaming_response(response_body, bedrock_attrs): + if response_body: + content = response_body.get("generation") + if "output_message_list" not in bedrock_attrs: + bedrock_attrs["output_message_list"] = [{"role": "assistant", "content": ""}] + bedrock_attrs["output_message_list"][0]["content"] += content + bedrock_attrs["response.choices.finish_reason"] = response_body.get("stop_reason") + return bedrock_attrs + + +def extract_bedrock_cohere_model_request(request_body, bedrock_attrs): + request_body = json.loads(request_body) + + input_message_list = [{"role": "user", "content": request_body.get("prompt")}] + + bedrock_attrs["request.max_tokens"] = request_body.get("max_tokens") + bedrock_attrs["request.temperature"] = request_body.get("temperature") + bedrock_attrs["input_message_list"] = input_message_list + + return bedrock_attrs + + +def extract_bedrock_cohere_model_response(response_body, bedrock_attrs): + if response_body: + response_body = json.loads(response_body) + + output_message_list = [ + {"role": "assistant", "content": result["text"]} for result in response_body.get("generations", []) + ] + + bedrock_attrs["response.choices.finish_reason"] = response_body["generations"][0]["finish_reason"] + bedrock_attrs["output_message_list"] = output_message_list + bedrock_attrs["response_id"] = str(response_body.get("id")) + + return bedrock_attrs + + +def extract_bedrock_cohere_model_streaming_response(response_body, bedrock_attrs): + if response_body: + bedrock_attrs["output_message_list"] = messages = bedrock_attrs.get("output_message_list", []) + messages.extend( + [{"role": "assistant", "content": result["text"]} for result in response_body.get("generations", [])] + ) + + bedrock_attrs["response.choices.finish_reason"] = response_body["generations"][0]["finish_reason"] + bedrock_attrs["response_id"] = str(response_body.get("id")) + + return bedrock_attrs + + +NULL_EXTRACTOR = lambda *args: {} # Empty extractor that returns nothing +MODEL_EXTRACTORS = [ # Order is important here, avoiding dictionaries + ( + "amazon.titan-embed", + extract_bedrock_titan_embedding_model_request, + NULL_EXTRACTOR, + NULL_EXTRACTOR, + ), + ( + "cohere.embed", + extract_bedrock_cohere_embedding_model_request, + NULL_EXTRACTOR, + NULL_EXTRACTOR, + ), + ( + "amazon.titan", + extract_bedrock_titan_text_model_request, + extract_bedrock_titan_text_model_response, + extract_bedrock_titan_text_model_streaming_response, + ), + ("ai21.j2", extract_bedrock_ai21_j2_model_request, extract_bedrock_ai21_j2_model_response, NULL_EXTRACTOR), + ( + "cohere", + extract_bedrock_cohere_model_request, + extract_bedrock_cohere_model_response, + extract_bedrock_cohere_model_streaming_response, + ), + ( + "anthropic.claude", + extract_bedrock_claude_model_request, + extract_bedrock_claude_model_response, + extract_bedrock_claude_model_streaming_response, + ), + ( + "meta.llama", + extract_bedrock_llama_model_request, + extract_bedrock_llama_model_response, + extract_bedrock_llama_model_streaming_response, + ), + ( + "mistral", + extract_bedrock_mistral_text_model_request, + extract_bedrock_mistral_text_model_response, + extract_bedrock_mistral_text_model_streaming_response, + ), +] + + +def wrap_bedrock_runtime_invoke_model(response_streaming=False): + @function_wrapper + def _wrap_bedrock_runtime_invoke_model(wrapped, instance, args, kwargs): + # Wrapped function only takes keyword arguments, no need for binding + transaction = current_transaction() + + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + transaction.add_ml_model_info("Bedrock", BOTOCORE_VERSION) + transaction._add_agent_attribute("llm", True) + + # Read and replace request file stream bodies + request_body = kwargs["body"] + if hasattr(request_body, "read"): + request_body = request_body.read() + kwargs["body"] = request_body + + # Determine model to be used with extractor + model = kwargs.get("modelId") + if not model: + return wrapped(*args, **kwargs) + + is_embedding = "embed" in model + + # Determine extractor by model type + for extractor_name, request_extractor, response_extractor, stream_extractor in MODEL_EXTRACTORS: + if model.startswith(extractor_name): + break + else: + # Model was not found in extractor list + global UNSUPPORTED_MODEL_WARNING_SENT + if not UNSUPPORTED_MODEL_WARNING_SENT: + # Only send warning once to avoid spam + _logger.warning( + "Unsupported Amazon Bedrock model in use (%s). Upgrade to a newer version of the agent, and contact New Relic support if the issue persists.", + model, + ) + UNSUPPORTED_MODEL_WARNING_SENT = True + + request_extractor = response_extractor = stream_extractor = NULL_EXTRACTOR + + function_name = wrapped.__name__ + operation = "embedding" if is_embedding else "completion" + + # Function trace may not be exited in this function in the case of streaming, so start manually + ft = FunctionTrace(name=function_name, group=f"Llm/{operation}/Bedrock") + ft.__enter__() + + # Get trace information + available_metadata = get_trace_linking_metadata() + span_id = available_metadata.get("span.id") + trace_id = available_metadata.get("trace.id") + + try: + response = wrapped(*args, **kwargs) + except Exception as exc: + try: + bedrock_attrs = { + "model": model, + "span_id": span_id, + "trace_id": trace_id, + } + try: + request_extractor(request_body, bedrock_attrs) + except json.decoder.JSONDecodeError: + pass + except Exception: + _logger.warning(REQUEST_EXTACTOR_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + error_attributes = bedrock_error_attributes(exc, bedrock_attrs) + notice_error_attributes = { + "http.statusCode": error_attributes.get("http.statusCode"), + "error.message": error_attributes.get("error.message"), + "error.code": error_attributes.get("error.code"), + } + + if is_embedding: + notice_error_attributes.update({"embedding_id": str(uuid.uuid4())}) + else: + notice_error_attributes.update({"completion_id": str(uuid.uuid4())}) + + ft.notice_error( + attributes=notice_error_attributes, + ) + + ft.__exit__(*sys.exc_info()) + error_attributes["duration"] = ft.duration * 1000 + + if operation == "embedding": + handle_embedding_event(transaction, error_attributes) + else: + handle_chat_completion_event(transaction, error_attributes) + except Exception: + _logger.warning(EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + raise + + if not response or response_streaming and not settings.ai_monitoring.streaming.enabled: + ft.__exit__(None, None, None) + return response + + if response_streaming and operation == "embedding": + # This combination is not supported at time of writing, but may become + # a supported feature in the future. Instrumentation will need to be written + # if this becomes available. + _logger.warning(EMBEDDING_STREAMING_UNSUPPORTED_LOG_MESSAGE) + ft.__exit__(None, None, None) + return response + + response_headers = response.get("ResponseMetadata", {}).get("HTTPHeaders") or {} + bedrock_attrs = { + "request_id": response_headers.get("x-amzn-requestid"), + "model": model, + "span_id": span_id, + "trace_id": trace_id, + } + + try: + request_extractor(request_body, bedrock_attrs) + except json.decoder.JSONDecodeError: + pass + except Exception: + _logger.warning(REQUEST_EXTACTOR_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + try: + if response_streaming: + # Wrap EventStream object here to intercept __iter__ method instead of instrumenting class. + # This class is used in numerous other services in botocore, and would cause conflicts. + response["body"] = body = EventStreamWrapper(response["body"]) + body._nr_ft = ft + body._nr_bedrock_attrs = bedrock_attrs + body._nr_model_extractor = stream_extractor + return response + + # Read and replace response streaming bodies + response_body = response["body"].read() + ft.__exit__(None, None, None) + bedrock_attrs["duration"] = ft.duration * 1000 + response["body"] = StreamingBody(BytesIO(response_body), len(response_body)) + + # Run response extractor for non-streaming responses + try: + response_extractor(response_body, bedrock_attrs) + except Exception: + _logger.warning(RESPONSE_EXTRACTOR_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + if operation == "embedding": + handle_embedding_event(transaction, bedrock_attrs) + else: + handle_chat_completion_event(transaction, bedrock_attrs) + + except Exception: + _logger.warning(RESPONSE_PROCESSING_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + return response + + return _wrap_bedrock_runtime_invoke_model + + +class EventStreamWrapper(ObjectProxy): + def __iter__(self): + g = GeneratorProxy(self.__wrapped__.__iter__()) + g._nr_ft = getattr(self, "_nr_ft", None) + g._nr_bedrock_attrs = getattr(self, "_nr_bedrock_attrs", {}) + g._nr_model_extractor = getattr(self, "_nr_model_extractor", NULL_EXTRACTOR) + return g + + +class GeneratorProxy(ObjectProxy): + def __init__(self, wrapped): + super(GeneratorProxy, self).__init__(wrapped) + + def __iter__(self): + return self + + def __next__(self): + transaction = current_transaction() + if not transaction: + return self.__wrapped__.__next__() + + return_val = None + try: + return_val = self.__wrapped__.__next__() + record_stream_chunk(self, return_val, transaction) + except StopIteration: + record_events_on_stop_iteration(self, transaction) + raise + except Exception as exc: + record_error(self, transaction, exc) + raise + return return_val + + def close(self): + return super(GeneratorProxy, self).close() + + +def record_stream_chunk(self, return_val, transaction): + if return_val: + try: + chunk = json.loads(return_val["chunk"]["bytes"].decode("utf-8")) + self._nr_model_extractor(chunk, self._nr_bedrock_attrs) + # In Langchain, the bedrock iterator exits early if type is "content_block_stop". + # So we need to call the record events here since stop iteration will not be raised. + _type = chunk.get("type") + if _type == "content_block_stop": + record_events_on_stop_iteration(self, transaction) + except Exception: + _logger.warning(RESPONSE_EXTRACTOR_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +def record_events_on_stop_iteration(self, transaction): + if hasattr(self, "_nr_ft"): + bedrock_attrs = getattr(self, "_nr_bedrock_attrs", {}) + self._nr_ft.__exit__(None, None, None) + + # If there are no bedrock attrs exit early as there's no data to record. + if not bedrock_attrs: + return + + try: + bedrock_attrs["duration"] = self._nr_ft.duration * 1000 + handle_chat_completion_event(transaction, bedrock_attrs) + except Exception: + _logger.warning(RESPONSE_PROCESSING_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + # Clear cached data as this can be very large. + self._nr_bedrock_attrs.clear() + + +def record_error(self, transaction, exc): + if hasattr(self, "_nr_ft"): + try: + ft = self._nr_ft + error_attributes = getattr(self, "_nr_bedrock_attrs", {}) + + # If there are no bedrock attrs exit early as there's no data to record. + if not error_attributes: + return + + error_attributes = bedrock_error_attributes(exc, error_attributes) + notice_error_attributes = { + "http.statusCode": error_attributes.get("http.statusCode"), + "error.message": error_attributes.get("error.message"), + "error.code": error_attributes.get("error.code"), + } + notice_error_attributes.update({"completion_id": str(uuid.uuid4())}) + + ft.notice_error( + attributes=notice_error_attributes, + ) + + ft.__exit__(*sys.exc_info()) + error_attributes["duration"] = ft.duration * 1000 + + handle_chat_completion_event(transaction, error_attributes) + + # Clear cached data as this can be very large. + error_attributes.clear() + except Exception: + _logger.warning(EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +def handle_embedding_event(transaction, bedrock_attrs): + embedding_id = str(uuid.uuid4()) + + settings = transaction.settings if transaction.settings is not None else global_settings() + + # Grab LLM-related custom attributes off of the transaction to store as metadata on LLM events + custom_attrs_dict = transaction._custom_params + llm_metadata_dict = {key: value for key, value in custom_attrs_dict.items() if key.startswith("llm.")} + + span_id = bedrock_attrs.get("span_id", None) + trace_id = bedrock_attrs.get("trace_id", None) + request_id = bedrock_attrs.get("request_id", None) + model = bedrock_attrs.get("model", None) + input = bedrock_attrs.get("input") + + embedding_dict = { + "vendor": "bedrock", + "ingest_source": "Python", + "id": embedding_id, + "span_id": span_id, + "trace_id": trace_id, + "token_count": ( + settings.ai_monitoring.llm_token_count_callback(model, input) + if settings.ai_monitoring.llm_token_count_callback + else None + ), + "request_id": request_id, + "duration": bedrock_attrs.get("duration", None), + "request.model": model, + "response.model": model, + "error": bedrock_attrs.get("error", None), + } + embedding_dict.update(llm_metadata_dict) + + if settings.ai_monitoring.record_content.enabled: + embedding_dict["input"] = input + + embedding_dict = {k: v for k, v in embedding_dict.items() if v is not None} + transaction.record_custom_event("LlmEmbedding", embedding_dict) + + +def handle_chat_completion_event(transaction, bedrock_attrs): + chat_completion_id = str(uuid.uuid4()) + + # Grab LLM-related custom attributes off of the transaction to store as metadata on LLM events + custom_attrs_dict = transaction._custom_params + llm_metadata_dict = {key: value for key, value in custom_attrs_dict.items() if key.startswith("llm.")} + + llm_context_attrs = getattr(transaction, "_llm_context_attrs", None) + if llm_context_attrs: + llm_metadata_dict.update(llm_context_attrs) + + span_id = bedrock_attrs.get("span_id", None) + trace_id = bedrock_attrs.get("trace_id", None) + request_id = bedrock_attrs.get("request_id", None) + response_id = bedrock_attrs.get("response_id", None) + model = bedrock_attrs.get("model", None) + + settings = transaction.settings if transaction.settings is not None else global_settings() + + input_message_list = bedrock_attrs.get("input_message_list", []) + output_message_list = bedrock_attrs.get("output_message_list", []) + number_of_messages = ( + len(input_message_list) + len(output_message_list) + ) or None # If 0, attribute will be set to None and removed + + chat_completion_summary_dict = { + "vendor": "bedrock", + "ingest_source": "Python", + "id": chat_completion_id, + "span_id": span_id, + "trace_id": trace_id, + "request_id": request_id, + "response_id": response_id, + "duration": bedrock_attrs.get("duration", None), + "request.max_tokens": bedrock_attrs.get("request.max_tokens", None), + "request.temperature": bedrock_attrs.get("request.temperature", None), + "request.model": model, + "response.model": model, # Duplicate data required by the UI + "response.number_of_messages": number_of_messages, + "response.choices.finish_reason": bedrock_attrs.get("response.choices.finish_reason", None), + "error": bedrock_attrs.get("error", None), + } + chat_completion_summary_dict.update(llm_metadata_dict) + chat_completion_summary_dict = {k: v for k, v in chat_completion_summary_dict.items() if v is not None} + + transaction.record_custom_event("LlmChatCompletionSummary", chat_completion_summary_dict) + + create_chat_completion_message_event( + transaction=transaction, + input_message_list=input_message_list, + output_message_list=output_message_list, + chat_completion_id=chat_completion_id, + span_id=span_id, + trace_id=trace_id, + request_model=model, + request_id=request_id, + llm_metadata_dict=llm_metadata_dict, + response_id=response_id, + ) + + +def sqs_message_trace( + operation, + destination_type, + destination_name, + params={}, + terminal=True, + async_wrapper=None, + extract_agent_attrs=None, +): + @function_wrapper + def _nr_sqs_message_trace_wrapper_(wrapped, instance, args, kwargs): + wrapper = async_wrapper if async_wrapper is not None else get_async_wrapper(wrapped) + if not wrapper: + parent = current_trace() + if not parent: + return wrapped(*args, **kwargs) + else: + parent = None + + _library = "SQS" + _operation = operation + _destination_type = destination_type + _destination_name = destination_name(*args, **kwargs) + + trace = MessageTrace( + _library, + _operation, + _destination_type, + _destination_name, + params=params, + terminal=terminal, + parent=parent, + source=wrapped, + ) + + # Attach extracted agent attributes. + _agent_attrs = extract_agent_attrs(*args, **kwargs) + trace.agent_attributes.update(_agent_attrs) + + if wrapper: # pylint: disable=W0125,W0126 + return wrapper(wrapped, trace)(*args, **kwargs) + + with trace: + return wrapped(*args, **kwargs) + + return _nr_sqs_message_trace_wrapper_ + + CUSTOM_TRACE_POINTS = { - ('sns', 'publish'): message_trace( - 'SNS', 'Produce', 'Topic', - extract(('TopicArn', 'TargetArn'), 'PhoneNumber')), - ('dynamodb', 'put_item'): datastore_trace( - 'DynamoDB', extract('TableName'), 'put_item'), - ('dynamodb', 'get_item'): datastore_trace( - 'DynamoDB', extract('TableName'), 'get_item'), - ('dynamodb', 'update_item'): datastore_trace( - 'DynamoDB', extract('TableName'), 'update_item'), - ('dynamodb', 'delete_item'): datastore_trace( - 'DynamoDB', extract('TableName'), 'delete_item'), - ('dynamodb', 'create_table'): datastore_trace( - 'DynamoDB', extract('TableName'), 'create_table'), - ('dynamodb', 'delete_table'): datastore_trace( - 'DynamoDB', extract('TableName'), 'delete_table'), - ('dynamodb', 'query'): datastore_trace( - 'DynamoDB', extract('TableName'), 'query'), - ('dynamodb', 'scan'): datastore_trace( - 'DynamoDB', extract('TableName'), 'scan'), - ('sqs', 'send_message'): message_trace( - 'SQS', 'Produce', 'Queue', extract_sqs), - ('sqs', 'send_message_batch'): message_trace( - 'SQS', 'Produce', 'Queue', extract_sqs), - ('sqs', 'receive_message'): message_trace( - 'SQS', 'Consume', 'Queue', extract_sqs), + ("sns", "publish"): message_trace("SNS", "Produce", "Topic", extract(("TopicArn", "TargetArn"), "PhoneNumber")), + ("dynamodb", "put_item"): datastore_trace("DynamoDB", extract("TableName"), "put_item"), + ("dynamodb", "get_item"): datastore_trace("DynamoDB", extract("TableName"), "get_item"), + ("dynamodb", "update_item"): datastore_trace("DynamoDB", extract("TableName"), "update_item"), + ("dynamodb", "delete_item"): datastore_trace("DynamoDB", extract("TableName"), "delete_item"), + ("dynamodb", "create_table"): datastore_trace("DynamoDB", extract("TableName"), "create_table"), + ("dynamodb", "delete_table"): datastore_trace("DynamoDB", extract("TableName"), "delete_table"), + ("dynamodb", "query"): datastore_trace("DynamoDB", extract("TableName"), "query"), + ("dynamodb", "scan"): datastore_trace("DynamoDB", extract("TableName"), "scan"), + ("sqs", "send_message"): sqs_message_trace( + "Produce", "Queue", extract_sqs, extract_agent_attrs=extract_sqs_agent_attrs + ), + ("sqs", "send_message_batch"): sqs_message_trace( + "Produce", "Queue", extract_sqs, extract_agent_attrs=extract_sqs_agent_attrs + ), + ("sqs", "receive_message"): sqs_message_trace( + "Consume", "Queue", extract_sqs, extract_agent_attrs=extract_sqs_agent_attrs + ), + ("bedrock-runtime", "invoke_model"): wrap_bedrock_runtime_invoke_model(response_streaming=False), + ("bedrock-runtime", "invoke_model_with_response_stream"): wrap_bedrock_runtime_invoke_model( + response_streaming=True + ), } -def bind__create_api_method(py_operation_name, operation_name, service_model, - *args, **kwargs): +def bind__create_api_method(py_operation_name, operation_name, service_model, *args, **kwargs): return (py_operation_name, service_model) def _nr_clientcreator__create_api_method_(wrapped, instance, args, kwargs): - (py_operation_name, service_model) = \ - bind__create_api_method(*args, **kwargs) + (py_operation_name, service_model) = bind__create_api_method(*args, **kwargs) service_name = service_model.service_name.lower() tracer = CUSTOM_TRACE_POINTS.get((service_name, py_operation_name)) @@ -89,36 +933,40 @@ def _nr_clientcreator__create_api_method_(wrapped, instance, args, kwargs): return tracer(wrapped) +def _nr_clientcreator__create_methods(wrapped, instance, args, kwargs): + class_attributes = wrapped(*args, **kwargs) + class_attributes["_nr_wrapped"] = True + return class_attributes + + def _bind_make_request_params(operation_model, request_dict, *args, **kwargs): return operation_model, request_dict def _nr_endpoint_make_request_(wrapped, instance, args, kwargs): operation_model, request_dict = _bind_make_request_params(*args, **kwargs) - url = request_dict.get('url', '') - method = request_dict.get('method', None) - - with ExternalTrace(library='botocore', url=url, method=method, source=wrapped) as trace: + url = request_dict.get("url") + method = request_dict.get("method") + with ExternalTrace(library="botocore", url=url, method=method, source=wrapped) as trace: try: - trace._add_agent_attribute('aws.operation', operation_model.name) + trace._add_agent_attribute("aws.operation", operation_model.name) except: pass result = wrapped(*args, **kwargs) try: - request_id = result[1]['ResponseMetadata']['RequestId'] - trace._add_agent_attribute('aws.requestId', request_id) + request_id = result[1]["ResponseMetadata"]["RequestId"] + trace._add_agent_attribute("aws.requestId", request_id) except: pass return result def instrument_botocore_endpoint(module): - wrap_function_wrapper(module, 'Endpoint.make_request', - _nr_endpoint_make_request_) + wrap_function_wrapper(module, "Endpoint.make_request", _nr_endpoint_make_request_) def instrument_botocore_client(module): - wrap_function_wrapper(module, 'ClientCreator._create_api_method', - _nr_clientcreator__create_api_method_) + wrap_function_wrapper(module, "ClientCreator._create_api_method", _nr_clientcreator__create_api_method_) + wrap_function_wrapper(module, "ClientCreator._create_methods", _nr_clientcreator__create_methods) diff --git a/newrelic/hooks/external_facepy.py b/newrelic/hooks/external_facepy.py index 2bf8605685..0cdfeaed14 100644 --- a/newrelic/hooks/external_facepy.py +++ b/newrelic/hooks/external_facepy.py @@ -17,7 +17,7 @@ def instrument(module): def url_query(graph_obj, method, path, *args, **kwargs): - return '/'.join([graph_obj.url, path]) + return f"{graph_obj.url}/{path}" newrelic.api.external_trace.wrap_external_trace( module, 'GraphAPI._query', 'facepy', url_query) diff --git a/newrelic/hooks/external_feedparser.py b/newrelic/hooks/external_feedparser.py index 13f9ebd63e..40d4ac7716 100644 --- a/newrelic/hooks/external_feedparser.py +++ b/newrelic/hooks/external_feedparser.py @@ -15,18 +15,18 @@ import sys import types -import newrelic.packages.six as six import newrelic.api.transaction import newrelic.api.object_wrapper +import newrelic.common.object_wrapper import newrelic.api.external_trace -class capture_external_trace(object): +class capture_external_trace(): def __init__(self, wrapped): newrelic.api.object_wrapper.update_wrapper(self, wrapped) self._nr_next_object = wrapped - if not hasattr(self, '_nr_last_object'): + if not hasattr(self, "_nr_last_object"): self._nr_last_object = wrapped def __call__(self, url, *args, **kwargs): @@ -34,7 +34,7 @@ def __call__(self, url, *args, **kwargs): # The URL be a string or a file like object. Pass call # through if not a string. - if not isinstance(url, six.string_types): + if not isinstance(url, str): return self._nr_next_object(url, *args, **kwargs) # Only then wrap the call if it looks like a URL. To @@ -43,16 +43,15 @@ def __call__(self, url, *args, **kwargs): parsed_url = url - if parsed_url.startswith('feed:http'): + if parsed_url.startswith("feed:http"): parsed_url = parsed_url[5:] - elif parsed_url.startswith('feed:'): - parsed_url = 'http:' + url[5:] + elif parsed_url.startswith("feed:"): + parsed_url = f"http:{url[5:]}" - if parsed_url.split(':')[0].lower() in ['http', 'https', 'ftp']: + if parsed_url.split(":")[0].lower() in ["http", "https", "ftp"]: current_transaction = newrelic.api.transaction.current_transaction() if current_transaction: - trace = newrelic.api.external_trace.ExternalTrace( - 'feedparser', parsed_url, 'GET') + trace = newrelic.api.external_trace.ExternalTrace("feedparser", parsed_url, "GET") context_manager = trace.__enter__() try: result = self._nr_next_object(url, *args, **kwargs) @@ -67,8 +66,8 @@ def __call__(self, url, *args, **kwargs): return self._nr_next_object(url, *args, **kwargs) def __getattr__(self, name): - return getattr(self._nr_next_object, name) + return getattr(self._nr_next_object, name) + def instrument(module): - newrelic.api.object_wrapper.wrap_object( - module, 'parse', capture_external_trace) + newrelic.common.object_wrapper.wrap_object(module, "parse", capture_external_trace) diff --git a/newrelic/hooks/external_httplib.py b/newrelic/hooks/external_httplib.py index 7d322f7194..93be1e9839 100644 --- a/newrelic/hooks/external_httplib.py +++ b/newrelic/hooks/external_httplib.py @@ -14,11 +14,10 @@ import functools -from newrelic.packages import six from newrelic.api.external_trace import ExternalTrace from newrelic.api.transaction import current_transaction -from newrelic.common.object_wrapper import ObjectWrapper +from newrelic.common.object_wrapper import wrap_function_wrapper def httplib_endheaders_wrapper(wrapped, instance, args, kwargs, @@ -39,7 +38,7 @@ def _connect_unbound(instance, *args, **kwargs): if hasattr(connection, '_nr_library_info'): library, scheme = connection._nr_library_info - url = '%s://%s:%s' % (scheme, connection.host, connection.port) + url = f'{scheme}://{connection.host}:{connection.port}' # Check if the NR headers have already been added. This is just in # case a higher level library which uses httplib underneath so @@ -119,30 +118,7 @@ def nr_header(header, *args, **kwargs): def instrument(module): - - if six.PY2: - library = 'httplib' - else: - library = 'http' - - module.HTTPConnection.endheaders = ObjectWrapper( - module.HTTPConnection.endheaders, - None, - functools.partial(httplib_endheaders_wrapper, scheme='http', - library=library)) - - module.HTTPSConnection.endheaders = ObjectWrapper( - module.HTTPConnection.endheaders, - None, - functools.partial(httplib_endheaders_wrapper, scheme='https', - library=library)) - - module.HTTPConnection.getresponse = ObjectWrapper( - module.HTTPConnection.getresponse, - None, - httplib_getresponse_wrapper) - - module.HTTPConnection.putheader = ObjectWrapper( - module.HTTPConnection.putheader, - None, - httplib_putheader_wrapper) + wrap_function_wrapper(module, "HTTPConnection.endheaders", functools.partial(httplib_endheaders_wrapper, scheme='http', library='http')) + wrap_function_wrapper(module, "HTTPSConnection.endheaders", functools.partial(httplib_endheaders_wrapper, scheme='https', library='http')) + wrap_function_wrapper(module, "HTTPConnection.getresponse", httplib_getresponse_wrapper) + wrap_function_wrapper(module, "HTTPConnection.putheader", httplib_putheader_wrapper) diff --git a/newrelic/hooks/external_pywapi.py b/newrelic/hooks/external_pywapi.py deleted file mode 100644 index 1820492c70..0000000000 --- a/newrelic/hooks/external_pywapi.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from newrelic.agent import wrap_external_trace - -def instrument_pywapi(module): - - if hasattr(module, 'get_weather_from_weather_com'): - wrap_external_trace(module, 'get_weather_from_weather_com', 'pywapi', - module.WEATHER_COM_URL) - - if hasattr(module, 'get_countries_from_google'): - wrap_external_trace(module, 'get_countries_from_google', 'pywapi', - module.GOOGLE_COUNTRIES_URL) - - if hasattr(module, 'get_cities_from_google'): - wrap_external_trace(module, 'get_cities_from_google', 'pywapi', - module.GOOGLE_CITIES_URL) - - if hasattr(module, 'get_weather_from_yahoo'): - wrap_external_trace(module, 'get_weather_from_yahoo', 'pywapi', - module.YAHOO_WEATHER_URL) - - if hasattr(module, 'get_weather_from_noaa'): - wrap_external_trace(module, 'get_weather_from_noaa', 'pywapi', - module.NOAA_WEATHER_URL) diff --git a/newrelic/hooks/external_s3transfer.py b/newrelic/hooks/external_s3transfer.py new file mode 100644 index 0000000000..1cb8cd0553 --- /dev/null +++ b/newrelic/hooks/external_s3transfer.py @@ -0,0 +1,34 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from newrelic.api.time_trace import current_trace +from newrelic.common.signature import bind_args +from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.core.context import context_wrapper + + +def instrument_s3transfer_futures(module): + if hasattr(module, "BoundedExecutor"): + wrap_function_wrapper(module, "BoundedExecutor.submit", wrap_BoundedExecutor_submit) + + +def wrap_BoundedExecutor_submit(wrapped, instance, args, kwargs): + trace = current_trace() + if not trace: + return wrapped(*args, **kwargs) + + bound_args = bind_args(wrapped, args, kwargs) + bound_args["task"] = context_wrapper(bound_args["task"], trace=trace, strict=True) + + return wrapped(**bound_args) diff --git a/newrelic/hooks/external_thrift.py b/newrelic/hooks/external_thrift.py index ceea07b80f..e2a94f3ced 100644 --- a/newrelic/hooks/external_thrift.py +++ b/newrelic/hooks/external_thrift.py @@ -19,9 +19,9 @@ def instrument(module): def tsocket_open_url(socket, *args, **kwargs): scheme = 'socket' if socket._unix_socket else 'http' if socket.port: - url = '%s://%s:%s' % (scheme, socket.host, socket.port) + url = f'{scheme}://{socket.host}:{socket.port}' else: - url = '%s://%s' % (scheme, socket.host) + url = f'{scheme}://{socket.host}' return url diff --git a/newrelic/hooks/external_urllib.py b/newrelic/hooks/external_urllib.py index e14477d0a0..c3ea4777f4 100644 --- a/newrelic/hooks/external_urllib.py +++ b/newrelic/hooks/external_urllib.py @@ -12,12 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - import urlparse -except ImportError: - import urllib.parse as urlparse - -import newrelic.packages.six as six +import urllib.parse as urlparse from newrelic.api.external_trace import ExternalTraceWrapper from newrelic.api.transaction import current_transaction @@ -56,7 +51,7 @@ def bind_params_urlretrieve(url, *args, **kwargs): def bind_params_open(fullurl, *args, **kwargs): - if isinstance(fullurl, six.string_types): + if isinstance(fullurl, str): return fullurl else: return fullurl.get_full_url() diff --git a/newrelic/hooks/external_urllib2.py b/newrelic/hooks/external_urllib2.py deleted file mode 100644 index c0a2f6e7e0..0000000000 --- a/newrelic/hooks/external_urllib2.py +++ /dev/null @@ -1,51 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -try: - import urlparse -except ImportError: - import urllib.parse as urlparse - -import newrelic.packages.six as six - -from newrelic.api.external_trace import ExternalTraceWrapper -from newrelic.api.transaction import current_transaction -from newrelic.common.object_wrapper import wrap_function_wrapper - -def _nr_wrapper_opener_director_open_(wrapped, instance, args, kwargs): - transaction = current_transaction() - - if transaction is None: - return wrapped(*args, **kwargs) - - def _bind_params(fullurl, *args, **kwargs): - if isinstance(fullurl, six.string_types): - return fullurl - else: - return fullurl.get_full_url() - - url = _bind_params(*args, **kwargs) - - details = urlparse.urlparse(url) - - if details.hostname is None: - return wrapped(*args, **kwargs) - - return ExternalTraceWrapper(wrapped, 'urllib2', url)(*args, **kwargs) - -def instrument(module): - - if hasattr(module, 'OpenerDirector'): - wrap_function_wrapper(module, 'OpenerDirector.open', - _nr_wrapper_opener_director_open_) diff --git a/newrelic/hooks/external_urllib3.py b/newrelic/hooks/external_urllib3.py index c2f969d9d2..307224f0af 100644 --- a/newrelic/hooks/external_urllib3.py +++ b/newrelic/hooks/external_urllib3.py @@ -14,25 +14,21 @@ from newrelic.api.external_trace import ExternalTraceWrapper from newrelic.common.object_wrapper import wrap_function_wrapper -from newrelic.hooks.external_httplib2 import ( - _nr_wrapper_httplib2_endheaders_wrapper) +from newrelic.hooks.external_httplib2 import _nr_wrapper_httplib2_endheaders_wrapper def _nr_wrapper_make_request_(wrapped, instance, args, kwargs): - def _bind_params(conn, method, url, *args, **kwargs): - return "%s://%s:%s" % (instance.scheme, conn.host, conn.port) + return method, f"{instance.scheme}://{conn.host}:{conn.port}" - url_for_apm_ui = _bind_params(*args, **kwargs) + method, url_for_apm_ui = _bind_params(*args, **kwargs) - return ExternalTraceWrapper(wrapped, 'urllib3', url_for_apm_ui)(*args, **kwargs) + return ExternalTraceWrapper(wrapped, "urllib3", url_for_apm_ui, method=method)(*args, **kwargs) def instrument_urllib3_connectionpool(module): - wrap_function_wrapper(module, 'HTTPSConnectionPool._make_request', - _nr_wrapper_make_request_) - wrap_function_wrapper(module, 'HTTPConnectionPool._make_request', - _nr_wrapper_make_request_) + wrap_function_wrapper(module, "HTTPSConnectionPool._make_request", _nr_wrapper_make_request_) + wrap_function_wrapper(module, "HTTPConnectionPool._make_request", _nr_wrapper_make_request_) def instrument_urllib3_connection(module): @@ -40,8 +36,10 @@ def instrument_urllib3_connection(module): # the 'connect' monkey patch separate, because it is also used to patch # urllib3 within the requests package. - wrap_function_wrapper(module, 'HTTPSConnection.endheaders', - _nr_wrapper_httplib2_endheaders_wrapper('urllib3', 'https')) + wrap_function_wrapper( + module, "HTTPSConnection.endheaders", _nr_wrapper_httplib2_endheaders_wrapper("urllib3", "https") + ) - wrap_function_wrapper(module, 'HTTPConnection.endheaders', - _nr_wrapper_httplib2_endheaders_wrapper('urllib3', 'http')) + wrap_function_wrapper( + module, "HTTPConnection.endheaders", _nr_wrapper_httplib2_endheaders_wrapper("urllib3", "http") + ) diff --git a/newrelic/hooks/external_xmlrpclib.py b/newrelic/hooks/external_xmlrpclib.py index 7b83852386..3c02ed48e6 100644 --- a/newrelic/hooks/external_xmlrpclib.py +++ b/newrelic/hooks/external_xmlrpclib.py @@ -15,7 +15,7 @@ import newrelic.api.external_trace def wrap_transport_request(self, host, handler, *args, **kwargs): - return "http://%s%s" % (host, handler) + return f"http://{host}{handler}" def instrument(module): diff --git a/newrelic/hooks/framework_aiohttp.py b/newrelic/hooks/framework_aiohttp.py index 68f4e70f1c..601d585f9e 100644 --- a/newrelic/hooks/framework_aiohttp.py +++ b/newrelic/hooks/framework_aiohttp.py @@ -152,7 +152,7 @@ class ResponseProxy: def __getattr__(self, name): # instance.response should be overwritten at this point if instance.response is self: - raise AttributeError("%r object has no attribute %r" % (type(instance).__name__, "response")) + raise AttributeError(f"{type(instance).__name__!r} object has no attribute response") return getattr(instance.response, name) instance.response = ResponseProxy() diff --git a/newrelic/hooks/framework_ariadne.py b/newrelic/hooks/framework_ariadne.py index 498c662c49..4927abe0b4 100644 --- a/newrelic/hooks/framework_ariadne.py +++ b/newrelic/hooks/framework_ariadne.py @@ -21,17 +21,12 @@ from newrelic.api.wsgi_application import wrap_wsgi_application from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version from newrelic.core.graphql_utils import graphql_statement -from newrelic.hooks.framework_graphql import ( - framework_version as graphql_framework_version, -) -from newrelic.hooks.framework_graphql import ignore_graphql_duplicate_exception +from newrelic.hooks.framework_graphql import GRAPHQL_VERSION, ignore_graphql_duplicate_exception - -def framework_details(): - import ariadne - - return ("Ariadne", getattr(ariadne, "__version__", None)) +ARIADNE_VERSION = get_package_version("ariadne") +ariadne_version_tuple = tuple(map(int, ARIADNE_VERSION.split("."))) def bind_graphql(schema, data, *args, **kwargs): @@ -49,9 +44,8 @@ def wrap_graphql_sync(wrapped, instance, args, kwargs): except TypeError: return wrapped(*args, **kwargs) - framework = framework_details() - transaction.add_framework_info(name=framework[0], version=framework[1]) # No version info available on ariadne - transaction.add_framework_info(name="GraphQL", version=graphql_framework_version()) + transaction.add_framework_info(name="Ariadne", version=ARIADNE_VERSION) + transaction.add_framework_info(name="GraphQL", version=GRAPHQL_VERSION) query = data["query"] if hasattr(query, "body"): @@ -83,9 +77,8 @@ async def wrap_graphql(wrapped, instance, args, kwargs): result = await result return result - framework = framework_details() - transaction.add_framework_info(name=framework[0], version=framework[1]) # No version info available on ariadne - transaction.add_framework_info(name="GraphQL", version=graphql_framework_version()) + transaction.add_framework_info(name="Ariadne", version=ARIADNE_VERSION) + transaction.add_framework_info(name="GraphQL", version=GRAPHQL_VERSION) query = data["query"] if hasattr(query, "body"): @@ -104,6 +97,9 @@ async def wrap_graphql(wrapped, instance, args, kwargs): def instrument_ariadne_execute(module): + # v0.9.0 is the version where ariadne started using graphql-core v3 + if ariadne_version_tuple < (0, 9): + return if hasattr(module, "graphql"): wrap_function_wrapper(module, "graphql", wrap_graphql) @@ -112,10 +108,14 @@ def instrument_ariadne_execute(module): def instrument_ariadne_asgi(module): + if ariadne_version_tuple < (0, 9): + return if hasattr(module, "GraphQL"): - wrap_asgi_application(module, "GraphQL.__call__", framework=framework_details()) + wrap_asgi_application(module, "GraphQL.__call__", framework=("Ariadne", ARIADNE_VERSION)) def instrument_ariadne_wsgi(module): + if ariadne_version_tuple < (0, 9): + return if hasattr(module, "GraphQL"): - wrap_wsgi_application(module, "GraphQL.__call__", framework=framework_details()) + wrap_wsgi_application(module, "GraphQL.__call__", framework=("Ariadne", ARIADNE_VERSION)) diff --git a/newrelic/hooks/framework_bottle.py b/newrelic/hooks/framework_bottle.py index 99caa844fd..5635fa782c 100644 --- a/newrelic/hooks/framework_bottle.py +++ b/newrelic/hooks/framework_bottle.py @@ -18,14 +18,21 @@ import functools -from newrelic.api.function_trace import (FunctionTrace, FunctionTraceWrapper, - wrap_function_trace) +from newrelic.api.function_trace import ( + FunctionTrace, + FunctionTraceWrapper, + wrap_function_trace, +) from newrelic.api.transaction import current_transaction from newrelic.api.wsgi_application import wrap_wsgi_application from newrelic.common.object_names import callable_name -from newrelic.common.object_wrapper import (wrap_out_function, - function_wrapper, ObjectProxy, wrap_object_attribute, - wrap_function_wrapper) +from newrelic.common.object_wrapper import ( + ObjectProxy, + function_wrapper, + wrap_function_wrapper, + wrap_object_attribute, + wrap_out_function, +) module_bottle = None @@ -34,17 +41,17 @@ def status_code(exc, value, tb): # The HTTPError class derives from HTTPResponse and so we do not # need to check for it seperately as isinstance() will pick it up. - if isinstance(value, module_bottle.HTTPResponse): - if hasattr(value, 'status_code'): + if isinstance(value, module_bottle.HTTPResponse): # pragma: no cover + if hasattr(value, "status_code"): return value.status_code - elif hasattr(value, 'status'): + elif hasattr(value, "status"): return value.status - elif hasattr(value, 'http_status_code'): + elif hasattr(value, "http_status_code"): return value.http_status_code def should_ignore(exc, value, tb): - if hasattr(module_bottle, 'RouteReset'): + if hasattr(module_bottle, "RouteReset"): if isinstance(value, module_bottle.RouteReset): return True @@ -113,8 +120,7 @@ def get(self, status, default=None): transaction.set_transaction_name(name, priority=1) handler = FunctionTraceWrapper(handler, name=name) else: - transaction.set_transaction_name(str(status), - group='StatusCode', priority=1) + transaction.set_transaction_name(str(status), group="StatusCode", priority=1) return handler or default @@ -140,43 +146,39 @@ def instrument_bottle(module): global module_bottle module_bottle = module - framework_details = ('Bottle', getattr(module, '__version__')) - - if hasattr(module.Bottle, 'wsgi'): # version >= 0.9 - wrap_wsgi_application(module, 'Bottle.wsgi', - framework=framework_details) - elif hasattr(module.Bottle, '__call__'): # version < 0.9 - wrap_wsgi_application(module, 'Bottle.__call__', - framework=framework_details) - - if (hasattr(module, 'Route') and - hasattr(module.Route, '_make_callback')): # version >= 0.10 - wrap_out_function(module, 'Route._make_callback', - output_wrapper_Route_make_callback) - elif hasattr(module.Bottle, '_match'): # version >= 0.9 - wrap_out_function(module, 'Bottle._match', - output_wrapper_Bottle_match) - elif hasattr(module.Bottle, 'match_url'): # version < 0.9 - wrap_out_function(module, 'Bottle.match_url', - output_wrapper_Bottle_match) - - wrap_object_attribute(module, 'Bottle.error_handler', - proxy_Bottle_error_handler) - - if hasattr(module, 'auth_basic'): - wrap_function_wrapper(module, 'auth_basic', wrapper_auth_basic) - - if hasattr(module, 'SimpleTemplate'): - wrap_function_trace(module, 'SimpleTemplate.render') - - if hasattr(module, 'MakoTemplate'): - wrap_function_trace(module, 'MakoTemplate.render') - - if hasattr(module, 'CheetahTemplate'): - wrap_function_trace(module, 'CheetahTemplate.render') - - if hasattr(module, 'Jinja2Template'): - wrap_function_trace(module, 'Jinja2Template.render') - - if hasattr(module, 'SimpleTALTemplate'): - wrap_function_trace(module, 'SimpleTALTemplate.render') + framework_details = ("Bottle", getattr(module, "__version__")) + # version >= 0.9 + if hasattr(module.Bottle, "wsgi"): # pragma: no cover + wrap_wsgi_application(module, "Bottle.wsgi", framework=framework_details) + # version < 0.9 + elif hasattr(module.Bottle, "__call__"): # pragma: no cover + wrap_wsgi_application(module, "Bottle.__call__", framework=framework_details) + # version >= 0.10 + if hasattr(module, "Route") and hasattr(module.Route, "_make_callback"): # pragma: no cover + wrap_out_function(module, "Route._make_callback", output_wrapper_Route_make_callback) + # version >= 0.9 + elif hasattr(module.Bottle, "_match"): # pragma: no cover + wrap_out_function(module, "Bottle._match", output_wrapper_Bottle_match) + # version < 0.9 + elif hasattr(module.Bottle, "match_url"): # pragma: no cover + wrap_out_function(module, "Bottle.match_url", output_wrapper_Bottle_match) + + wrap_object_attribute(module, "Bottle.error_handler", proxy_Bottle_error_handler) + + if hasattr(module, "auth_basic"): + wrap_function_wrapper(module, "auth_basic", wrapper_auth_basic) + + if hasattr(module, "SimpleTemplate"): + wrap_function_trace(module, "SimpleTemplate.render") + + if hasattr(module, "MakoTemplate"): + wrap_function_trace(module, "MakoTemplate.render") + + if hasattr(module, "CheetahTemplate"): + wrap_function_trace(module, "CheetahTemplate.render") + + if hasattr(module, "Jinja2Template"): + wrap_function_trace(module, "Jinja2Template.render") + + if hasattr(module, "SimpleTALTemplate"): # pragma: no cover + wrap_function_trace(module, "SimpleTALTemplate.render") diff --git a/newrelic/hooks/framework_django.py b/newrelic/hooks/framework_django.py index 005f282795..d9bc0e4c69 100644 --- a/newrelic/hooks/framework_django.py +++ b/newrelic/hooks/framework_django.py @@ -12,48 +12,59 @@ # See the License for the specific language governing permissions and # limitations under the License. +import functools +import logging import sys import threading -import logging -import functools - -from newrelic.packages import six +import warnings from newrelic.api.application import register_application from newrelic.api.background_task import BackgroundTaskWrapper from newrelic.api.error_trace import wrap_error_trace -from newrelic.api.function_trace import (FunctionTrace, wrap_function_trace, - FunctionTraceWrapper) +from newrelic.api.function_trace import ( + FunctionTrace, + FunctionTraceWrapper, + wrap_function_trace, +) from newrelic.api.html_insertion import insert_html_snippet -from newrelic.api.transaction import current_transaction from newrelic.api.time_trace import notice_error +from newrelic.api.transaction import current_transaction from newrelic.api.transaction_name import wrap_transaction_name from newrelic.api.wsgi_application import WSGIApplicationWrapper - -from newrelic.common.object_wrapper import (FunctionWrapper, wrap_in_function, - wrap_post_function, wrap_function_wrapper, function_wrapper) +from newrelic.common.coroutine import is_asyncio_coroutine, is_coroutine_function from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import ( + FunctionWrapper, + function_wrapper, + wrap_function_wrapper, + wrap_in_function, + wrap_post_function, +) from newrelic.config import extra_settings from newrelic.core.config import global_settings -from newrelic.common.coroutine import is_coroutine_function, is_asyncio_coroutine -if six.PY3: - from newrelic.hooks.framework_django_py3 import ( - _nr_wrapper_BaseHandler_get_response_async_, - _nr_wrap_converted_middleware_async_, - ) +from newrelic.hooks.framework_django_py3 import ( + _nr_wrap_converted_middleware_async_, + _nr_wrapper_BaseHandler_get_response_async_, +) _logger = logging.getLogger(__name__) _boolean_states = { - '1': True, 'yes': True, 'true': True, 'on': True, - '0': False, 'no': False, 'false': False, 'off': False + "1": True, + "yes": True, + "true": True, + "on": True, + "0": False, + "no": False, + "false": False, + "off": False, } def _setting_boolean(value): if value.lower() not in _boolean_states: - raise ValueError('Not a boolean: %s' % value) + raise ValueError(f"Not a boolean: {value}") return _boolean_states[value.lower()] @@ -62,25 +73,23 @@ def _setting_set(value): _settings_types = { - 'browser_monitoring.auto_instrument': _setting_boolean, - 'instrumentation.templates.inclusion_tag': _setting_set, - 'instrumentation.background_task.startup_timeout': float, - 'instrumentation.scripts.django_admin': _setting_set, + "browser_monitoring.auto_instrument": _setting_boolean, + "instrumentation.templates.inclusion_tag": _setting_set, + "instrumentation.background_task.startup_timeout": float, + "instrumentation.scripts.django_admin": _setting_set, } _settings_defaults = { - 'browser_monitoring.auto_instrument': True, - 'instrumentation.templates.inclusion_tag': set(), - 'instrumentation.background_task.startup_timeout': 10.0, - 'instrumentation.scripts.django_admin': set(), + "browser_monitoring.auto_instrument": True, + "instrumentation.templates.inclusion_tag": set(), + "instrumentation.background_task.startup_timeout": 10.0, + "instrumentation.scripts.django_admin": set(), } -django_settings = extra_settings('import-hook:django', - types=_settings_types, defaults=_settings_defaults) +django_settings = extra_settings("import-hook:django", types=_settings_types, defaults=_settings_defaults) def should_add_browser_timing(response, transaction): - # Don't do anything if receive a streaming response which # was introduced in Django 1.5. Need to avoid this as there # will be no 'content' attribute. Alternatively there may be @@ -92,7 +101,7 @@ def should_add_browser_timing(response, transaction): # do RUM insertion, need to move to a WSGI middleware and # deal with how to update the content length. - if hasattr(response, 'streaming_content'): + if hasattr(response, "streaming_content"): return False # Need to be running within a valid web transaction. @@ -100,7 +109,7 @@ def should_add_browser_timing(response, transaction): if not transaction or not transaction.enabled: return False - # Only insert RUM JavaScript headers and footers if enabled + # Only insert RUM JavaScript headers if enabled # in configuration and not already likely inserted. if not transaction.settings.browser_monitoring.enabled: @@ -121,89 +130,74 @@ def should_add_browser_timing(response, transaction): # a user may want to also perform insertion for # 'application/xhtml+xml'. - ctype = response.get('Content-Type', '').lower().split(';')[0] + ctype = response.get("Content-Type", "").lower().split(";")[0] if ctype not in transaction.settings.browser_monitoring.content_type: return False # Don't risk it if content encoding already set. - if response.has_header('Content-Encoding'): + if response.has_header("Content-Encoding"): return False # Don't risk it if content is actually within an attachment. - cdisposition = response.get('Content-Disposition', '').lower() + cdisposition = response.get("Content-Disposition", "").lower() - if cdisposition.split(';')[0].strip().lower() == 'attachment': + if cdisposition.split(";")[0].strip().lower() == "attachment": return False return True -# Response middleware for automatically inserting RUM header and -# footer into HTML response returned by application - -def browser_timing_insertion(response, transaction): - - # No point continuing if header is empty. This can occur if - # RUM is not enabled within the UI. It is assumed at this - # point that if header is not empty, then footer will not be - # empty. We don't want to generate the footer just yet as - # want to do that as late as possible so that application - # server time in footer is as accurate as possible. In - # particular, if the response content is generated on demand - # then the flattening of the response could take some time - # and we want to track that. We thus generate footer below - # at point of insertion. - - header = transaction.browser_timing_header() - - if not header: - return response +# Response middleware for automatically inserting RUM header into HTML response returned by application - def html_to_be_inserted(): - return six.b(header) + six.b(transaction.browser_timing_footer()) - # Make sure we flatten any content first as it could be - # stored as a list of strings in the response object. We - # assign it back to the response object to avoid having - # multiple copies of the string in memory at the same time +def browser_timing_insertion(response, transaction): + # No point continuing if header is empty. This can occur if RUM is not enabled within the UI. We don't want to + # generate the header just yet as we want to do that as late as possible so that application server time in header + # is as accurate as possible. In particular, if the response content is generated on demand then the flattening + # of the response could take some time and we want to track that. We thus generate header below at + # the point of insertion. + + # Make sure we flatten any content first as it could be stored as a list of strings in the response object. We + # assign it back to the response object to avoid having multiple copies of the string in memory at the same time # as we progress through steps below. - result = insert_html_snippet(response.content, html_to_be_inserted) + result = insert_html_snippet(response.content, lambda: transaction.browser_timing_header().encode("latin-1")) if result is not None: if transaction.settings.debug.log_autorum_middleware: - _logger.debug('RUM insertion from Django middleware ' - 'triggered. Bytes added was %r.', - len(result) - len(response.content)) + _logger.debug( + "RUM insertion from Django middleware triggered. Bytes added was %r.", + len(result) - len(response.content), + ) response.content = result - if response.get('Content-Length', None): - response['Content-Length'] = str(len(response.content)) + if response.get("Content-Length", None): + response["Content-Length"] = str(len(response.content)) return response -# Template tag functions for manually inserting RUM header and -# footer into HTML response. A template tag library for -# 'newrelic' will be automatically inserted into set of tag -# libraries when performing step to instrument the middleware. +# Template tag functions for manually inserting RUM header into HTML response. A template tag library for 'newrelic' +# will be automatically inserted into set of tag libraries when performing step to instrument the middleware. + def newrelic_browser_timing_header(): from django.utils.safestring import mark_safe transaction = current_transaction() - return transaction and mark_safe(transaction.browser_timing_header()) or '' + return transaction and mark_safe(transaction.browser_timing_header()) or "" # nosec def newrelic_browser_timing_footer(): - from django.utils.safestring import mark_safe - - transaction = current_transaction() - return transaction and mark_safe(transaction.browser_timing_footer()) or '' + warnings.warn( + "The newrelic_browser_timing_footer function is deprecated. Please migrate to only using the newrelic_browser_timing_header API instead.", + DeprecationWarning, + ) + return "" # nosec # Addition of instrumentation for middleware. Can only do this @@ -214,7 +208,6 @@ def newrelic_browser_timing_footer(): def wrap_leading_middleware(middleware): - # Wrapper to be applied to middleware executed prior to the # view handler being executed. Records the time spent in the # middleware as separate function node and also attempts to @@ -256,9 +249,13 @@ def wrapper(wrapped, instance, args, kwargs): yield wrapper(wrapped) -def wrap_view_middleware(middleware): - - # XXX This is no longer being used. The changes to strip the +# Because this is not being used in any version of Django that is +# within New Relic's support window, no tests will be added +# for this. However, value exists to keeping backwards compatible +# functionality, so instead of removing this instrumentation, this +# will be excluded from the coverage analysis. +def wrap_view_middleware(middleware): # pragma: no cover + # This is no longer being used. The changes to strip the # wrapper from the view handler when passed into the function # urlresolvers.reverse() solves most of the problems. To back # that up, the object wrapper now proxies various special @@ -293,7 +290,7 @@ def wrapper(wrapped, instance, args, kwargs): def _wrapped(request, view_func, view_args, view_kwargs): # This strips the view handler wrapper before call. - if hasattr(view_func, '_nr_last_object'): + if hasattr(view_func, "_nr_last_object"): view_func = view_func._nr_last_object return wrapped(request, view_func, view_args, view_kwargs) @@ -323,7 +320,6 @@ def _wrapped(request, view_func, view_args, view_kwargs): def wrap_trailing_middleware(middleware): - # Wrapper to be applied to trailing middleware executed # after the view handler. Records the time spent in the # middleware as separate function node. Transaction is never @@ -339,7 +335,6 @@ def wrap_trailing_middleware(middleware): def insert_and_wrap_middleware(handler, *args, **kwargs): - # Use lock to control access by single thread but also as # flag to indicate if done the initialisation. Lock will be # None if have already done this. @@ -364,44 +359,33 @@ def insert_and_wrap_middleware(handler, *args, **kwargs): middleware_instrumentation_lock = None try: - # Wrap the middleware to undertake timing and name # the web transaction. The naming is done as lower # priority than that for view handler so view handler # name always takes precedence. - if hasattr(handler, '_request_middleware'): - handler._request_middleware = list( - wrap_leading_middleware( - handler._request_middleware)) + if hasattr(handler, "_request_middleware"): + handler._request_middleware = list(wrap_leading_middleware(handler._request_middleware)) - if hasattr(handler, '_view_middleware'): - handler._view_middleware = list( - wrap_leading_middleware( - handler._view_middleware)) + if hasattr(handler, "_view_middleware"): + handler._view_middleware = list(wrap_leading_middleware(handler._view_middleware)) - if hasattr(handler, '_template_response_middleware'): + if hasattr(handler, "_template_response_middleware"): handler._template_response_middleware = list( - wrap_trailing_middleware( - handler._template_response_middleware)) + wrap_trailing_middleware(handler._template_response_middleware) + ) - if hasattr(handler, '_response_middleware'): - handler._response_middleware = list( - wrap_trailing_middleware( - handler._response_middleware)) + if hasattr(handler, "_response_middleware"): + handler._response_middleware = list(wrap_trailing_middleware(handler._response_middleware)) - if hasattr(handler, '_exception_middleware'): - handler._exception_middleware = list( - wrap_trailing_middleware( - handler._exception_middleware)) + if hasattr(handler, "_exception_middleware"): + handler._exception_middleware = list(wrap_trailing_middleware(handler._exception_middleware)) finally: lock.release() -def _nr_wrapper_GZipMiddleware_process_response_(wrapped, instance, args, - kwargs): - +def _nr_wrapper_GZipMiddleware_process_response_(wrapped, instance, args, kwargs): transaction = current_transaction() if transaction is None: @@ -433,40 +417,34 @@ def _nr_wrapper_BaseHandler_get_response_(wrapped, instance, args, kwargs): request = _bind_get_response(*args, **kwargs) - if hasattr(request, '_nr_exc_info'): + if hasattr(request, "_nr_exc_info"): notice_error(error=request._nr_exc_info, status_code=response.status_code) - delattr(request, '_nr_exc_info') + delattr(request, "_nr_exc_info") return response # Post import hooks for modules. -def instrument_django_core_handlers_base(module): +def instrument_django_core_handlers_base(module): # Attach a post function to load_middleware() method of # BaseHandler to trigger insertion of browser timing # middleware and wrapping of middleware for timing etc. - wrap_post_function(module, 'BaseHandler.load_middleware', - insert_and_wrap_middleware) + wrap_post_function(module, "BaseHandler.load_middleware", insert_and_wrap_middleware) - if six.PY3 and hasattr(module.BaseHandler, 'get_response_async'): - wrap_function_wrapper(module, 'BaseHandler.get_response_async', - _nr_wrapper_BaseHandler_get_response_async_) + if hasattr(module.BaseHandler, "get_response_async"): + wrap_function_wrapper(module, "BaseHandler.get_response_async", _nr_wrapper_BaseHandler_get_response_async_) - wrap_function_wrapper(module, 'BaseHandler.get_response', - _nr_wrapper_BaseHandler_get_response_) + wrap_function_wrapper(module, "BaseHandler.get_response", _nr_wrapper_BaseHandler_get_response_) def instrument_django_gzip_middleware(module): - - wrap_function_wrapper(module, 'GZipMiddleware.process_response', - _nr_wrapper_GZipMiddleware_process_response_) + wrap_function_wrapper(module, "GZipMiddleware.process_response", _nr_wrapper_GZipMiddleware_process_response_) def wrap_handle_uncaught_exception(middleware): - # Wrapper to be applied to handler called when exceptions # propagate up to top level from middleware. Records the # time spent in the handler as separate function node. Names @@ -499,17 +477,15 @@ def _wrapped(request, resolver, exc_info): def instrument_django_core_handlers_wsgi(module): - # Wrap the WSGI application entry point. If this is also # wrapped from the WSGI script file or by the WSGI hosting # mechanism then those will take precedence. import django - framework = ('Django', django.get_version()) + framework = ("Django", django.get_version()) - module.WSGIHandler.__call__ = WSGIApplicationWrapper( - module.WSGIHandler.__call__, framework=framework) + module.WSGIHandler.__call__ = WSGIApplicationWrapper(module.WSGIHandler.__call__, framework=framework) # Wrap handle_uncaught_exception() of WSGIHandler so that # can capture exception details of any exception which @@ -519,20 +495,19 @@ def instrument_django_core_handlers_wsgi(module): # exception, so last chance to do this as exception will not # propagate up to the WSGI application. - if hasattr(module.WSGIHandler, 'handle_uncaught_exception'): - module.WSGIHandler.handle_uncaught_exception = ( - wrap_handle_uncaught_exception( - module.WSGIHandler.handle_uncaught_exception)) + if hasattr(module.WSGIHandler, "handle_uncaught_exception"): + module.WSGIHandler.handle_uncaught_exception = wrap_handle_uncaught_exception( + module.WSGIHandler.handle_uncaught_exception + ) def wrap_view_handler(wrapped, priority=3): - # Ensure we don't wrap the view handler more than once. This # looks like it may occur in cases where the resolver is # called recursively. We flag that view handler was wrapped # using the '_nr_django_view_handler' attribute. - if hasattr(wrapped, '_nr_django_view_handler'): + if hasattr(wrapped, "_nr_django_view_handler"): return wrapped if hasattr(wrapped, "view_class"): @@ -568,7 +543,6 @@ def wrapper(wrapped, instance, args, kwargs): def wrap_url_resolver(wrapped): - # Wrap URL resolver. If resolver returns valid result then # wrap the view handler returned. The type of the result # changes across Django versions so need to check and adapt @@ -584,7 +558,7 @@ def wrapper(wrapped, instance, args, kwargs): if transaction is None: return wrapped(*args, **kwargs) - if hasattr(transaction, '_nr_django_url_resolver'): + if hasattr(transaction, "_nr_django_url_resolver"): return wrapped(*args, **kwargs) # Tag the transaction so we know when we are in the top @@ -602,8 +576,7 @@ def _wrapped(path): if type(result) is tuple: callback, callback_args, callback_kwargs = result - result = (wrap_view_handler(callback, priority=5), - callback_args, callback_kwargs) + result = (wrap_view_handler(callback, priority=5), callback_args, callback_kwargs) else: result.func = wrap_view_handler(result.func, priority=5) @@ -619,7 +592,6 @@ def _wrapped(path): def wrap_url_resolver_nnn(wrapped, priority=1): - # Wrapper to be applied to the URL resolver for errors. name = callable_name(wrapped) @@ -636,14 +608,12 @@ def wrapper(wrapped, instance, args, kwargs): return wrap_view_handler(result, priority=priority) else: callback, param_dict = result - return (wrap_view_handler(callback, priority=priority), - param_dict) + return (wrap_view_handler(callback, priority=priority), param_dict) return FunctionWrapper(wrapped, wrapper) def wrap_url_reverse(wrapped): - # Wrap the URL resolver reverse lookup. Where the view # handler is passed in we need to strip any instrumentation # wrapper to ensure that it doesn't interfere with the @@ -653,16 +623,16 @@ def wrap_url_reverse(wrapped): def wrapper(wrapped, instance, args, kwargs): def execute(viewname, *args, **kwargs): - if hasattr(viewname, '_nr_last_object'): + if hasattr(viewname, "_nr_last_object"): viewname = viewname._nr_last_object return wrapped(viewname, *args, **kwargs) + return execute(*args, **kwargs) return FunctionWrapper(wrapped, wrapper) def instrument_django_core_urlresolvers(module): - # Wrap method which maps a string version of a function # name as used in urls.py pattern so can capture any # exception which is raised during that process. @@ -672,20 +642,19 @@ def instrument_django_core_urlresolvers(module): # lost. We thus intercept it here so can capture that # traceback which is otherwise lost. - wrap_error_trace(module, 'get_callable') + wrap_error_trace(module, "get_callable") # Wrap methods which resolves a request to a view handler. # This can be called against a resolver initialised against # a custom URL conf associated with a specific request, or a # resolver which uses the default URL conf. - if hasattr(module, 'RegexURLResolver'): + if hasattr(module, "RegexURLResolver"): urlresolver = module.RegexURLResolver else: urlresolver = module.URLResolver - urlresolver.resolve = wrap_url_resolver( - urlresolver.resolve) + urlresolver.resolve = wrap_url_resolver(urlresolver.resolve) # Wrap methods which resolve error handlers. For 403 and 404 # we give these higher naming priority over any prior @@ -695,40 +664,34 @@ def instrument_django_core_urlresolvers(module): # handler in place so error details identify the correct # transaction. - if hasattr(urlresolver, 'resolve403'): - urlresolver.resolve403 = wrap_url_resolver_nnn( - urlresolver.resolve403, priority=3) + if hasattr(urlresolver, "resolve403"): + urlresolver.resolve403 = wrap_url_resolver_nnn(urlresolver.resolve403, priority=3) - if hasattr(urlresolver, 'resolve404'): - urlresolver.resolve404 = wrap_url_resolver_nnn( - urlresolver.resolve404, priority=3) + if hasattr(urlresolver, "resolve404"): + urlresolver.resolve404 = wrap_url_resolver_nnn(urlresolver.resolve404, priority=3) - if hasattr(urlresolver, 'resolve500'): - urlresolver.resolve500 = wrap_url_resolver_nnn( - urlresolver.resolve500, priority=1) + if hasattr(urlresolver, "resolve500"): + urlresolver.resolve500 = wrap_url_resolver_nnn(urlresolver.resolve500, priority=1) - if hasattr(urlresolver, 'resolve_error_handler'): - urlresolver.resolve_error_handler = wrap_url_resolver_nnn( - urlresolver.resolve_error_handler, priority=1) + if hasattr(urlresolver, "resolve_error_handler"): + urlresolver.resolve_error_handler = wrap_url_resolver_nnn(urlresolver.resolve_error_handler, priority=1) # Wrap function for performing reverse URL lookup to strip any # instrumentation wrapper when view handler is passed in. - if hasattr(module, 'reverse'): + if hasattr(module, "reverse"): module.reverse = wrap_url_reverse(module.reverse) def instrument_django_urls_base(module): - # Wrap function for performing reverse URL lookup to strip any # instrumentation wrapper when view handler is passed in. - if hasattr(module, 'reverse'): + if hasattr(module, "reverse"): module.reverse = wrap_url_reverse(module.reverse) def instrument_django_template(module): - # Wrap methods for rendering of Django templates. The name # of the method changed in between Django versions so need # to check for which one we have. The name of the function @@ -742,21 +705,18 @@ def instrument_django_template(module): def template_name(template, *args): return template.name - if hasattr(module.Template, '_render'): - wrap_function_trace(module, 'Template._render', - name=template_name, group='Template/Render') + if hasattr(module.Template, "_render"): + wrap_function_trace(module, "Template._render", name=template_name, group="Template/Render") else: - wrap_function_trace(module, 'Template.render', - name=template_name, group='Template/Render') + wrap_function_trace(module, "Template.render", name=template_name, group="Template/Render") # Django 1.8 no longer has module.libraries. As automatic way is not # preferred we can just skip this now. - if not hasattr(module, 'libraries'): + if not hasattr(module, "libraries"): return - # Register template tags used for manual insertion of RUM - # header and footer. + # Register template tags used for manual insertion of RUM header. # # TODO This can now be installed as a separate tag library # so should possibly look at deprecating this automatic @@ -766,18 +726,17 @@ def template_name(template, *args): library.simple_tag(newrelic_browser_timing_header) library.simple_tag(newrelic_browser_timing_footer) - module.libraries['django.templatetags.newrelic'] = library + module.libraries["django.templatetags.newrelic"] = library def wrap_template_block(wrapped): def wrapper(wrapped, instance, args, kwargs): - return FunctionTraceWrapper(wrapped, name=instance.name, group='Template/Block')(*args, **kwargs) + return FunctionTraceWrapper(wrapped, name=instance.name, group="Template/Block")(*args, **kwargs) return FunctionWrapper(wrapped, wrapper) def instrument_django_template_loader_tags(module): - # Wrap template block node for timing, naming the node after # the block name as defined in the template rather than # function name. @@ -786,7 +745,6 @@ def instrument_django_template_loader_tags(module): def instrument_django_core_servers_basehttp(module): - # Allow 'runserver' to be used with Django <= 1.3. To do # this we wrap the WSGI application argument on the way in # so that the run() method gets the wrapped instance. @@ -812,12 +770,15 @@ def instrument_django_core_servers_basehttp(module): # instrumentation of the wsgiref module or some other means. def wrap_wsgi_application_entry_point(server, application, **kwargs): - return ((server, WSGIApplicationWrapper(application, - framework='Django'),), kwargs) - - if (not hasattr(module, 'simple_server') and - hasattr(module.ServerHandler, 'run')): - + return ( + ( + server, + WSGIApplicationWrapper(application, framework="Django"), + ), + kwargs, + ) + + if not hasattr(module, "simple_server") and hasattr(module.ServerHandler, "run"): # Patch the server to make it work properly. def run(self, application): @@ -833,11 +794,10 @@ def run(self, application): def close(self): if self.result is not None: try: - self.request_handler.log_request( - self.status.split(' ', 1)[0], self.bytes_sent) + self.request_handler.log_request(self.status.split(" ", 1)[0], self.bytes_sent) finally: try: - if hasattr(self.result, 'close'): + if hasattr(self.result, "close"): self.result.close() finally: self.result = None @@ -855,21 +815,19 @@ def close(self): # Now wrap it with our instrumentation. - wrap_in_function(module, 'ServerHandler.run', - wrap_wsgi_application_entry_point) + wrap_in_function(module, "ServerHandler.run", wrap_wsgi_application_entry_point) def instrument_django_contrib_staticfiles_views(module): - if not hasattr(module.serve, '_nr_django_view_handler'): + if not hasattr(module.serve, "_nr_django_view_handler"): module.serve = wrap_view_handler(module.serve, priority=3) def instrument_django_contrib_staticfiles_handlers(module): - wrap_transaction_name(module, 'StaticFilesHandler.serve') + wrap_transaction_name(module, "StaticFilesHandler.serve") def instrument_django_views_debug(module): - # Wrap methods for handling errors when Django debug # enabled. For 404 we give this higher naming priority over # any prior middleware or view handler to give them @@ -878,10 +836,8 @@ def instrument_django_views_debug(module): # from a middleware or view handler in place so error # details identify the correct transaction. - module.technical_404_response = wrap_view_handler( - module.technical_404_response, priority=3) - module.technical_500_response = wrap_view_handler( - module.technical_500_response, priority=1) + module.technical_404_response = wrap_view_handler(module.technical_404_response, priority=3) + module.technical_500_response = wrap_view_handler(module.technical_500_response, priority=1) def resolve_view_handler(view, request): @@ -890,8 +846,7 @@ def resolve_view_handler(view, request): # duplicate the lookup mechanism. if request.method.lower() in view.http_method_names: - handler = getattr(view, request.method.lower(), - view.http_method_not_allowed) + handler = getattr(view, request.method.lower(), view.http_method_not_allowed) else: handler = view.http_method_not_allowed @@ -899,7 +854,6 @@ def resolve_view_handler(view, request): def wrap_view_dispatch(wrapped): - # Wrapper to be applied to dispatcher for class based views. def wrapper(wrapped, instance, args, kwargs): @@ -936,7 +890,7 @@ def _args(request, *args, **kwargs): priority = 4 - if transaction.group == 'Function': + if transaction.group == "Function": if transaction.name == callable_name(view): priority = 5 @@ -953,22 +907,22 @@ def instrument_django_views_generic_base(module): def instrument_django_http_multipartparser(module): - wrap_function_trace(module, 'MultiPartParser.parse') + wrap_function_trace(module, "MultiPartParser.parse") def instrument_django_core_mail(module): - wrap_function_trace(module, 'mail_admins') - wrap_function_trace(module, 'mail_managers') - wrap_function_trace(module, 'send_mail') + wrap_function_trace(module, "mail_admins") + wrap_function_trace(module, "mail_managers") + wrap_function_trace(module, "send_mail") def instrument_django_core_mail_message(module): - wrap_function_trace(module, 'EmailMessage.send') + wrap_function_trace(module, "EmailMessage.send") def _nr_wrapper_BaseCommand___init___(wrapped, instance, args, kwargs): instance.handle = FunctionTraceWrapper(instance.handle) - if hasattr(instance, 'handle_noargs'): + if hasattr(instance, "handle_noargs"): instance.handle_noargs = FunctionTraceWrapper(instance.handle_noargs) return wrapped(*args, **kwargs) @@ -982,29 +936,24 @@ def _args(argv, *args, **kwargs): subcommand = _argv[1] commands = django_settings.instrumentation.scripts.django_admin - startup_timeout = \ - django_settings.instrumentation.background_task.startup_timeout + startup_timeout = django_settings.instrumentation.background_task.startup_timeout if subcommand not in commands: return wrapped(*args, **kwargs) application = register_application(timeout=startup_timeout) - return BackgroundTaskWrapper(wrapped, application, subcommand, 'Django')(*args, **kwargs) + return BackgroundTaskWrapper(wrapped, application, subcommand, "Django")(*args, **kwargs) def instrument_django_core_management_base(module): - wrap_function_wrapper(module, 'BaseCommand.__init__', - _nr_wrapper_BaseCommand___init___) - wrap_function_wrapper(module, 'BaseCommand.run_from_argv', - _nr_wrapper_BaseCommand_run_from_argv_) + wrap_function_wrapper(module, "BaseCommand.__init__", _nr_wrapper_BaseCommand___init___) + wrap_function_wrapper(module, "BaseCommand.run_from_argv", _nr_wrapper_BaseCommand_run_from_argv_) @function_wrapper -def _nr_wrapper_django_inclusion_tag_wrapper_(wrapped, instance, - args, kwargs): - - name = hasattr(wrapped, '__name__') and wrapped.__name__ +def _nr_wrapper_django_inclusion_tag_wrapper_(wrapped, instance, args, kwargs): + name = hasattr(wrapped, "__name__") and wrapped.__name__ if name is None: return wrapped(*args, **kwargs) @@ -1013,16 +962,14 @@ def _nr_wrapper_django_inclusion_tag_wrapper_(wrapped, instance, tags = django_settings.instrumentation.templates.inclusion_tag - if '*' not in tags and name not in tags and qualname not in tags: + if "*" not in tags and name not in tags and qualname not in tags: return wrapped(*args, **kwargs) - return FunctionTraceWrapper(wrapped, name=name, group='Template/Tag')(*args, **kwargs) + return FunctionTraceWrapper(wrapped, name=name, group="Template/Tag")(*args, **kwargs) @function_wrapper -def _nr_wrapper_django_inclusion_tag_decorator_(wrapped, instance, - args, kwargs): - +def _nr_wrapper_django_inclusion_tag_decorator_(wrapped, instance, args, kwargs): def _bind_params(func, *args, **kwargs): return func, args, kwargs @@ -1033,63 +980,53 @@ def _bind_params(func, *args, **kwargs): return wrapped(func, *_args, **_kwargs) -def _nr_wrapper_django_template_base_Library_inclusion_tag_(wrapped, - instance, args, kwargs): - - return _nr_wrapper_django_inclusion_tag_decorator_( - wrapped(*args, **kwargs)) +def _nr_wrapper_django_template_base_Library_inclusion_tag_(wrapped, instance, args, kwargs): + return _nr_wrapper_django_inclusion_tag_decorator_(wrapped(*args, **kwargs)) @function_wrapper -def _nr_wrapper_django_template_base_InclusionNode_render_(wrapped, - instance, args, kwargs): - +def _nr_wrapper_django_template_base_InclusionNode_render_(wrapped, instance, args, kwargs): if wrapped.__self__ is None: return wrapped(*args, **kwargs) - file_name = getattr(wrapped.__self__, '_nr_file_name', None) + file_name = getattr(wrapped.__self__, "_nr_file_name", None) if file_name is None: return wrapped(*args, **kwargs) name = wrapped.__self__._nr_file_name - return FunctionTraceWrapper(wrapped, name=name, group='Template/Include')(*args, **kwargs) - + return FunctionTraceWrapper(wrapped, name=name, group="Template/Include")(*args, **kwargs) -def _nr_wrapper_django_template_base_generic_tag_compiler_(wrapped, instance, - args, kwargs): +def _nr_wrapper_django_template_base_generic_tag_compiler_(wrapped, instance, args, kwargs): if wrapped.__code__.co_argcount > 6: # Django > 1.3. - def _bind_params(parser, token, params, varargs, varkw, defaults, - name, takes_context, node_class, *args, **kwargs): + def _bind_params( + parser, token, params, varargs, varkw, defaults, name, takes_context, node_class, *args, **kwargs + ): return node_class + else: # Django <= 1.3. - def _bind_params(params, defaults, name, node_class, parser, token, - *args, **kwargs): + def _bind_params(params, defaults, name, node_class, parser, token, *args, **kwargs): return node_class node_class = _bind_params(*args, **kwargs) - if node_class.__name__ == 'InclusionNode': + if node_class.__name__ == "InclusionNode": result = wrapped(*args, **kwargs) - result.render = ( - _nr_wrapper_django_template_base_InclusionNode_render_( - result.render)) + result.render = _nr_wrapper_django_template_base_InclusionNode_render_(result.render) return result return wrapped(*args, **kwargs) -def _nr_wrapper_django_template_base_Library_tag_(wrapped, instance, - args, kwargs): - +def _nr_wrapper_django_template_base_Library_tag_(wrapped, instance, args, kwargs): def _bind_params(name=None, compile_function=None, *args, **kwargs): return compile_function @@ -1099,21 +1036,21 @@ def _bind_params(name=None, compile_function=None, *args, **kwargs): return wrapped(*args, **kwargs) def _get_node_class(compile_function): - node_class = None # Django >= 1.4 uses functools.partial if isinstance(compile_function, functools.partial): - node_class = compile_function.keywords.get('node_class') + node_class = compile_function.keywords.get("node_class") # Django < 1.4 uses their home-grown "curry" function, # not functools.partial. - if (hasattr(compile_function, 'func_closure') and - hasattr(compile_function, '__name__') and - compile_function.__name__ == '_curried'): - + if ( + hasattr(compile_function, "func_closure") + and hasattr(compile_function, "__name__") + and compile_function.__name__ == "_curried" + ): # compile_function here is generic_tag_compiler(), which has been # curried. To get node_class, we first get the function obj, args, # and kwargs of the curried function from the cells in @@ -1121,19 +1058,20 @@ def _get_node_class(compile_function): # is not consistent from platform to platform, so we need to map # them to the variables in compile_function.__code__.co_freevars. - cells = dict(zip(compile_function.__code__.co_freevars, - (c.cell_contents for c in compile_function.func_closure))) + cells = dict( + zip(compile_function.__code__.co_freevars, (c.cell_contents for c in compile_function.func_closure)) + ) # node_class is the 4th arg passed to generic_tag_compiler() - if 'args' in cells and len(cells['args']) > 3: - node_class = cells['args'][3] + if "args" in cells and len(cells["args"]) > 3: + node_class = cells["args"][3] return node_class node_class = _get_node_class(compile_function) - if node_class is None or node_class.__name__ != 'InclusionNode': + if node_class is None or node_class.__name__ != "InclusionNode": return wrapped(*args, **kwargs) # Climb stack to find the file_name of the include template. @@ -1146,9 +1084,8 @@ def _get_node_class(compile_function): for i in range(1, stack_levels + 1): frame = sys._getframe(i) - if ('generic_tag_compiler' in frame.f_code.co_names and - 'file_name' in frame.f_code.co_freevars): - file_name = frame.f_locals.get('file_name') + if "generic_tag_compiler" in frame.f_code.co_names and "file_name" in frame.f_code.co_freevars: + file_name = frame.f_locals.get("file_name") if file_name is None: return wrapped(*args, **kwargs) @@ -1167,22 +1104,21 @@ def instrument_django_template_base(module): settings = global_settings() - if 'django.instrumentation.inclusion-tags.r1' in settings.feature_flag: + if "django.instrumentation.inclusion-tags.r1" in settings.feature_flag: + if hasattr(module, "generic_tag_compiler"): + wrap_function_wrapper( + module, "generic_tag_compiler", _nr_wrapper_django_template_base_generic_tag_compiler_ + ) - if hasattr(module, 'generic_tag_compiler'): - wrap_function_wrapper(module, 'generic_tag_compiler', - _nr_wrapper_django_template_base_generic_tag_compiler_) + if hasattr(module, "Library"): + wrap_function_wrapper(module, "Library.tag", _nr_wrapper_django_template_base_Library_tag_) - if hasattr(module, 'Library'): - wrap_function_wrapper(module, 'Library.tag', - _nr_wrapper_django_template_base_Library_tag_) - - wrap_function_wrapper(module, 'Library.inclusion_tag', - _nr_wrapper_django_template_base_Library_inclusion_tag_) + wrap_function_wrapper( + module, "Library.inclusion_tag", _nr_wrapper_django_template_base_Library_inclusion_tag_ + ) def _nr_wrap_converted_middleware_(middleware, name): - @function_wrapper def _wrapper(wrapped, instance, args, kwargs): transaction = current_transaction() @@ -1197,9 +1133,7 @@ def _wrapper(wrapped, instance, args, kwargs): return _wrapper(middleware) -def _nr_wrapper_convert_exception_to_response_(wrapped, instance, args, - kwargs): - +def _nr_wrapper_convert_exception_to_response_(wrapped, instance, args, kwargs): def _bind_params(original_middleware, *args, **kwargs): return original_middleware @@ -1213,22 +1147,19 @@ def _bind_params(original_middleware, *args, **kwargs): def instrument_django_core_handlers_exception(module): + if hasattr(module, "convert_exception_to_response"): + wrap_function_wrapper(module, "convert_exception_to_response", _nr_wrapper_convert_exception_to_response_) - if hasattr(module, 'convert_exception_to_response'): - wrap_function_wrapper(module, 'convert_exception_to_response', - _nr_wrapper_convert_exception_to_response_) - - if hasattr(module, 'handle_uncaught_exception'): - module.handle_uncaught_exception = ( - wrap_handle_uncaught_exception( - module.handle_uncaught_exception)) + if hasattr(module, "handle_uncaught_exception"): + module.handle_uncaught_exception = wrap_handle_uncaught_exception(module.handle_uncaught_exception) def instrument_django_core_handlers_asgi(module): import django - framework = ('Django', django.get_version()) + framework = ("Django", django.get_version()) - if hasattr(module, 'ASGIHandler'): + if hasattr(module, "ASGIHandler"): from newrelic.api.asgi_application import wrap_asgi_application - wrap_asgi_application(module, 'ASGIHandler.__call__', framework=framework) + + wrap_asgi_application(module, "ASGIHandler.__call__", framework=framework) diff --git a/newrelic/hooks/framework_flask.py b/newrelic/hooks/framework_flask.py index c0540a60d8..ed115d3456 100644 --- a/newrelic/hooks/framework_flask.py +++ b/newrelic/hooks/framework_flask.py @@ -28,12 +28,9 @@ from newrelic.api.wsgi_application import wrap_wsgi_application from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version - -def framework_details(): - import flask - - return ("Flask", getattr(flask, "__version__", None)) +FLASK_VERSION = ("Flask", get_package_version("flask")) def status_code(exc, value, tb): @@ -96,7 +93,7 @@ def _bind_params(rule, endpoint=None, view_func=None, **options): def _nr_wrapper_Flask_views_View_as_view_(wrapped, instance, args, kwargs): view = wrapped(*args, **kwargs) - view._nr_view_func_name = "%s:%s" % (view.__module__, view.__name__) + view._nr_view_func_name = f"{view.__module__}:{view.__name__}" return view @@ -166,7 +163,7 @@ def _nr_wrapper_error_handler_(wrapped, instance, args, kwargs): return FunctionTraceWrapper(wrapped, name=name)(*args, **kwargs) -def _nr_wrapper_Flask__register_error_handler_(wrapped, instance, args, kwargs): +def _nr_wrapper_Flask__register_error_handler_(wrapped, instance, args, kwargs): # pragma: no cover def _bind_params(key, code_or_exception, f): return key, code_or_exception, f @@ -189,7 +186,6 @@ def _bind_params(code_or_exception, f): def _nr_wrapper_Flask_try_trigger_before_first_request_functions_(wrapped, instance, args, kwargs): - transaction = current_transaction() if transaction is None: @@ -277,7 +273,7 @@ def instrument_flask_views(module): def instrument_flask_app(module): - wrap_wsgi_application(module, "Flask.wsgi_app", framework=framework_details) + wrap_wsgi_application(module, "Flask.wsgi_app", framework=FLASK_VERSION) wrap_function_wrapper(module, "Flask.add_url_rule", _nr_wrapper_Flask_add_url_rule_input_) @@ -355,7 +351,6 @@ def _nr_wrapper_Blueprint_endpoint_(wrapped, instance, args, kwargs): @function_wrapper def _nr_wrapper_Blueprint_before_request_wrapped_(wrapped, instance, args, kwargs): - transaction = current_transaction() if transaction is None: diff --git a/newrelic/hooks/framework_graphql.py b/newrelic/hooks/framework_graphql.py index d261b2e9fd..c555b3981a 100644 --- a/newrelic/hooks/framework_graphql.py +++ b/newrelic/hooks/framework_graphql.py @@ -13,7 +13,10 @@ # limitations under the License. import logging +import sys +import time from collections import deque +from inspect import isawaitable from newrelic.api.error_trace import ErrorTrace from newrelic.api.function_trace import FunctionTrace @@ -22,7 +25,14 @@ from newrelic.api.transaction import current_transaction, ignore_transaction from newrelic.common.object_names import callable_name, parse_exc_info from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version from newrelic.core.graphql_utils import graphql_statement +from newrelic.hooks.framework_graphql_py3 import ( + nr_coro_execute_name_wrapper, + nr_coro_graphql_impl_wrapper, + nr_coro_resolver_error_wrapper, + nr_coro_resolver_wrapper, +) _logger = logging.getLogger(__name__) @@ -32,23 +42,8 @@ VERSION = None -def framework_version(): - """Framework version string.""" - global VERSION - if VERSION is None: - from graphql import __version__ as version - - VERSION = version - - return VERSION - - -def graphql_version(): - """Minor version tuple.""" - version = framework_version() - - # Take first two values in version to avoid ValueErrors with pre-releases (ex: 3.2.0a0) - return tuple(int(v) for v in version.split(".")[:2]) +GRAPHQL_VERSION = get_package_version("graphql-core") +major_version = int(GRAPHQL_VERSION.split(".")[0]) def ignore_graphql_duplicate_exception(exc, val, tb): @@ -94,14 +89,6 @@ def wrap_executor_context_init(wrapped, instance, args, kwargs): return result -def bind_operation_v3(operation, root_value): - return operation - - -def bind_operation_v2(exe_context, operation, root_value): - return operation - - def wrap_execute_operation(wrapped, instance, args, kwargs): transaction = current_transaction() trace = current_trace() @@ -115,18 +102,13 @@ def wrap_execute_operation(wrapped, instance, args, kwargs): ) return wrapped(*args, **kwargs) - try: - operation = bind_operation_v3(*args, **kwargs) - except TypeError: - try: - operation = bind_operation_v2(*args, **kwargs) - except TypeError: - return wrapped(*args, **kwargs) + execution_context = instance - if graphql_version() < (3, 0): - execution_context = args[0] - else: - execution_context = instance + try: + # Works for both v3.2 and v3.3+ + operation = execution_context.operation + except (TypeError, AttributeError): + return wrapped(*args, **kwargs) trace.operation_name = get_node_value(operation, "name") or "" @@ -145,12 +127,17 @@ def wrap_execute_operation(wrapped, instance, args, kwargs): transaction.set_transaction_name(callable_name(wrapped), "GraphQL", priority=11) result = wrapped(*args, **kwargs) - if not execution_context.errors: - if hasattr(trace, "set_transaction_name"): + + def set_name(value=None): + if not execution_context.errors and hasattr(trace, "set_transaction_name"): # Operation trace sets transaction name trace.set_transaction_name(priority=14) + return value - return result + if isawaitable(result): + return nr_coro_execute_name_wrapper(wrapped, result, set_name) + else: + return set_name(result) def get_node_value(field, attr, subattr="value"): @@ -161,39 +148,25 @@ def get_node_value(field, attr, subattr="value"): def is_fragment_spread_node(field): - # Resolve version specific imports - try: - from graphql.language.ast import FragmentSpread - except ImportError: - from graphql import FragmentSpreadNode as FragmentSpread + from graphql.language.ast import FragmentSpreadNode - return isinstance(field, FragmentSpread) + return isinstance(field, FragmentSpreadNode) def is_fragment(field): - # Resolve version specific imports - try: - from graphql.language.ast import FragmentSpread, InlineFragment - except ImportError: - from graphql import FragmentSpreadNode as FragmentSpread - from graphql import InlineFragmentNode as InlineFragment - - _fragment_types = (InlineFragment, FragmentSpread) + from graphql.language.ast import FragmentSpreadNode, InlineFragmentNode + _fragment_types = (InlineFragmentNode, FragmentSpreadNode) return isinstance(field, _fragment_types) def is_named_fragment(field): - # Resolve version specific imports - try: - from graphql.language.ast import NamedType - except ImportError: - from graphql import NamedTypeNode as NamedType + from graphql.language.ast import NamedTypeNode return ( is_fragment(field) and getattr(field, "type_condition", None) is not None - and isinstance(field.type_condition, NamedType) + and isinstance(field.type_condition, NamedTypeNode) ) @@ -215,7 +188,7 @@ def traverse_deepest_unique_path(fields, fragments): if is_named_fragment(field): name = get_node_value(field.type_condition, "name") if name: - deepest_path.append("%s<%s>" % (deepest_path.pop(), name)) + deepest_path.append(f"{deepest_path.pop()}<{name}>") elif is_fragment(field): if len(list(fragments.values())) != 1: @@ -294,6 +267,7 @@ def wrap_get_field_resolver(wrapped, instance, args, kwargs): def wrap_get_field_def(wrapped, instance, args, kwargs): + """In v3.3+ this is called `get_field` and it is called from schema itself""" result = wrapped(*args, **kwargs) if hasattr(result, "resolve"): @@ -321,12 +295,25 @@ def wrap_resolver(wrapped, instance, args, kwargs): if transaction is None: return wrapped(*args, **kwargs) - name = callable_name(wrapped) + base_resolver = getattr(wrapped, "_nr_base_resolver", wrapped) + + name = callable_name(base_resolver) transaction.set_transaction_name(name, "GraphQL", priority=13) + trace = FunctionTrace(name, source=base_resolver) - with FunctionTrace(name, source=wrapped): - with ErrorTrace(ignore=ignore_graphql_duplicate_exception): - return wrapped(*args, **kwargs) + with ErrorTrace(ignore=ignore_graphql_duplicate_exception): + sync_start_time = time.time() + result = wrapped(*args, **kwargs) + + if isawaitable(result): + # Grab any async resolvers and wrap with traces + return nr_coro_resolver_error_wrapper( + wrapped, name, trace, ignore_graphql_duplicate_exception, result, transaction + ) + else: + with trace: + trace.start_time = sync_start_time + return result def wrap_error_handler(wrapped, instance, args, kwargs): @@ -364,44 +351,53 @@ def wrap_parse(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) -def bind_resolve_field_v3(parent_type, source, field_nodes, path): +def bind_resolve_field(parent_type, source, field_nodes, path, *args, **kwargs): + """This is called execute_field in GraphQL-core v3.3+""" return parent_type, field_nodes, path -def bind_resolve_field_v2(exe_context, parent_type, source, field_asts, parent_info, field_path): - return parent_type, field_asts, field_path - - def wrap_resolve_field(wrapped, instance, args, kwargs): + """This is called execute_field in GraphQL-core v3.3+""" transaction = current_transaction() if transaction is None: return wrapped(*args, **kwargs) - if graphql_version() < (3, 0): - bind_resolve_field = bind_resolve_field_v2 - else: - bind_resolve_field = bind_resolve_field_v3 - try: parent_type, field_asts, field_path = bind_resolve_field(*args, **kwargs) except TypeError: return wrapped(*args, **kwargs) field_name = field_asts[0].name.value - field_def = parent_type.fields.get(field_name) + field_def = parent_type.fields.get(field_name, None) field_return_type = str(field_def.type) if field_def else "" + if isinstance(field_path, list): + field_path = field_path[0] + else: + field_path = field_path.key - with GraphQLResolverTrace(field_name) as trace: - with ErrorTrace(ignore=ignore_graphql_duplicate_exception): - trace._add_agent_attribute("graphql.field.parentType", parent_type.name) - trace._add_agent_attribute("graphql.field.returnType", field_return_type) + trace = GraphQLResolverTrace( + field_name, field_parent_type=parent_type.name, field_return_type=field_return_type, field_path=field_path + ) + start_time = time.time() - if isinstance(field_path, list): - trace._add_agent_attribute("graphql.field.path", field_path[0]) - else: - trace._add_agent_attribute("graphql.field.path", field_path.key) + try: + result = wrapped(*args, **kwargs) + except Exception: + # Synchonous resolver with exception raised + with trace: + trace.start_time = start_time + notice_error(ignore=ignore_graphql_duplicate_exception) + raise - return wrapped(*args, **kwargs) + if isawaitable(result): + # Asynchronous resolvers (returned coroutines from non-coroutine functions) + # Return a coroutine that handles wrapping in a resolver trace + return nr_coro_resolver_wrapper(wrapped, trace, ignore_graphql_duplicate_exception, result) + else: + # Synchonous resolver with no exception raised + with trace: + trace.start_time = start_time + return result def bind_graphql_impl_query(schema, source, *args, **kwargs): @@ -428,11 +424,8 @@ def wrap_graphql_impl(wrapped, instance, args, kwargs): if not transaction: return wrapped(*args, **kwargs) - transaction.add_framework_info(name="GraphQL", version=framework_version()) - if graphql_version() < (3, 0): - bind_query = bind_execute_graphql_query - else: - bind_query = bind_graphql_impl_query + transaction.add_framework_info(name="GraphQL", version=GRAPHQL_VERSION) + bind_query = bind_graphql_impl_query try: schema, query = bind_query(*args, **kwargs) @@ -444,20 +437,42 @@ def wrap_graphql_impl(wrapped, instance, args, kwargs): transaction.set_transaction_name(callable_name(wrapped), "GraphQL", priority=10) - with GraphQLOperationTrace() as trace: - trace.statement = graphql_statement(query) + trace = GraphQLOperationTrace() - # Handle Schemas created from frameworks - if hasattr(schema, "_nr_framework"): - framework = schema._nr_framework - trace.product = framework[0] - transaction.add_framework_info(name=framework[0], version=framework[1]) + trace.statement = graphql_statement(query) + # Handle Schemas created from frameworks + if hasattr(schema, "_nr_framework"): + framework = schema._nr_framework + trace.product = framework[0] + transaction.add_framework_info(name=framework[0], version=framework[1]) + + # Trace must be manually started and stopped to ensure it exists prior to and during the entire duration of the query. + # Otherwise subsequent instrumentation will not be able to find an operation trace and will have issues. + trace.__enter__() + try: with ErrorTrace(ignore=ignore_graphql_duplicate_exception): result = wrapped(*args, **kwargs) + except Exception as e: + # Execution finished synchronously, exit immediately. + trace.__exit__(*sys.exc_info()) + raise + else: + if isawaitable(result): + # Asynchronous implementations + # Return a coroutine that handles closing the operation trace + return nr_coro_graphql_impl_wrapper(wrapped, trace, ignore_graphql_duplicate_exception, result) + else: + # Execution finished synchronously, exit immediately. + trace.__exit__(None, None, None) return result +def instrument_graphql_schema_get_field(module): + if hasattr(module, "GraphQLSchema") and hasattr(module.GraphQLSchema, "get_field"): + wrap_function_wrapper(module, "GraphQLSchema.get_field", wrap_get_field_def) + + def instrument_graphql_execute(module): if hasattr(module, "get_field_def"): wrap_function_wrapper(module, "get_field_def", wrap_get_field_def) @@ -480,11 +495,15 @@ def instrument_graphql_execute(module): def instrument_graphql_execution_utils(module): + if major_version == 2: + return if hasattr(module, "ExecutionContext"): wrap_function_wrapper(module, "ExecutionContext.__init__", wrap_executor_context_init) def instrument_graphql_execution_middleware(module): + if major_version == 2: + return if hasattr(module, "get_middleware_resolvers"): wrap_function_wrapper(module, "get_middleware_resolvers", wrap_get_middleware_resolvers) if hasattr(module, "MiddlewareManager"): @@ -492,20 +511,26 @@ def instrument_graphql_execution_middleware(module): def instrument_graphql_error_located_error(module): + if major_version == 2: + return if hasattr(module, "located_error"): wrap_function_wrapper(module, "located_error", wrap_error_handler) def instrument_graphql_validate(module): + if major_version == 2: + return wrap_function_wrapper(module, "validate", wrap_validate) def instrument_graphql(module): + if major_version == 2: + return if hasattr(module, "graphql_impl"): wrap_function_wrapper(module, "graphql_impl", wrap_graphql_impl) - if hasattr(module, "execute_graphql"): - wrap_function_wrapper(module, "execute_graphql", wrap_graphql_impl) def instrument_graphql_parser(module): + if major_version == 2: + return wrap_function_wrapper(module, "parse", wrap_parse) diff --git a/newrelic/hooks/framework_graphql_py3.py b/newrelic/hooks/framework_graphql_py3.py new file mode 100644 index 0000000000..3931aa6ed6 --- /dev/null +++ b/newrelic/hooks/framework_graphql_py3.py @@ -0,0 +1,68 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import functools +import sys + +from newrelic.api.error_trace import ErrorTrace +from newrelic.api.function_trace import FunctionTrace + + +def nr_coro_execute_name_wrapper(wrapped, result, set_name): + @functools.wraps(wrapped) + async def _nr_coro_execute_name_wrapper(): + result_ = await result + set_name() + return result_ + + return _nr_coro_execute_name_wrapper() + + +def nr_coro_resolver_error_wrapper(wrapped, name, trace, ignore, result, transaction): + @functools.wraps(wrapped) + async def _nr_coro_resolver_error_wrapper(): + with trace: + with ErrorTrace(ignore=ignore): + try: + return await result + except Exception: + transaction.set_transaction_name(name, "GraphQL", priority=15) + raise + + return _nr_coro_resolver_error_wrapper() + + +def nr_coro_resolver_wrapper(wrapped, trace, ignore, result): + @functools.wraps(wrapped) + async def _nr_coro_resolver_wrapper(): + with trace: + with ErrorTrace(ignore=ignore): + return await result + + return _nr_coro_resolver_wrapper() + +def nr_coro_graphql_impl_wrapper(wrapped, trace, ignore, result): + @functools.wraps(wrapped) + async def _nr_coro_graphql_impl_wrapper(): + try: + with ErrorTrace(ignore=ignore): + result_ = await result + except: + trace.__exit__(*sys.exc_info()) + raise + else: + trace.__exit__(None, None, None) + return result_ + + + return _nr_coro_graphql_impl_wrapper() \ No newline at end of file diff --git a/newrelic/hooks/framework_grpc.py b/newrelic/hooks/framework_grpc.py index 7b11cce03e..70f296132c 100644 --- a/newrelic/hooks/framework_grpc.py +++ b/newrelic/hooks/framework_grpc.py @@ -16,47 +16,40 @@ import time from newrelic.api.external_trace import ExternalTrace -from newrelic.api.web_transaction import WebTransactionWrapper -from newrelic.api.transaction import current_transaction from newrelic.api.time_trace import notice_error -from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.api.transaction import current_transaction +from newrelic.api.web_transaction import WebTransactionWrapper from newrelic.common.object_names import callable_name +from newrelic.common.object_wrapper import wrap_function_wrapper def _get_uri_method(instance, *args, **kwargs): - target = instance._channel.target().decode('utf-8') - method = instance._method.decode('utf-8').lstrip('/') - uri = 'grpc://%s/%s' % (target, method) + target = instance._channel.target().decode("utf-8").replace("dns:///", "") + method = instance._method.decode("utf-8").lstrip("/") + uri = f"grpc://{target}/{method}" return (uri, method) -def _prepare_request( - transaction, guid, request, - timeout=None, metadata=None, *args, **kwargs): +def _prepare_request(transaction, guid, request, timeout=None, metadata=None, *args, **kwargs): metadata = metadata and list(metadata) or [] dt_metadata = transaction._create_distributed_trace_data_with_guid(guid) - metadata.extend( - transaction._generate_distributed_trace_headers(dt_metadata) - ) + metadata.extend(transaction._generate_distributed_trace_headers(dt_metadata)) args = (request, timeout, metadata) + args return args, kwargs -def _prepare_request_stream( - transaction, guid, request_iterator, *args, **kwargs): - return _prepare_request( - transaction, guid, request_iterator, *args, **kwargs) +def _prepare_request_stream(transaction, guid, request_iterator, *args, **kwargs): + return _prepare_request(transaction, guid, request_iterator, *args, **kwargs) def wrap_call(module, object_path, prepare): - def _call_wrapper(wrapped, instance, args, kwargs): transaction = current_transaction() if transaction is None: return wrapped(*args, **kwargs) uri, method = _get_uri_method(instance) - with ExternalTrace('gRPC', uri, method, source=wrapped): + with ExternalTrace("gRPC", uri, method, source=wrapped): args, kwargs = prepare(transaction, None, *args, **kwargs) return wrapped(*args, **kwargs) @@ -64,19 +57,18 @@ def _call_wrapper(wrapped, instance, args, kwargs): def wrap_future(module, object_path, prepare): - def _future_wrapper(wrapped, instance, args, kwargs): transaction = current_transaction() if transaction is None: return wrapped(*args, **kwargs) - guid = '%016x' % random.getrandbits(64) + guid = f"{random.getrandbits(64):016x}" uri, method = _get_uri_method(instance) args, kwargs = prepare(transaction, guid, *args, **kwargs) future = wrapped(*args, **kwargs) future._nr_guid = guid - future._nr_args = {"library": 'gRPC', "url": uri, "method": method, "source": wrapped} + future._nr_args = {"library": "gRPC", "url": uri, "method": method, "source": wrapped} future._nr_start_time = time.time() # In non-streaming responses, result is typically called instead of @@ -89,16 +81,16 @@ def _future_wrapper(wrapped, instance, args, kwargs): def wrap_next(_wrapped, _instance, _args, _kwargs): - _nr_args = getattr(_instance, '_nr_args', None) + _nr_args = getattr(_instance, "_nr_args", None) if not _nr_args: return _wrapped(*_args, **_kwargs) try: return _wrapped(*_args, **_kwargs) except Exception: - delattr(_instance, '_nr_args') - _nr_start_time = getattr(_instance, '_nr_start_time', 0.0) - _nr_guid = getattr(_instance, '_nr_guid', None) + delattr(_instance, "_nr_args") + _nr_start_time = getattr(_instance, "_nr_start_time", 0.0) + _nr_guid = getattr(_instance, "_nr_guid", None) with ExternalTrace(**_nr_args) as t: t.start_time = _nr_start_time or t.start_time @@ -107,12 +99,12 @@ def wrap_next(_wrapped, _instance, _args, _kwargs): def wrap_result(_wrapped, _instance, _args, _kwargs): - _nr_args = getattr(_instance, '_nr_args', None) + _nr_args = getattr(_instance, "_nr_args", None) if not _nr_args: return _wrapped(*_args, **_kwargs) - delattr(_instance, '_nr_args') - _nr_start_time = getattr(_instance, '_nr_start_time', 0.0) - _nr_guid = getattr(_instance, '_nr_guid', None) + delattr(_instance, "_nr_args") + _nr_start_time = getattr(_instance, "_nr_start_time", 0.0) + _nr_guid = getattr(_instance, "_nr_guid", None) try: result = _wrapped(*_args, **_kwargs) @@ -136,31 +128,22 @@ def grpc_web_transaction(wrapped, instance, args, kwargs): rpc_event, behavior = _bind_transaction_args(*args, **kwargs) behavior_name = callable_name(behavior) - call_details = ( - getattr(rpc_event, 'call_details', None) or - getattr(rpc_event, 'request_call_details', None)) + call_details = getattr(rpc_event, "call_details", None) or getattr(rpc_event, "request_call_details", None) - metadata = ( - getattr(rpc_event, 'invocation_metadata', None) or - getattr(rpc_event, 'request_metadata', None)) + metadata = getattr(rpc_event, "invocation_metadata", None) or getattr(rpc_event, "request_metadata", None) host = port = None if call_details: try: - host, port = call_details.host.split(b':', 1) + host, port = call_details.host.split(b":", 1) except Exception: pass request_path = call_details.method return WebTransactionWrapper( - wrapped, - name=behavior_name, - request_path=request_path, - host=host, - port=port, - headers=metadata, - source=behavior)(*args, **kwargs) + wrapped, name=behavior_name, request_path=request_path, host=host, port=port, headers=metadata, source=behavior + )(*args, **kwargs) def _trailing_metadata(state, *args, **kwargs): @@ -185,44 +168,26 @@ def _nr_wrap_abort(wrapped, instance, args, kwargs): def instrument_grpc__channel(module): - wrap_call(module, '_UnaryUnaryMultiCallable.__call__', - _prepare_request) - wrap_call(module, '_UnaryUnaryMultiCallable.with_call', - _prepare_request) - wrap_future(module, '_UnaryUnaryMultiCallable.future', - _prepare_request) - wrap_future(module, '_UnaryStreamMultiCallable.__call__', - _prepare_request) - wrap_call(module, '_StreamUnaryMultiCallable.__call__', - _prepare_request_stream) - wrap_call(module, '_StreamUnaryMultiCallable.with_call', - _prepare_request_stream) - wrap_future(module, '_StreamUnaryMultiCallable.future', - _prepare_request_stream) - wrap_future(module, '_StreamStreamMultiCallable.__call__', - _prepare_request_stream) - if hasattr(module, '_MultiThreadedRendezvous'): - wrap_function_wrapper(module, '_MultiThreadedRendezvous.result', - wrap_result) - wrap_function_wrapper(module, '_MultiThreadedRendezvous._next', - wrap_next) + wrap_call(module, "_UnaryUnaryMultiCallable.__call__", _prepare_request) + wrap_call(module, "_UnaryUnaryMultiCallable.with_call", _prepare_request) + wrap_future(module, "_UnaryUnaryMultiCallable.future", _prepare_request) + wrap_future(module, "_UnaryStreamMultiCallable.__call__", _prepare_request) + wrap_call(module, "_StreamUnaryMultiCallable.__call__", _prepare_request_stream) + wrap_call(module, "_StreamUnaryMultiCallable.with_call", _prepare_request_stream) + wrap_future(module, "_StreamUnaryMultiCallable.future", _prepare_request_stream) + wrap_future(module, "_StreamStreamMultiCallable.__call__", _prepare_request_stream) + if hasattr(module, "_MultiThreadedRendezvous"): + wrap_function_wrapper(module, "_MultiThreadedRendezvous.result", wrap_result) + wrap_function_wrapper(module, "_MultiThreadedRendezvous._next", wrap_next) else: - wrap_function_wrapper(module, '_Rendezvous.result', - wrap_result) - wrap_function_wrapper(module, '_Rendezvous._next', - wrap_next) - wrap_function_wrapper(module, '_Rendezvous.cancel', - wrap_result) + wrap_function_wrapper(module, "_Rendezvous.result", wrap_result) + wrap_function_wrapper(module, "_Rendezvous._next", wrap_next) + wrap_function_wrapper(module, "_Rendezvous.cancel", wrap_result) def instrument_grpc_server(module): - wrap_function_wrapper(module, '_unary_response_in_pool', - grpc_web_transaction) - wrap_function_wrapper(module, '_stream_response_in_pool', - grpc_web_transaction) - wrap_function_wrapper(module, '_completion_code', - _nr_wrap_status_code) - wrap_function_wrapper(module, '_abortion_code', - _nr_wrap_status_code) - wrap_function_wrapper(module, '_abort', - _nr_wrap_abort) + wrap_function_wrapper(module, "_unary_response_in_pool", grpc_web_transaction) + wrap_function_wrapper(module, "_stream_response_in_pool", grpc_web_transaction) + wrap_function_wrapper(module, "_completion_code", _nr_wrap_status_code) + wrap_function_wrapper(module, "_abortion_code", _nr_wrap_status_code) + wrap_function_wrapper(module, "_abort", _nr_wrap_abort) diff --git a/newrelic/hooks/framework_pylons.py b/newrelic/hooks/framework_pylons.py deleted file mode 100644 index 9c5c457cd7..0000000000 --- a/newrelic/hooks/framework_pylons.py +++ /dev/null @@ -1,84 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import newrelic.api.transaction -import newrelic.api.transaction_name -import newrelic.api.function_trace -import newrelic.api.error_trace -import newrelic.api.object_wrapper -import newrelic.api.import_hook - -from newrelic.api.time_trace import notice_error - -def name_controller(self, environ, start_response): - action = environ['pylons.routes_dict']['action'] - return "%s.%s" % (newrelic.api.object_wrapper.callable_name(self), action) - -class capture_error(object): - def __init__(self, wrapped): - if isinstance(wrapped, tuple): - (instance, wrapped) = wrapped - else: - instance = None - self.__instance = instance - self.__wrapped = wrapped - - def __get__(self, instance, klass): - if instance is None: - return self - descriptor = self.__wrapped.__get__(instance, klass) - return self.__class__((instance, descriptor)) - - def __call__(self, *args, **kwargs): - current_transaction = newrelic.api.transaction.current_transaction() - if current_transaction: - webob_exc = newrelic.api.import_hook.import_module('webob.exc') - try: - return self.__wrapped(*args, **kwargs) - except webob_exc.HTTPException: - raise - except: # Catch all - notice_error() - raise - else: - return self.__wrapped(*args, **kwargs) - - def __getattr__(self, name): - return getattr(self.__wrapped, name) - -def instrument(module): - - if module.__name__ == 'pylons.wsgiapp': - newrelic.api.error_trace.wrap_error_trace(module, 'PylonsApp.__call__') - - elif module.__name__ == 'pylons.controllers.core': - newrelic.api.transaction_name.wrap_transaction_name( - module, 'WSGIController.__call__', name_controller) - newrelic.api.function_trace.wrap_function_trace( - module, 'WSGIController.__call__') - - def name_WSGIController_perform_call(self, func, args): - return newrelic.api.object_wrapper.callable_name(func) - - newrelic.api.function_trace.wrap_function_trace( - module, 'WSGIController._perform_call', - name_WSGIController_perform_call) - newrelic.api.object_wrapper.wrap_object( - module, 'WSGIController._perform_call', capture_error) - - elif module.__name__ == 'pylons.templating': - - newrelic.api.function_trace.wrap_function_trace(module, 'render_genshi') - newrelic.api.function_trace.wrap_function_trace(module, 'render_mako') - newrelic.api.function_trace.wrap_function_trace(module, 'render_jinja2') diff --git a/newrelic/hooks/framework_pyramid.py b/newrelic/hooks/framework_pyramid.py index efe3c4468e..ba5e5e07af 100644 --- a/newrelic/hooks/framework_pyramid.py +++ b/newrelic/hooks/framework_pyramid.py @@ -48,25 +48,23 @@ from newrelic.api.transaction import current_transaction from newrelic.api.wsgi_application import wrap_wsgi_application from newrelic.common.object_names import callable_name -from newrelic.common.object_wrapper import (FunctionWrapper, wrap_out_function, - wrap_function_wrapper) +from newrelic.common.object_wrapper import ( + FunctionWrapper, + wrap_function_wrapper, + wrap_out_function, +) +from newrelic.common.package_version_utils import get_package_version def instrument_pyramid_router(module): - pyramid_version = None + pyramid_version = get_package_version("pyramid") - try: - import pkg_resources - pyramid_version = pkg_resources.get_distribution('pyramid').version - except Exception: - pass - - wrap_wsgi_application( - module, 'Router.__call__', framework=('Pyramid', pyramid_version)) + wrap_wsgi_application(module, "Router.__call__", framework=("Pyramid", pyramid_version)) def status_code(exc, value, tb): from pyramid.httpexceptions import HTTPException + # Ignore certain exceptions based on HTTP status codes. if isinstance(value, HTTPException): @@ -75,6 +73,7 @@ def status_code(exc, value, tb): def should_ignore(exc, value, tb): from pyramid.exceptions import PredicateMismatch + # Always ignore PredicateMismatch as it is raised by views to force # subsequent views to be consulted when multi views are being used. # It isn't therefore strictly an error as such as a subsequent view @@ -100,9 +99,7 @@ def view_handler_wrapper(wrapped, instance, args, kwargs): # set exception views to priority=1 so they won't take precedence over # the original view callable - transaction.set_transaction_name( - name, - priority=1 if args and isinstance(args[0], Exception) else 2) + transaction.set_transaction_name(name, priority=1 if args and isinstance(args[0], Exception) else 2) with FunctionTrace(name, source=view_callable) as trace: try: @@ -114,7 +111,7 @@ def view_handler_wrapper(wrapped, instance, args, kwargs): def wrap_view_handler(mapped_view): - if hasattr(mapped_view, '_nr_wrapped'): + if hasattr(mapped_view, "_nr_wrapped"): # pragma: no cover return mapped_view else: wrapped = FunctionWrapper(mapped_view, view_handler_wrapper) @@ -157,7 +154,7 @@ def _wrapper(context, request): return wrapper(context, request) finally: attr = instance.attr - inst = getattr(request, '__view__', None) + inst = getattr(request, "__view__", None) if inst is not None: if attr: handler = getattr(inst, attr) @@ -166,7 +163,7 @@ def _wrapper(context, request): tracer.name = name tracer.add_code_level_metrics(handler) else: - method = getattr(inst, '__call__') + method = getattr(inst, "__call__") if method: name = callable_name(method) transaction.set_transaction_name(name, priority=2) @@ -180,22 +177,21 @@ def instrument_pyramid_config_views(module): # Location of the ViewDeriver class changed from pyramid.config to # pyramid.config.views so check if present before trying to update. - if hasattr(module, 'ViewDeriver'): - wrap_out_function(module, 'ViewDeriver.__call__', wrap_view_handler) - elif hasattr(module, 'Configurator'): - wrap_out_function(module, 'Configurator._derive_view', - wrap_view_handler) + if hasattr(module, "ViewDeriver"): # pragma: no cover + wrap_out_function(module, "ViewDeriver.__call__", wrap_view_handler) + elif hasattr(module, "Configurator"): + wrap_out_function(module, "Configurator._derive_view", wrap_view_handler) - if hasattr(module, 'DefaultViewMapper'): + if hasattr(module, "DefaultViewMapper"): module.DefaultViewMapper.map_class_requestonly = FunctionWrapper( - module.DefaultViewMapper.map_class_requestonly, - default_view_mapper_wrapper) + module.DefaultViewMapper.map_class_requestonly, default_view_mapper_wrapper + ) module.DefaultViewMapper.map_class_native = FunctionWrapper( - module.DefaultViewMapper.map_class_native, - default_view_mapper_wrapper) + module.DefaultViewMapper.map_class_native, default_view_mapper_wrapper + ) def instrument_pyramid_config_tweens(module): - wrap_function_wrapper(module, 'Tweens.add_explicit', wrap_add_tween) + wrap_function_wrapper(module, "Tweens.add_explicit", wrap_add_tween) - wrap_function_wrapper(module, 'Tweens.add_implicit', wrap_add_tween) + wrap_function_wrapper(module, "Tweens.add_implicit", wrap_add_tween) diff --git a/newrelic/hooks/framework_sanic.py b/newrelic/hooks/framework_sanic.py index 94b5179c28..4941751c2a 100644 --- a/newrelic/hooks/framework_sanic.py +++ b/newrelic/hooks/framework_sanic.py @@ -40,7 +40,7 @@ def _nr_wrapper_handler_(wrapped, instance, args, kwargs): if view_class: try: method = args[0].method.lower() - name = callable_name(view_class) + "." + method + name = f"{callable_name(view_class)}.{method}" view = getattr(view_class, method) except: pass diff --git a/newrelic/hooks/framework_strawberry.py b/newrelic/hooks/framework_strawberry.py index 92a0ea8b4e..e6d06bb042 100644 --- a/newrelic/hooks/framework_strawberry.py +++ b/newrelic/hooks/framework_strawberry.py @@ -16,20 +16,14 @@ from newrelic.api.error_trace import ErrorTrace from newrelic.api.graphql_trace import GraphQLOperationTrace from newrelic.api.transaction import current_transaction -from newrelic.api.transaction_name import TransactionNameWrapper from newrelic.common.object_names import callable_name from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version from newrelic.core.graphql_utils import graphql_statement -from newrelic.hooks.framework_graphql import ( - framework_version as graphql_framework_version, -) -from newrelic.hooks.framework_graphql import ignore_graphql_duplicate_exception +from newrelic.hooks.framework_graphql import GRAPHQL_VERSION, ignore_graphql_duplicate_exception - -def framework_details(): - import strawberry - - return ("Strawberry", getattr(strawberry, "__version__", None)) +STRAWBERRY_GRAPHQL_VERSION = get_package_version("strawberry-graphql") +strawberry_version_tuple = tuple(map(int, STRAWBERRY_GRAPHQL_VERSION.split("."))) def bind_execute(query, *args, **kwargs): @@ -47,9 +41,8 @@ def wrap_execute_sync(wrapped, instance, args, kwargs): except TypeError: return wrapped(*args, **kwargs) - framework = framework_details() - transaction.add_framework_info(name=framework[0], version=framework[1]) - transaction.add_framework_info(name="GraphQL", version=graphql_framework_version()) + transaction.add_framework_info(name="Strawberry", version=STRAWBERRY_GRAPHQL_VERSION) + transaction.add_framework_info(name="GraphQL", version=GRAPHQL_VERSION) if hasattr(query, "body"): query = query.body @@ -74,9 +67,8 @@ async def wrap_execute(wrapped, instance, args, kwargs): except TypeError: return await wrapped(*args, **kwargs) - framework = framework_details() - transaction.add_framework_info(name=framework[0], version=framework[1]) - transaction.add_framework_info(name="GraphQL", version=graphql_framework_version()) + transaction.add_framework_info(name="Strawberry", version=STRAWBERRY_GRAPHQL_VERSION) + transaction.add_framework_info(name="GraphQL", version=GRAPHQL_VERSION) if hasattr(query, "body"): query = query.body @@ -98,19 +90,20 @@ def wrap_from_resolver(wrapped, instance, args, kwargs): result = wrapped(*args, **kwargs) try: - field = bind_from_resolver(*args, **kwargs) + field = bind_from_resolver(*args, **kwargs) except TypeError: pass else: if hasattr(field, "base_resolver"): if hasattr(field.base_resolver, "wrapped_func"): - resolver_name = callable_name(field.base_resolver.wrapped_func) - result = TransactionNameWrapper(result, resolver_name, "GraphQL", priority=13) + result._nr_base_resolver = field.base_resolver.wrapped_func return result def instrument_strawberry_schema(module): + if strawberry_version_tuple < (0, 23, 3): + return if hasattr(module, "Schema"): if hasattr(module.Schema, "execute"): wrap_function_wrapper(module, "Schema.execute", wrap_execute) @@ -119,11 +112,15 @@ def instrument_strawberry_schema(module): def instrument_strawberry_asgi(module): + if strawberry_version_tuple < (0, 23, 3): + return if hasattr(module, "GraphQL"): - wrap_asgi_application(module, "GraphQL.__call__", framework=framework_details()) + wrap_asgi_application(module, "GraphQL.__call__", framework=("Strawberry", STRAWBERRY_GRAPHQL_VERSION)) def instrument_strawberry_schema_converter(module): + if strawberry_version_tuple < (0, 23, 3): + return if hasattr(module, "GraphQLCoreConverter"): if hasattr(module.GraphQLCoreConverter, "from_resolver"): wrap_function_wrapper(module, "GraphQLCoreConverter.from_resolver", wrap_from_resolver) diff --git a/newrelic/hooks/framework_tornado.py b/newrelic/hooks/framework_tornado.py index bfdd4dba5d..658670a663 100644 --- a/newrelic/hooks/framework_tornado.py +++ b/newrelic/hooks/framework_tornado.py @@ -415,7 +415,7 @@ def instrument_tornado_web(module): track_loop_time) -class TornadoContext(object): +class TornadoContext(): def __init__(self): self.transaction = None diff --git a/newrelic/hooks/framework_web2py.py b/newrelic/hooks/framework_web2py.py deleted file mode 100644 index e9785e02f5..0000000000 --- a/newrelic/hooks/framework_web2py.py +++ /dev/null @@ -1,255 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import os - -import newrelic.api.transaction -import newrelic.api.import_hook -import newrelic.api.wsgi_application -import newrelic.api.external_trace -import newrelic.api.function_trace -import newrelic.api.transaction_name -import newrelic.api.object_wrapper -import newrelic.api.pre_function - -from newrelic.api.time_trace import notice_error - -def instrument_gluon_compileapp(module): - - # Wrap the run_models_in() function as first phase - # in executing a request after URL has been mapped - # to a specific view. The name given to the web - # transaction is combination of the application name - # and view path. - - def transaction_name_run_models_in(environment): - return '%s::%s' % (environment['request'].application, - environment['response'].view) - - newrelic.api.transaction_name.wrap_transaction_name(module, - 'run_models_in', name=transaction_name_run_models_in, - group='Web2Py') - - # Wrap functions which coordinate the execution of - # the separate models, controller and view phases of - # the request handling. This is done for timing how - # long taken within these phases of request - # handling. - - def name_function_run_models_in(environment): - return '%s/%s' % (environment['request'].controller, - environment['request'].function) - - newrelic.api.function_trace.wrap_function_trace(module, - 'run_models_in', name=name_function_run_models_in, - group='Python/Web2Py/Models') - - def name_function_run_controller_in(controller, function, environment): - return '%s/%s' % (controller, function) - - newrelic.api.function_trace.wrap_function_trace(module, - 'run_controller_in', name=name_function_run_controller_in, - group='Python/Web2Py/Controller') - - def name_function_run_view_in(environment): - return '%s/%s' % (environment['request'].controller, - environment['request'].function) - - newrelic.api.function_trace.wrap_function_trace(module, - 'run_view_in', name=name_function_run_view_in, - group='Python/Web2Py/View') - -def instrument_gluon_restricted(module): - - # Wrap function which executes all the compiled - # Python code files. The name used corresponds to - # path of the resource within the context of the - # application directory. The group used is either - # 'Script/Execute' or 'Template/Render' based on - # whether we can work out whether code object - # corresponded to compiled template file or not. - - def name_function_restricted(code, environment={}, layer='Unknown'): - if 'request' in environment: - folder = environment['request'].folder - if layer.startswith(folder): - return layer[len(folder):] - return layer - - def group_function_restricted(code, environment={}, layer='Unknown'): - parts = layer.split('.') - if parts[-1] in ['html'] or parts[-2:] in [['html','pyc']] : - return 'Template/Render' - return 'Script/Execute' - - newrelic.api.function_trace.wrap_function_trace(module, 'restricted', - name=name_function_restricted, group=group_function_restricted) - -def instrument_gluon_main(module): - - newrelic.api.wsgi_application.wrap_wsgi_application(module, 'wsgibase') - - # Wrap main function which dispatches the various - # phases of a request in order to capture any - # errors. Need to use a custom object wrapper as we - # need to ignore exceptions of type HTTP as that - # type of exception is used to programmatically - # return a valid response. For the case of a 404, - # where we want to name the web transactions as - # such, we pick that up later. - - class error_serve_controller(object): - def __init__(self, wrapped): - newrelic.api.object_wrapper.update_wrapper(self, wrapped) - self._nr_next_object = wrapped - if not hasattr(self, '_nr_last_object'): - self._nr_last_object = wrapped - def __call__(self, request, response, session): - txn = newrelic.api.transaction.current_transaction() - if txn: - HTTP = newrelic.api.import_hook.import_module('gluon.http').HTTP - try: - return self._nr_next_object(request, response, session) - except HTTP: - raise - except: # Catch all - notice_error() - raise - else: - return self._nr_next_object(request, response, session) - def __getattr__(self, name): - return getattr(self._nr_next_object, name) - - newrelic.api.object_wrapper.wrap_object( - module, 'serve_controller', error_serve_controller) - -def instrument_gluon_template(module): - - # Wrap parsing/compilation of template files, using - # the name of the template relative to the context - # of the application it is contained in. Use a group - # of 'Template/Compile'. Rendering of template is - # picked up when executing the code object created - # from this compilation step. - - def name_function_parse_template(filename, path='views/', - context=dict(), *args, **kwargs): - if 'request' in context: - folder = context['request'].folder - if path.startswith(folder): - return '%s/%s' % (path[len(folder):], filename) - else: - return '%s/%s' % (path, filename) - - newrelic.api.function_trace.wrap_function_trace(module, 'parse_template', - name=name_function_parse_template, group='Template/Compile') - -def instrument_gluon_tools(module): - - # Wrap utility function for fetching an external URL. - - def url_external_fetch(url, *args, **kwargs): - return url - - newrelic.api.external_trace.wrap_external_trace( - module, 'fetch', library='gluon.tools.fetch', - url=url_external_fetch) - - # Wrap utility function for fetching GEOCODE data. - # The URL in this case is hardwired in code to point - # at Google service and not part of arguments to we - # need to hard code it here as well. - - newrelic.api.external_trace.wrap_external_trace( - module, 'geocode', library='gluon.tools.geocode', - url='http://maps.google.com/maps/geo') - -def instrument_gluon_http(module): - - # This one is tricky. The only way to pick up that a - # static file is being served up is to wrap the to() - # method of a HTTP response object when actual - # response is being generated. We need to qualify - # this so only actually do anything when called from - # the wsgibase() function within 'gluon.main'. To do - # this need to go stack diving and look back at the - # parent stack frame. Doing that we can look at - # details of where calling code is located as well - # as sneak a peak at local variables in the calling - # stack to determine if we were handling a static - # file and what type of file was being served. - # Normally static file URLs would be left alone but - # don't want to risk black hole rule and instead - # generate custom wildcard URLs with precedence to - # extension. When can work out how to reliably get - # the application name then can incorporate that - # into the pattern as well in style used for web - # transaction names for views. The application name - # should normally be the first path segment, but the - # fact that arbitrary rewrite rules can be used may - # mean that isn't always the case. - - def transaction_name_name_not_found(response, *args, **kwargs): - txn = newrelic.api.transaction.current_transaction() - if not txn: - return - - frame = sys._getframe(1) - - if os.path.split(frame.f_code.co_filename)[-1] == 'pre_function.py': - frame = frame.f_back - - if os.path.split(frame.f_code.co_filename)[-1] != 'main.py': - return - - if frame.f_code.co_name != 'wsgibase': - return - - if response.status == 400: - txn.set_transaction_name('400', 'Uri') - return - - if response.status == 404: - txn.set_transaction_name('404', 'Uri') - return - - if 'static_file' not in frame.f_locals: - return - - if frame.f_locals['static_file']: - if 'environ' in frame.f_locals: - environ = frame.f_locals['environ'] - path_info = environ.get('PATH_INFO', '') - - if path_info: - parts = os.path.split(path_info) - if parts[1] == '': - if parts[0] == '/': - txn.set_transaction_name('*', 'Web2Py') - else: - name = '%s/*' % parts[0].lstrip('/') - txn.set_transaction_name(name, 'Web2Py') - else: - extension = os.path.splitext(parts[1])[-1] - name = '%s/*%s' % (parts[0].lstrip('/'), extension) - txn.set_transaction_name(name, 'Web2Py') - else: - txn.set_transaction_name('*', 'Web2Py') - - else: - txn.set_transaction_name('*', 'Web2Py') - - newrelic.api.pre_function.wrap_pre_function( - module, 'HTTP.to', transaction_name_name_not_found) diff --git a/newrelic/hooks/framework_webpy.py b/newrelic/hooks/framework_webpy.py index c1785a89f3..d89c979557 100644 --- a/newrelic/hooks/framework_webpy.py +++ b/newrelic/hooks/framework_webpy.py @@ -12,47 +12,44 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - -import newrelic.packages.six as six import newrelic.api.transaction import newrelic.api.function_trace import newrelic.api.in_function import newrelic.api.out_function import newrelic.api.pre_function -from newrelic.api.object_wrapper import callable_name +from newrelic.common.object_names import callable_name from newrelic.api.wsgi_application import WSGIApplicationWrapper from newrelic.api.time_trace import notice_error + def transaction_name_delegate(*args, **kwargs): transaction = newrelic.api.transaction.current_transaction() if transaction: - if isinstance(args[1], six.string_types): + if isinstance(args[1], str): f = args[1] else: f = callable_name(args[1]) transaction.set_transaction_name(f) return (args, kwargs) + def wrap_handle_exception(self): transaction = newrelic.api.transaction.current_transaction() if transaction: notice_error() + def template_name(render_obj, name): return name + def instrument(module): - if module.__name__ == 'web.application': - newrelic.api.out_function.wrap_out_function( - module, 'application.wsgifunc', WSGIApplicationWrapper) - newrelic.api.in_function.wrap_in_function( - module, 'application._delegate', transaction_name_delegate) - newrelic.api.pre_function.wrap_pre_function( - module, 'application.internalerror', wrap_handle_exception) - - elif module.__name__ == 'web.template': - newrelic.api.function_trace.wrap_function_trace( - module, 'render.__getattr__', template_name, 'Template/Render') + if module.__name__ == "web.application": + newrelic.api.out_function.wrap_out_function(module, "application.wsgifunc", WSGIApplicationWrapper) + newrelic.api.in_function.wrap_in_function(module, "application._delegate", transaction_name_delegate) + newrelic.api.pre_function.wrap_pre_function(module, "application.internalerror", wrap_handle_exception) + + elif module.__name__ == "web.template": + newrelic.api.function_trace.wrap_function_trace(module, "render.__getattr__", template_name, "Template/Render") diff --git a/newrelic/hooks/logger_logging.py b/newrelic/hooks/logger_logging.py index 67fb46525c..a11c2b328c 100644 --- a/newrelic/hooks/logger_logging.py +++ b/newrelic/hooks/logger_logging.py @@ -12,16 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +from urllib.parse import quote + from newrelic.api.application import application_instance from newrelic.api.time_trace import get_linking_metadata from newrelic.api.transaction import current_transaction, record_log_event from newrelic.common.object_wrapper import function_wrapper, wrap_function_wrapper from newrelic.core.config import global_settings -try: - from urllib import quote -except ImportError: - from urllib.parse import quote + +IGNORED_LOG_RECORD_KEYS = set(["message", "msg"]) def add_nr_linking_metadata(message): @@ -32,8 +32,8 @@ def add_nr_linking_metadata(message): trace_id = available_metadata.get("trace.id", "") hostname = available_metadata.get("hostname", "") - nr_linking_str = "|".join(("NR-LINKING", entity_guid, hostname, trace_id, span_id, entity_name)) - return "%s %s|" % (message, nr_linking_str) + nr_linking_str = f"NR-LINKING|{entity_guid}|{hostname}|{trace_id}|{span_id}|{entity_name}" + return f"{message} {nr_linking_str}|" @function_wrapper @@ -65,17 +65,27 @@ def wrap_callHandlers(wrapped, instance, args, kwargs): if settings.application_logging.metrics and settings.application_logging.metrics.enabled: if transaction: transaction.record_custom_metric("Logging/lines", {"count": 1}) - transaction.record_custom_metric("Logging/lines/%s" % level_name, {"count": 1}) + transaction.record_custom_metric(f"Logging/lines/{level_name}", {"count": 1}) else: application = application_instance(activate=False) if application and application.enabled: application.record_custom_metric("Logging/lines", {"count": 1}) - application.record_custom_metric("Logging/lines/%s" % level_name, {"count": 1}) + application.record_custom_metric(f"Logging/lines/{level_name}", {"count": 1}) if settings.application_logging.forwarding and settings.application_logging.forwarding.enabled: try: - message = record.getMessage() - record_log_event(message, level_name, int(record.created * 1000)) + message = record.msg + if not isinstance(message, dict): + # Allow python to convert the message to a string and template it with args. + message = record.getMessage() + + # Grab and filter context attributes from log record + record_attrs = vars(record) + context_attrs = {k: record_attrs[k] for k in record_attrs if k not in IGNORED_LOG_RECORD_KEYS} + + record_log_event( + message=message, level=level_name, timestamp=int(record.created * 1000), attributes=context_attrs + ) except Exception: pass diff --git a/newrelic/hooks/logger_loguru.py b/newrelic/hooks/logger_loguru.py index 9e7ed3eaef..3c21d4d8a6 100644 --- a/newrelic/hooks/logger_loguru.py +++ b/newrelic/hooks/logger_loguru.py @@ -18,19 +18,23 @@ from newrelic.api.application import application_instance from newrelic.api.transaction import current_transaction, record_log_event from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version_tuple from newrelic.common.signature import bind_args from newrelic.core.config import global_settings from newrelic.hooks.logger_logging import add_nr_linking_metadata -from newrelic.packages import six _logger = logging.getLogger(__name__) -is_pypy = hasattr(sys, "pypy_version_info") +IS_PYPY = hasattr(sys, "pypy_version_info") +LOGURU_FILTERED_RECORD_ATTRS = {"extra", "message", "time", "level", "_nr_original_message", "record"} +ALLOWED_LOGURU_OPTIONS_LENGTHS = frozenset((8, 9)) -def loguru_version(): - from loguru import __version__ - return tuple(int(x) for x in __version__.split(".")) +def _filter_record_attributes(record): + attrs = {k: v for k, v in record.items() if k not in LOGURU_FILTERED_RECORD_ATTRS} + extra_attrs = dict(record.get("extra", {})) + attrs.update({f"extra.{k}": v for k, v in extra_attrs.items()}) + return attrs def _nr_log_forwarder(message_instance): @@ -51,23 +55,25 @@ def _nr_log_forwarder(message_instance): if settings.application_logging.metrics and settings.application_logging.metrics.enabled: if transaction: transaction.record_custom_metric("Logging/lines", {"count": 1}) - transaction.record_custom_metric("Logging/lines/%s" % level_name, {"count": 1}) + transaction.record_custom_metric(f"Logging/lines/{level_name}", {"count": 1}) else: application = application_instance(activate=False) if application and application.enabled: application.record_custom_metric("Logging/lines", {"count": 1}) - application.record_custom_metric("Logging/lines/%s" % level_name, {"count": 1}) + application.record_custom_metric(f"Logging/lines/{level_name}", {"count": 1}) if settings.application_logging.forwarding and settings.application_logging.forwarding.enabled: + attrs = _filter_record_attributes(record) + try: - record_log_event(message, level_name, int(record["time"].timestamp())) + time = record.get("time", None) + if time: + time = int(time.timestamp() * 1000) + record_log_event(message, level_name, time, attributes=attrs) except Exception: pass -ALLOWED_LOGURU_OPTIONS_LENGTHS = frozenset((8, 9)) - - def wrap_log(wrapped, instance, args, kwargs): try: bound_args = bind_args(wrapped, args, kwargs) @@ -78,7 +84,7 @@ def wrap_log(wrapped, instance, args, kwargs): # Loguru looks into the stack trace to find the caller's module and function names. # options[1] tells loguru how far up to look in the stack trace to find the caller. # Because wrap_log is an extra call in the stack trace, loguru needs to look 1 level higher. - if not is_pypy: + if not IS_PYPY: options[1] += 1 else: # PyPy inspection requires an additional frame of offset, as the wrapt internals seem to @@ -86,7 +92,7 @@ def wrap_log(wrapped, instance, args, kwargs): options[1] += 2 except Exception as e: - _logger.debug("Exception in loguru handling: %s" % str(e)) + _logger.debug(f"Exception in loguru handling: {str(e)}") return wrapped(*args, **kwargs) else: return wrapped(**bound_args) @@ -109,18 +115,15 @@ def _nr_log_patcher(record): record["_nr_original_message"] = message = record["message"] record["message"] = add_nr_linking_metadata(message) - if loguru_version() > (0, 6, 0): - if original_patcher is not None: - patchers = [p for p in original_patcher] # Consumer iterable into list so we can modify - # Wipe out reference so patchers aren't called twice, as the framework will handle calling other patchers. - original_patcher = None - else: - patchers = [] - - patchers.append(_nr_log_patcher) - return patchers + if original_patcher is not None: + patchers = [p for p in original_patcher] # Consumer iterable into list so we can modify + # Wipe out reference so patchers aren't called twice, as the framework will handle calling other patchers. + original_patcher = None else: - return _nr_log_patcher + patchers = [] + + patchers.append(_nr_log_patcher) + return patchers def wrap_Logger_init(wrapped, instance, args, kwargs): @@ -134,8 +137,8 @@ def patch_loguru_logger(logger): if not hasattr(logger._core, "_nr_instrumented"): logger.add(_nr_log_forwarder, format="{message}") logger._core._nr_instrumented = True - elif not hasattr(logger, "_nr_instrumented"): - for _, handler in six.iteritems(logger._handlers): + elif not hasattr(logger, "_nr_instrumented"): # pragma: no cover + for _, handler in logger._handlers.items(): if handler._writer is _nr_log_forwarder: logger._nr_instrumented = True return diff --git a/newrelic/hooks/logger_structlog.py b/newrelic/hooks/logger_structlog.py new file mode 100644 index 0000000000..f07a85fd58 --- /dev/null +++ b/newrelic/hooks/logger_structlog.py @@ -0,0 +1,135 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools + +from newrelic.api.application import application_instance +from newrelic.api.transaction import current_transaction, record_log_event +from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.signature import bind_args +from newrelic.core.config import global_settings +from newrelic.hooks.logger_logging import add_nr_linking_metadata + + +@functools.lru_cache(maxsize=None) +def normalize_level_name(method_name): + # Look up level number for method name, using result to look up level name for that level number. + # Convert result to upper case, and default to UNKNOWN in case of errors or missing values. + try: + from structlog._log_levels import _LEVEL_TO_NAME, _NAME_TO_LEVEL + + return _LEVEL_TO_NAME[_NAME_TO_LEVEL[method_name]].upper() + except Exception: + return "UNKNOWN" + + +def bind_process_event(method_name, event, event_kw): + return method_name, event, event_kw + + +def new_relic_event_consumer(logger, level, event): + transaction = current_transaction() + + if transaction: + settings = transaction.settings + else: + settings = global_settings() + + # Return early if application logging not enabled + if settings and settings.application_logging.enabled: + if isinstance(event, (str, bytes, bytearray)): + message = original_message = event + event_attrs = {} + elif isinstance(event, dict): + message = original_message = event.get("event", "") + event_attrs = {k: v for k, v in event.items() if k != "event"} + else: + # Unclear how to proceed, ignore log. Avoid logging an error message or we may incur an infinite loop. + return event + + if settings.application_logging.local_decorating.enabled: + message = add_nr_linking_metadata(message) + if isinstance(event, (str, bytes, bytearray)): + event = message + elif isinstance(event, dict) and "event" in event: + event["event"] = message + + level_name = normalize_level_name(level) + + if settings.application_logging.metrics.enabled: + if transaction: + transaction.record_custom_metric("Logging/lines", {"count": 1}) + transaction.record_custom_metric(f"Logging/lines/{level_name}", {"count": 1}) + else: + application = application_instance(activate=False) + if application and application.enabled: + application.record_custom_metric("Logging/lines", {"count": 1}) + application.record_custom_metric(f"Logging/lines/{level_name}", {"count": 1}) + + if settings.application_logging.forwarding.enabled: + try: + record_log_event(original_message, level_name, attributes=event_attrs) + + except Exception: + pass + + return event + + +def wrap__process_event(wrapped, instance, args, kwargs): + transaction = current_transaction() + + if transaction: + settings = transaction.settings + else: + settings = global_settings() + + # Return early if application logging not enabled + if settings and settings.application_logging.enabled: + processors = instance._processors + if not processors: + instance._processors = [new_relic_event_consumer] + else: + instance._processors = processors = list(processors) + if processors[-1] != new_relic_event_consumer: + # Remove our processor if it exists and add it to the end + if new_relic_event_consumer in processors: + processors.remove(new_relic_event_consumer) + processors.append(new_relic_event_consumer) + + return wrapped(*args, **kwargs) + + +def wrap__find_first_app_frame_and_name(wrapped, instance, args, kwargs): + try: + bound_args = bind_args(wrapped, args, kwargs) + if bound_args["additional_ignores"]: + bound_args["additional_ignores"] = list(bound_args["additional_ignores"]) + bound_args["additional_ignores"].append("newrelic") + else: + bound_args["additional_ignores"] = ["newrelic"] + except Exception: + return wrapped(*args, **kwargs) + + return wrapped(**bound_args) + + +def instrument_structlog__base(module): + if hasattr(module, "BoundLoggerBase") and hasattr(module.BoundLoggerBase, "_process_event"): + wrap_function_wrapper(module, "BoundLoggerBase._process_event", wrap__process_event) + + +def instrument_structlog__frames(module): + if hasattr(module, "_find_first_app_frame_and_name"): + wrap_function_wrapper(module, "_find_first_app_frame_and_name", wrap__find_first_app_frame_and_name) diff --git a/newrelic/hooks/messagebroker_confluentkafka.py b/newrelic/hooks/messagebroker_confluentkafka.py index 81d9fa59af..626056e1dc 100644 --- a/newrelic/hooks/messagebroker_confluentkafka.py +++ b/newrelic/hooks/messagebroker_confluentkafka.py @@ -56,19 +56,25 @@ def wrap_Producer_produce(wrapped, instance, args, kwargs): args = args[1:] else: topic = kwargs.pop("topic", None) + topic = topic or "Default" transaction.add_messagebroker_info("Confluent-Kafka", get_package_version("confluent-kafka")) + if hasattr(instance, "_nr_bootstrap_servers"): + for server_name in instance._nr_bootstrap_servers: + transaction.record_custom_metric(f"MessageBroker/Kafka/Nodes/{server_name}/Produce/{topic}", 1) with MessageTrace( library="Kafka", operation="Produce", destination_type="Topic", - destination_name=topic or "Default", + destination_name=topic, source=wrapped, - ) as trace: - dt_headers = {k: v.encode("utf-8") for k, v in trace.generate_request_headers(transaction)} + ): + dt_headers = {k: v.encode("utf-8") for k, v in MessageTrace.generate_request_headers(transaction)} # headers can be a list of tuples or a dict so convert to dict for consistency. - dt_headers.update(dict(headers) if headers else {}) + if headers: + dt_headers.update(dict(headers)) + try: return wrapped(topic, headers=dt_headers, *args, **kwargs) except Exception as error: @@ -160,10 +166,15 @@ def wrap_Consumer_poll(wrapped, instance, args, kwargs): # Don't add metrics if there was an inactive transaction. # Name the metrics using the same format as the transaction, but in case the active transaction # was an existing one and not a message transaction, reproduce the naming logic here. - group = "Message/%s/%s" % (library, destination_type) - name = "Named/%s" % destination_name - transaction.record_custom_metric("%s/%s/Received/Bytes" % (group, name), received_bytes) - transaction.record_custom_metric("%s/%s/Received/Messages" % (group, name), message_count) + group = f"Message/{library}/{destination_type}" + name = f"Named/{destination_name}" + transaction.record_custom_metric(f"{group}/{name}/Received/Bytes", received_bytes) + transaction.record_custom_metric(f"{group}/{name}/Received/Messages", message_count) + if hasattr(instance, "_nr_bootstrap_servers"): + for server_name in instance._nr_bootstrap_servers: + transaction.record_custom_metric( + f"MessageBroker/Kafka/Nodes/{server_name}/Consume/{destination_name}", 1 + ) transaction.add_messagebroker_info("Confluent-Kafka", get_package_version("confluent-kafka")) return record @@ -189,8 +200,8 @@ def _wrap_serializer(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) topic = args[1].topic - group = "%s/Kafka/Topic" % group_prefix - name = "Named/%s/%s" % (topic, serializer_name) + group = f"{group_prefix}/Kafka/Topic" + name = f"Named/{topic}/{serializer_name}" return FunctionTraceWrapper(wrapped, name=name, group=group)(*args, **kwargs) @@ -217,6 +228,32 @@ def wrap_DeserializingConsumer_init(wrapped, instance, args, kwargs): instance._value_deserializer = wrap_serializer("Deserialization/Value", "Message")(instance._value_deserializer) +def wrap_Producer_init(wrapped, instance, args, kwargs): + wrapped(*args, **kwargs) + + # Try to capture the boostrap server info that is passed in in the configuration. + try: + conf = args[0] + servers = conf.get("bootstrap.servers") + if servers: + instance._nr_bootstrap_servers = servers.split(",") + except Exception: + pass + + +def wrap_Consumer_init(wrapped, instance, args, kwargs): + wrapped(*args, **kwargs) + + # Try to capture the boostrap server info that is passed in in the configuration. + try: + conf = args[0] + servers = conf.get("bootstrap.servers") + if servers: + instance._nr_bootstrap_servers = servers.split(",") + except Exception: + pass + + def wrap_immutable_class(module, class_name): # Wrap immutable binary extension class with a mutable Python subclass new_class = type(class_name, (getattr(module, class_name),), {}) @@ -228,10 +265,12 @@ def instrument_confluentkafka_cimpl(module): if hasattr(module, "Producer"): wrap_immutable_class(module, "Producer") wrap_function_wrapper(module, "Producer.produce", wrap_Producer_produce) + wrap_function_wrapper(module, "Producer.__init__", wrap_Producer_init) if hasattr(module, "Consumer"): wrap_immutable_class(module, "Consumer") wrap_function_wrapper(module, "Consumer.poll", wrap_Consumer_poll) + wrap_function_wrapper(module, "Consumer.__init__", wrap_Consumer_init) def instrument_confluentkafka_serializing_producer(module): diff --git a/newrelic/hooks/messagebroker_kafkapython.py b/newrelic/hooks/messagebroker_kafkapython.py index 9124a16dcd..763bbc67a0 100644 --- a/newrelic/hooks/messagebroker_kafkapython.py +++ b/newrelic/hooks/messagebroker_kafkapython.py @@ -47,22 +47,33 @@ def wrap_KafkaProducer_send(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) topic, value, key, headers, partition, timestamp_ms = _bind_send(*args, **kwargs) + topic = topic or "Default" headers = list(headers) if headers else [] - transaction.add_messagebroker_info("Kafka-Python", get_package_version("kafka-python")) + transaction.add_messagebroker_info( + "Kafka-Python", get_package_version("kafka-python") or get_package_version("kafka-python-ng") + ) with MessageTrace( library="Kafka", operation="Produce", destination_type="Topic", - destination_name=topic or "Default", + destination_name=topic, source=wrapped, terminal=False, - ) as trace: - dt_headers = [(k, v.encode("utf-8")) for k, v in trace.generate_request_headers(transaction)] - headers.extend(dt_headers) + ): + dt_headers = [(k, v.encode("utf-8")) for k, v in MessageTrace.generate_request_headers(transaction)] + # headers can be a list of tuples or a dict so convert to dict for consistency. + if headers: + dt_headers.extend(headers) + + if hasattr(instance, "config"): + for server_name in instance.config.get("bootstrap_servers", []): + transaction.record_custom_metric(f"MessageBroker/Kafka/Nodes/{server_name}/Produce/{topic}", 1) try: - return wrapped(topic, value=value, key=key, headers=headers, partition=partition, timestamp_ms=timestamp_ms) + return wrapped( + topic, value=value, key=key, headers=dt_headers, partition=partition, timestamp_ms=timestamp_ms + ) except Exception: notice_error() raise @@ -143,11 +154,18 @@ def wrap_kafkaconsumer_next(wrapped, instance, args, kwargs): # Don't add metrics if there was an inactive transaction. # Name the metrics using the same format as the transaction, but in case the active transaction # was an existing one and not a message transaction, reproduce the naming logic here. - group = "Message/%s/%s" % (library, destination_type) - name = "Named/%s" % destination_name - transaction.record_custom_metric("%s/%s/Received/Bytes" % (group, name), received_bytes) - transaction.record_custom_metric("%s/%s/Received/Messages" % (group, name), message_count) - transaction.add_messagebroker_info("Kafka-Python", get_package_version("kafka-python")) + group = f"Message/{library}/{destination_type}" + name = f"Named/{destination_name}" + transaction.record_custom_metric(f"{group}/{name}/Received/Bytes", received_bytes) + transaction.record_custom_metric(f"{group}/{name}/Received/Messages", message_count) + if hasattr(instance, "config"): + for server_name in instance.config.get("bootstrap_servers", []): + transaction.record_custom_metric( + f"MessageBroker/Kafka/Nodes/{server_name}/Consume/{destination_name}", 1 + ) + transaction.add_messagebroker_info( + "Kafka-Python", get_package_version("kafka-python") or get_package_version("kafka-python-ng") + ) return record @@ -180,8 +198,8 @@ def serialize(self, topic, object): if not current_transaction(): return wrapped(*args, **kwargs) - group = "%s/Kafka/Topic" % self._nr_group_prefix - name = "Named/%s/%s" % (topic, self._nr_serializer_name) + group = f"{self._nr_group_prefix}/Kafka/Topic" + name = f"Named/{topic}/{self._nr_serializer_name}" return FunctionTraceWrapper(wrapped, name=name, group=group)(*args, **kwargs) @@ -204,8 +222,8 @@ def _wrap_serializer(wrapped, instance, args, kwargs): if message_trace: topic = message_trace.destination_name - group = "%s/Kafka/Topic" % group_prefix - name = "Named/%s/%s" % (topic, serializer_name) + group = f"{group_prefix}/Kafka/Topic" + name = f"Named/{topic}/{serializer_name}" return FunctionTraceWrapper(wrapped, name=name, group=group)(*args, **kwargs) diff --git a/newrelic/hooks/messagebroker_pika.py b/newrelic/hooks/messagebroker_pika.py index cecc1b9343..58cab675e7 100644 --- a/newrelic/hooks/messagebroker_pika.py +++ b/newrelic/hooks/messagebroker_pika.py @@ -31,12 +31,25 @@ wrap_object, ) -_START_KEY = "_nr_start_time" KWARGS_ERROR = "Supportability/hooks/pika/kwargs_error" -def _add_consume_rabbitmq_trace(transaction, method, properties, nr_start_time, queue_name=None): +def instance_info(channel): + # Only host is currently used, so we only extract that. + try: + connection = channel.connection + if not hasattr(connection, "params") and hasattr(connection, "_impl"): + # Check for _impl attribute used by BlockingConnection to wrap actual connection objects + connection = connection._impl + host = connection.params.host + except Exception: + host = None + + return host + + +def _add_consume_rabbitmq_trace(transaction, method, properties, nr_start_time, queue_name=None, channel=None): routing_key = None if hasattr(method, "routing_key"): routing_key = method.routing_key @@ -80,7 +93,16 @@ def _add_consume_rabbitmq_trace(transaction, method, properties, nr_start_time, params=params, ) trace.__enter__() + + # Set start time and attributes after trace has started trace.start_time = nr_start_time + + # Extract host from channel to add as an agent attribute + host = instance_info(channel) + if trace and host: + trace._add_agent_attribute("server.address", host) + + # Exit trace immediately and complete trace.__exit__(None, None, None) @@ -127,9 +149,15 @@ def _nr_wrapper_basic_publish(wrapped, instance, args, kwargs): destination_name=exchange or "Default", params=params, source=wrapped, - ): + ) as trace: cat_headers = MessageTrace.generate_request_headers(transaction) properties.headers.update(cat_headers) + + # Extract host from channel to add as an agent attribute + host = instance_info(instance) + if trace and host: + trace._add_agent_attribute("server.address", host) + return wrapped(*args, **kwargs) @@ -145,9 +173,15 @@ def callback_wrapper(callback, _instance, _args, _kwargs): if not _kwargs: method, properties = _args[1:3] start_time = getattr(callback_wrapper, "_nr_start_time", None) + channel = getattr(callback_wrapper, "_nr_channel", None) _add_consume_rabbitmq_trace( - transaction, method=method, properties=properties, nr_start_time=start_time, queue_name=queue + transaction, + method=method, + properties=properties, + nr_start_time=start_time, + queue_name=queue, + channel=channel, ) else: m = transaction._transaction_metrics.get(KWARGS_ERROR, 0) @@ -156,7 +190,12 @@ def callback_wrapper(callback, _instance, _args, _kwargs): name = callable_name(callback) return FunctionTraceWrapper(callback, name=name)(*_args, **_kwargs) - callback_wrapper._nr_start_time = time.time() + try: + callback_wrapper._nr_start_time = time.time() + callback_wrapper._nr_channel = instance + except Exception: + pass + queue, args, kwargs = wrap_get(callback_wrapper, *args, **kwargs) return wrapped(*args, **kwargs) @@ -197,7 +236,7 @@ def _wrap_basic_get_Channel(wrapper, queue, callback, *args, **kwargs): return queue, args, kwargs -def _wrap_basic_get_Channel_old(wrapper, callback=None, queue="", *args, **kwargs): +def _wrap_basic_get_Channel_old(wrapper, callback=None, queue="", *args, **kwargs): # pragma: no cover if callback is not None: callback = wrapper(callback) args = (callback, queue) + args @@ -267,6 +306,11 @@ def _possibly_create_traces(yielded): ) bt.__enter__() + # Extract host from channel to add as an agent attribute + host = instance_info(instance) + if bt and host: + bt._add_agent_attribute("server.address", host) + return bt def _generator(generator): @@ -279,7 +323,7 @@ def _generator(generator): if any(exc): to_throw = exc exc = (None, None, None) - yielded = generator.throw(*to_throw) + yielded = generator.throw(to_throw[1]) else: yielded = generator.send(value) @@ -368,9 +412,8 @@ def callback_wrapper(wrapped, instance, args, kwargs): correlation_id=correlation_id, source=wrapped, ) as mt: - # Improve transaction naming - _new_txn_name = "RabbitMQ/Exchange/%s/%s" % (exchange, name) + _new_txn_name = f"RabbitMQ/Exchange/{exchange}/{name}" mt.set_transaction_name(_new_txn_name, group="Message") # Record that something went horribly wrong @@ -378,6 +421,11 @@ def callback_wrapper(wrapped, instance, args, kwargs): m = mt._transaction_metrics.get(KWARGS_ERROR, 0) mt._transaction_metrics[KWARGS_ERROR] = m + 1 + # Extract host from channel to add as an agent attribute + host = instance_info(channel) + if mt and host: + mt._add_agent_attribute("server.address", host) + return wrapped(*args, **kwargs) queue, args, kwargs = wrap_consume(callback_wrapper, *args, **kwargs) @@ -404,7 +452,7 @@ def instrument_pika_adapters(module): version = tuple(int(num) for num in pika.__version__.split(".", 1)[0]) - if version[0] < 1: + if version[0] < 1: # pragma: no cover wrap_consume = _wrap_basic_consume_BlockingChannel_old else: wrap_consume = _wrap_basic_consume_Channel @@ -426,7 +474,7 @@ def instrument_pika_channel(module): version = tuple(int(num) for num in pika.__version__.split(".", 1)[0]) - if version[0] < 1: + if version[0] < 1: # pragma: no cover wrap_consume = _wrap_basic_consume_Channel_old wrap_get = _wrap_basic_get_Channel_old else: diff --git a/newrelic/hooks/middleware_flask_compress.py b/newrelic/hooks/middleware_flask_compress.py index 09e35b3cd2..f1cad224fb 100644 --- a/newrelic/hooks/middleware_flask_compress.py +++ b/newrelic/hooks/middleware_flask_compress.py @@ -19,34 +19,39 @@ from newrelic.common.object_wrapper import wrap_function_wrapper from newrelic.config import extra_settings -from newrelic.packages import six - _logger = logging.getLogger(__name__) _boolean_states = { - '1': True, 'yes': True, 'true': True, 'on': True, - '0': False, 'no': False, 'false': False, 'off': False + "1": True, + "yes": True, + "true": True, + "on": True, + "0": False, + "no": False, + "false": False, + "off": False, } def _setting_boolean(value): if value.lower() not in _boolean_states: - raise ValueError('Not a boolean: %s' % value) + raise ValueError(f"Not a boolean: {value}") return _boolean_states[value.lower()] _settings_types = { - 'browser_monitoring.auto_instrument': _setting_boolean, - 'browser_monitoring.auto_instrument_passthrough': _setting_boolean, + "browser_monitoring.auto_instrument": _setting_boolean, + "browser_monitoring.auto_instrument_passthrough": _setting_boolean, } _settings_defaults = { - 'browser_monitoring.auto_instrument': True, - 'browser_monitoring.auto_instrument_passthrough': True, + "browser_monitoring.auto_instrument": True, + "browser_monitoring.auto_instrument_passthrough": True, } -flask_compress_settings = extra_settings('import-hook:flask_compress', - types=_settings_types, defaults=_settings_defaults) +flask_compress_settings = extra_settings( + "import-hook:flask_compress", types=_settings_types, defaults=_settings_defaults +) def _nr_wrapper_Compress_after_request(wrapped, instance, args, kwargs): @@ -62,7 +67,7 @@ def _params(response, *args, **kwargs): if not transaction: return wrapped(*args, **kwargs) - # Only insert RUM JavaScript headers and footers if enabled + # Only insert RUM JavaScript headers if enabled # in configuration and not already likely inserted. if not transaction.settings.browser_monitoring.enabled: @@ -83,45 +88,34 @@ def _params(response, *args, **kwargs): # a user may want to also perform insertion for # 'application/xhtml+xml'. - ctype = (response.mimetype or '').lower() + ctype = (response.mimetype or "").lower() if ctype not in transaction.settings.browser_monitoring.content_type: return wrapped(*args, **kwargs) # Don't risk it if content encoding already set. - if 'Content-Encoding' in response.headers: + if "Content-Encoding" in response.headers: return wrapped(*args, **kwargs) # Don't risk it if content is actually within an attachment. - cdisposition = response.headers.get('Content-Disposition', '').lower() + cdisposition = response.headers.get("Content-Disposition", "").lower() - if cdisposition.split(';')[0].strip() == 'attachment': + if cdisposition.split(";")[0].strip() == "attachment": return wrapped(*args, **kwargs) - # No point continuing if header is empty. This can occur if - # RUM is not enabled within the UI. It is assumed at this - # point that if header is not empty, then footer will not be - # empty. We don't want to generate the footer just yet as - # want to do that as late as possible so that application - # server time in footer is as accurate as possible. In - # particular, if the response content is generated on demand - # then the flattening of the response could take some time - # and we want to track that. We thus generate footer below - # at point of insertion. - - header = transaction.browser_timing_header() - - if not header: - return wrapped(*args, **kwargs) + # No point continuing if header is empty. This can occur if RUM is not enabled within the UI. We don't want to + # generate the header just yet as we want to do that as late as possible so that application server time in header + # is as accurate as possible. In particular, if the response content is generated on demand then the flattening + # of the response could take some time and we want to track that. We thus generate header below at + # the point of insertion. # If the response has direct_passthrough flagged, then is # likely to be streaming a file or other large response. - direct_passthrough = getattr(response, 'direct_passthrough', None) + direct_passthrough = getattr(response, "direct_passthrough", None) if direct_passthrough: - if not (flask_compress_settings. - browser_monitoring.auto_instrument_passthrough): + if not (flask_compress_settings.browser_monitoring.auto_instrument_passthrough): return wrapped(*args, **kwargs) # In those cases, if the mimetype is still a supported browser @@ -131,34 +125,31 @@ def _params(response, *args, **kwargs): # # In order to do that, we have to disable direct_passthrough on the # response since we have to immediately read the contents of the file. - elif ctype == 'text/html': + elif ctype == "text/html": response.direct_passthrough = False else: return wrapped(*args, **kwargs) - def html_to_be_inserted(): - return six.b(header) + six.b(transaction.browser_timing_footer()) - # Make sure we flatten any content first as it could be # stored as a list of strings in the response object. We # assign it back to the response object to avoid having # multiple copies of the string in memory at the same time # as we progress through steps below. - result = insert_html_snippet(response.get_data(), html_to_be_inserted) + result = insert_html_snippet(response.get_data(), lambda: transaction.browser_timing_header().encode("latin-1")) if result is not None: if transaction.settings.debug.log_autorum_middleware: - _logger.debug('RUM insertion from flask_compress ' - 'triggered. Bytes added was %r.', - len(result) - len(response.get_data())) + _logger.debug( + "RUM insertion from flask_compress " "triggered. Bytes added was %r.", + len(result) - len(response.get_data()), + ) response.set_data(result) - response.headers['Content-Length'] = str(len(response.get_data())) + response.headers["Content-Length"] = str(len(response.get_data())) return wrapped(*args, **kwargs) def instrument_flask_compress(module): - wrap_function_wrapper(module, 'Compress.after_request', - _nr_wrapper_Compress_after_request) + wrap_function_wrapper(module, "Compress.after_request", _nr_wrapper_Compress_after_request) diff --git a/newrelic/hooks/middleware_weberror.py b/newrelic/hooks/middleware_weberror.py deleted file mode 100644 index c99f68799e..0000000000 --- a/newrelic/hooks/middleware_weberror.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from newrelic.api.external_trace import wrap_external_trace -from newrelic.api.function_trace import wrap_function_trace - -def instrument_weberror_errormiddleware(module): - - wrap_function_trace(module, 'handle_exception') - -def instrument_weberror_reporter(module): - - def smtp_url(reporter, *args, **kwargs): - return 'smtp://' + reporter.smtp_server - - wrap_external_trace(module, 'EmailReporter.report', 'weberror', smtp_url) - wrap_function_trace(module, 'EmailReporter.report') - - wrap_function_trace(module, 'LogReporter.report') - wrap_function_trace(module, 'FileReporter.report') diff --git a/newrelic/hooks/mlmodel_langchain.py b/newrelic/hooks/mlmodel_langchain.py new file mode 100644 index 0000000000..2ae76ef2cf --- /dev/null +++ b/newrelic/hooks/mlmodel_langchain.py @@ -0,0 +1,916 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +import traceback +import uuid + +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.time_trace import current_trace, get_trace_linking_metadata +from newrelic.api.transaction import current_transaction +from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version +from newrelic.common.signature import bind_args +from newrelic.core.config import global_settings +from newrelic.core.context import context_wrapper + +_logger = logging.getLogger(__name__) +LANGCHAIN_VERSION = get_package_version("langchain") +EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE = "Exception occurred in langchain instrumentation: While reporting an exception in langchain, another exception occurred. Report this issue to New Relic Support.\n%s" +RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in langchain instrumentation: Failed to record LLM events. Report this issue to New Relic Support.\n%s" +VECTORSTORE_CLASSES = { + "langchain_community.vectorstores.aerospike": "Aerospike", + "langchain_community.vectorstores.alibabacloud_opensearch": "AlibabaCloudOpenSearch", + "langchain_community.vectorstores.analyticdb": "AnalyticDB", + "langchain_community.vectorstores.annoy": "Annoy", + "langchain_community.vectorstores.apache_doris": "ApacheDoris", + "langchain_community.vectorstores.aperturedb": "ApertureDB", + "langchain_community.vectorstores.astradb": "AstraDB", + "langchain_community.vectorstores.atlas": "AtlasDB", + "langchain_community.vectorstores.awadb": "AwaDB", + "langchain_community.vectorstores.azure_cosmos_db_no_sql": "AzureCosmosDBNoSqlVectorSearch", + "langchain_community.vectorstores.azure_cosmos_db": "AzureCosmosDBVectorSearch", + "langchain_community.vectorstores.azuresearch": "AzureSearch", + "langchain_community.vectorstores.baiduvectordb": "BaiduVectorDB", + "langchain_community.vectorstores.bageldb": "Bagel", + "langchain_community.vectorstores.baiducloud_vector_search": "BESVectorStore", + "langchain_community.vectorstores.bigquery_vector_search": "BigQueryVectorSearch", + "langchain_community.vectorstores.cassandra": "Cassandra", + "langchain_community.vectorstores.chroma": "Chroma", + "langchain_community.vectorstores.clarifai": "Clarifai", + "langchain_community.vectorstores.clickhouse": "Clickhouse", + "langchain_community.vectorstores.couchbase": "CouchbaseVectorStore", + "langchain_community.vectorstores.dashvector": "DashVector", + "langchain_community.vectorstores.databricks_vector_search": "DatabricksVectorSearch", + "langchain_community.vectorstores.deeplake": "DeepLake", + "langchain_community.vectorstores.dingo": "Dingo", + "langchain_community.vectorstores.documentdb": "DocumentDBVectorSearch", + "langchain_community.vectorstores.duckdb": "DuckDB", + "langchain_community.vectorstores.ecloud_vector_search": "EcloudESVectorStore", + "langchain_community.vectorstores.elastic_vector_search": "ElasticVectorSearch", + # "langchain_community.vectorstores.elastic_vector_search": "ElasticKnnSearch", # Deprecated + "langchain_community.vectorstores.elasticsearch": "ElasticsearchStore", + "langchain_community.vectorstores.epsilla": "Epsilla", + "langchain_community.vectorstores.faiss": "FAISS", + "langchain_community.vectorstores.hanavector": "HanaDB", + "langchain_community.vectorstores.hippo": "Hippo", + "langchain_community.vectorstores.hologres": "Hologres", + "langchain_community.vectorstores.infinispanvs": "InfinispanVS", + "langchain_community.vectorstores.inmemory": "InMemoryVectorStore", + "langchain_community.vectorstores.kdbai": "KDBAI", + "langchain_community.vectorstores.kinetica": "Kinetica", + "langchain_community.vectorstores.lancedb": "LanceDB", + "langchain_community.vectorstores.lantern": "Lantern", + "langchain_community.vectorstores.llm_rails": "LLMRails", + "langchain_community.vectorstores.manticore_search": "ManticoreSearch", + "langchain_community.vectorstores.marqo": "Marqo", + "langchain_community.vectorstores.matching_engine": "MatchingEngine", + "langchain_community.vectorstores.meilisearch": "Meilisearch", + "langchain_community.vectorstores.milvus": "Milvus", + "langchain_community.vectorstores.momento_vector_index": "MomentoVectorIndex", + "langchain_community.vectorstores.mongodb_atlas": "MongoDBAtlasVectorSearch", + "langchain_community.vectorstores.myscale": "MyScale", + "langchain_community.vectorstores.neo4j_vector": "Neo4jVector", + "langchain_community.vectorstores.thirdai_neuraldb": ["NeuralDBClientVectorStore", "NeuralDBVectorStore"], + "langchain_community.vectorstores.nucliadb": "NucliaDB", + "langchain_community.vectorstores.oraclevs": "OracleVS", + "langchain_community.vectorstores.opensearch_vector_search": "OpenSearchVectorSearch", + "langchain_community.vectorstores.pathway": "PathwayVectorClient", + "langchain_community.vectorstores.pgembedding": "PGEmbedding", + "langchain_community.vectorstores.pgvecto_rs": "PGVecto_rs", + "langchain_community.vectorstores.pgvector": "PGVector", + "langchain_community.vectorstores.pinecone": "Pinecone", + "langchain_community.vectorstores.qdrant": "Qdrant", + "langchain_community.vectorstores.redis.base": "Redis", + "langchain_community.vectorstores.relyt": "Relyt", + "langchain_community.vectorstores.rocksetdb": "Rockset", + "langchain_community.vectorstores.scann": "ScaNN", + "langchain_community.vectorstores.semadb": "SemaDB", + "langchain_community.vectorstores.singlestoredb": "SingleStoreDB", + "langchain_community.vectorstores.sklearn": "SKLearnVectorStore", + "langchain_community.vectorstores.sqlitevec": "SQLiteVec", + "langchain_community.vectorstores.sqlitevss": "SQLiteVSS", + "langchain_community.vectorstores.starrocks": "StarRocks", + "langchain_community.vectorstores.supabase": "SupabaseVectorStore", + "langchain_community.vectorstores.surrealdb": "SurrealDBStore", + "langchain_community.vectorstores.tair": "Tair", + "langchain_community.vectorstores.tencentvectordb": "TencentVectorDB", + "langchain_community.vectorstores.tidb_vector": "TiDBVectorStore", + "langchain_community.vectorstores.tigris": "Tigris", + "langchain_community.vectorstores.tiledb": "TileDB", + "langchain_community.vectorstores.timescalevector": "TimescaleVector", + "langchain_community.vectorstores.typesense": "Typesense", + "langchain_community.vectorstores.upstash": "UpstashVectorStore", + "langchain_community.vectorstores.usearch": "USearch", + "langchain_community.vectorstores.vald": "Vald", + "langchain_community.vectorstores.vdms": "VDMS", + "langchain_community.vectorstores.vearch": "Vearch", + "langchain_community.vectorstores.vectara": "Vectara", + "langchain_community.vectorstores.vespa": "VespaStore", + "langchain_community.vectorstores.vlite": "VLite", + "langchain_community.vectorstores.weaviate": "Weaviate", + "langchain_community.vectorstores.xata": "XataVectorStore", + "langchain_community.vectorstores.yellowbrick": "Yellowbrick", + "langchain_community.vectorstores.zep_cloud": "ZepCloudVectorStore", + "langchain_community.vectorstores.zep": "ZepVectorStore", + "langchain_community.vectorstores.docarray.hnsw": "DocArrayHnswSearch", + "langchain_community.vectorstores.docarray.in_memory": "DocArrayInMemorySearch", +} + + +def bind_submit(func, *args, **kwargs): + return {"func": func, "args": args, "kwargs": kwargs} + + +def wrap_ContextThreadPoolExecutor_submit(wrapped, instance, args, kwargs): + trace = current_trace() + if not trace: + return wrapped(*args, **kwargs) + + # Use hardened function signature bind so we have safety net catchall of args and kwargs. + bound_args = bind_submit(*args, **kwargs) + bound_args["func"] = context_wrapper(bound_args["func"], trace=trace, strict=True) + return wrapped(bound_args["func"], *bound_args["args"], **bound_args["kwargs"]) + + +def _create_error_vectorstore_events(transaction, search_id, args, kwargs, linking_metadata, wrapped): + settings = transaction.settings if transaction.settings is not None else global_settings() + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + bound_args = bind_args(wrapped, args, kwargs) + request_query = bound_args["query"] + request_k = bound_args["k"] + llm_metadata_dict = _get_llm_metadata(transaction) + vectorstore_error_dict = { + "request.k": request_k, + "id": search_id, + "span_id": span_id, + "trace_id": trace_id, + "vendor": "langchain", + "ingest_source": "Python", + "error": True, + } + + if settings.ai_monitoring.record_content.enabled: + vectorstore_error_dict["request.query"] = request_query + + vectorstore_error_dict.update(llm_metadata_dict) + transaction.record_custom_event("LlmVectorSearch", vectorstore_error_dict) + + +async def wrap_asimilarity_search(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + transaction.add_ml_model_info("LangChain", LANGCHAIN_VERSION) + transaction._add_agent_attribute("llm", True) + + search_id = str(uuid.uuid4()) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/vectorstore/LangChain") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + response = await wrapped(*args, **kwargs) + except Exception as exc: + ft.notice_error(attributes={"vector_store_id": search_id}) + ft.__exit__(*sys.exc_info()) + _create_error_vectorstore_events(transaction, search_id, args, kwargs, linking_metadata, wrapped) + raise + ft.__exit__(None, None, None) + + if not response: + return response + + _record_vector_search_success(transaction, linking_metadata, ft, search_id, args, kwargs, response, wrapped) + return response + + +def wrap_similarity_search(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + transaction.add_ml_model_info("LangChain", LANGCHAIN_VERSION) + transaction._add_agent_attribute("llm", True) + + search_id = str(uuid.uuid4()) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/vectorstore/LangChain") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + response = wrapped(*args, **kwargs) + except Exception as exc: + ft.notice_error(attributes={"vector_store_id": search_id}) + ft.__exit__(*sys.exc_info()) + _create_error_vectorstore_events(transaction, search_id, args, kwargs, linking_metadata, wrapped) + raise + ft.__exit__(None, None, None) + + if not response: + return response + + _record_vector_search_success(transaction, linking_metadata, ft, search_id, args, kwargs, response, wrapped) + return response + + +def _record_vector_search_success(transaction, linking_metadata, ft, search_id, args, kwargs, response, wrapped): + settings = transaction.settings if transaction.settings is not None else global_settings() + bound_args = bind_args(wrapped, args, kwargs) + request_query = bound_args["query"] + request_k = bound_args["k"] + duration = ft.duration * 1000 + response_number_of_documents = len(response) + llm_metadata_dict = _get_llm_metadata(transaction) + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + + llm_vector_search = { + "request.k": request_k, + "duration": duration, + "response.number_of_documents": response_number_of_documents, + "span_id": span_id, + "trace_id": trace_id, + "id": search_id, + "vendor": "langchain", + "ingest_source": "Python", + } + + if settings.ai_monitoring.record_content.enabled: + llm_vector_search["request.query"] = request_query + + llm_vector_search.update(llm_metadata_dict) + transaction.record_custom_event("LlmVectorSearch", llm_vector_search) + + for index, doc in enumerate(response): + sequence = index + page_content = getattr(doc, "page_content") + metadata = getattr(doc, "metadata") or {} + + metadata_dict = {f"metadata.{key}": value for key, value in metadata.items()} + + llm_vector_search_result = { + "id": str(uuid.uuid4()), + "search_id": search_id, + "sequence": sequence, + "span_id": span_id, + "trace_id": trace_id, + "vendor": "langchain", + "ingest_source": "Python", + } + + if settings.ai_monitoring.record_content.enabled: + llm_vector_search_result["page_content"] = page_content + llm_vector_search_result.update(metadata_dict) + llm_vector_search_result.update(llm_metadata_dict) + transaction.record_custom_event("LlmVectorSearchResult", llm_vector_search_result) + + +def wrap_tool_sync_run(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("LangChain", LANGCHAIN_VERSION) + transaction._add_agent_attribute("llm", True) + + tool_id, metadata, tags, tool_input, tool_name, tool_description, run_args = _capture_tool_info( + instance, wrapped, args, kwargs + ) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/tool/LangChain") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + return_val = wrapped(**run_args) + except Exception as exc: + _record_tool_error( + instance, + transaction, + linking_metadata, + tags, + metadata, + tool_id, + tool_input, + tool_name, + tool_description, + ft, + ) + raise + ft.__exit__(None, None, None) + + if not return_val: + return return_val + + _record_tool_success( + instance, + transaction, + linking_metadata, + tags, + metadata, + tool_id, + tool_input, + tool_name, + tool_description, + ft, + return_val, + ) + return return_val + + +async def wrap_tool_async_run(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("LangChain", LANGCHAIN_VERSION) + transaction._add_agent_attribute("llm", True) + + tool_id, metadata, tags, tool_input, tool_name, tool_description, run_args = _capture_tool_info( + instance, wrapped, args, kwargs + ) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/tool/LangChain") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + return_val = await wrapped(**run_args) + except Exception as exc: + _record_tool_error( + instance, + transaction, + linking_metadata, + tags, + metadata, + tool_id, + tool_input, + tool_name, + tool_description, + ft, + ) + raise + ft.__exit__(None, None, None) + + if not return_val: + return return_val + + _record_tool_success( + instance, + transaction, + linking_metadata, + tags, + metadata, + tool_id, + tool_input, + tool_name, + tool_description, + ft, + return_val, + ) + return return_val + + +def _capture_tool_info(instance, wrapped, args, kwargs): + run_args = bind_args(wrapped, args, kwargs) + + tool_id = str(uuid.uuid4()) + metadata = run_args.get("metadata") or {} + metadata["nr_tool_id"] = tool_id + run_args["metadata"] = metadata + tags = run_args.get("tags") or [] + tool_input = run_args.get("tool_input") + tool_name = getattr(instance, "name", None) + tool_description = getattr(instance, "description", None) + return tool_id, metadata, tags, tool_input, tool_name, tool_description, run_args + + +def _record_tool_success( + instance, + transaction, + linking_metadata, + tags, + metadata, + tool_id, + tool_input, + tool_name, + tool_description, + ft, + response, +): + settings = transaction.settings if transaction.settings is not None else global_settings() + run_id = getattr(transaction, "_nr_tool_run_ids", {}).pop(tool_id, None) + # Update tags and metadata previously obtained from run_args with instance values + metadata.update(getattr(instance, "metadata", None) or {}) + tags.extend(getattr(instance, "tags", None) or []) + full_tool_event_dict = {f"metadata.{key}": value for key, value in metadata.items() if key != "nr_tool_id"} + full_tool_event_dict.update( + { + "id": tool_id, + "run_id": run_id, + "name": tool_name, + "description": tool_description, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "vendor": "langchain", + "ingest_source": "Python", + "duration": ft.duration * 1000, + "tags": tags or None, + } + ) + result = None + try: + result = str(response) + except Exception: + _logger.debug(f"Failed to convert tool response into a string.\n{traceback.format_exception(*sys.exc_info())}") + if settings.ai_monitoring.record_content.enabled: + full_tool_event_dict.update( + { + "input": tool_input, + "output": result, + } + ) + full_tool_event_dict.update(_get_llm_metadata(transaction)) + transaction.record_custom_event("LlmTool", full_tool_event_dict) + + +def _record_tool_error( + instance, transaction, linking_metadata, tags, metadata, tool_id, tool_input, tool_name, tool_description, ft +): + settings = transaction.settings if transaction.settings is not None else global_settings() + ft.notice_error( + attributes={ + "tool_id": tool_id, + } + ) + ft.__exit__(*sys.exc_info()) + run_id = getattr(transaction, "_nr_tool_run_ids", {}).pop(tool_id, None) + # Update tags and metadata previously obtained from run_args with instance values + metadata.update(getattr(instance, "metadata", None) or {}) + tags.extend(getattr(instance, "tags", None) or []) + + # Make sure the builtin attributes take precedence over metadata attributes. + error_tool_event_dict = {f"metadata.{key}": value for key, value in metadata.items() if key != "nr_tool_id"} + error_tool_event_dict.update( + { + "id": tool_id, + "run_id": run_id, + "name": tool_name, + "description": tool_description, + "span_id": linking_metadata.get("span.id"), + "trace_id": linking_metadata.get("trace.id"), + "vendor": "langchain", + "ingest_source": "Python", + "duration": ft.duration * 1000, + "tags": tags or None, + "error": True, + } + ) + if settings.ai_monitoring.record_content.enabled: + error_tool_event_dict["input"] = tool_input + error_tool_event_dict.update(_get_llm_metadata(transaction)) + transaction.record_custom_event("LlmTool", error_tool_event_dict) + + +def wrap_on_tool_start_sync(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + tool_id = _get_tool_id(instance) + run_manager = wrapped(*args, **kwargs) + _capture_tool_run_id(transaction, run_manager, tool_id) + return run_manager + + +async def wrap_on_tool_start_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + tool_id = _get_tool_id(instance) + run_manager = await wrapped(*args, **kwargs) + _capture_tool_run_id(transaction, run_manager, tool_id) + return run_manager + + +def _get_tool_id(instance): + return (getattr(instance, "metadata", None) or {}).pop("nr_tool_id", None) + + +def _capture_tool_run_id(transaction, run_manager, tool_id): + if tool_id: + if not hasattr(transaction, "_nr_tool_run_ids"): + transaction._nr_tool_run_ids = {} + if tool_id not in transaction._nr_tool_run_ids: + transaction._nr_tool_run_ids[tool_id] = getattr(run_manager, "run_id", None) + + +async def wrap_chain_async_run(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("LangChain", LANGCHAIN_VERSION) + transaction._add_agent_attribute("llm", True) + + run_args = bind_args(wrapped, args, kwargs) + completion_id = str(uuid.uuid4()) + add_nr_completion_id(run_args, completion_id) + # Check to see if launched from agent or directly from chain. + # The trace group will reflect from where it has started. + # The AgentExecutor class has an attribute "agent" that does + # not exist within the Chain class + group_name = "Llm/agent/LangChain" if hasattr(instance, "agent") else "Llm/chain/LangChain" + ft = FunctionTrace(name=wrapped.__name__, group=group_name) + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + response = await wrapped(input=run_args["input"], config=run_args["config"], **run_args.get("kwargs", {})) + except Exception as exc: + ft.notice_error( + attributes={ + "completion_id": completion_id, + } + ) + ft.__exit__(*sys.exc_info()) + _create_error_chain_run_events( + transaction, instance, run_args, completion_id, linking_metadata, ft.duration * 1000 + ) + raise + ft.__exit__(None, None, None) + + if not response: + return response + + _create_successful_chain_run_events( + transaction, instance, run_args, completion_id, response, linking_metadata, ft.duration * 1000 + ) + return response + + +def wrap_chain_sync_run(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("LangChain", LANGCHAIN_VERSION) + transaction._add_agent_attribute("llm", True) + + run_args = bind_args(wrapped, args, kwargs) + completion_id = str(uuid.uuid4()) + add_nr_completion_id(run_args, completion_id) + # Check to see if launched from agent or directly from chain. + # The trace group will reflect from where it has started. + # The AgentExecutor class has an attribute "agent" that does + # not exist within the Chain class + group_name = "Llm/agent/LangChain" if hasattr(instance, "agent") else "Llm/chain/LangChain" + ft = FunctionTrace(name=wrapped.__name__, group=group_name) + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + response = wrapped(input=run_args["input"], config=run_args["config"], **run_args.get("kwargs", {})) + except Exception as exc: + ft.notice_error( + attributes={ + "completion_id": completion_id, + } + ) + ft.__exit__(*sys.exc_info()) + _create_error_chain_run_events( + transaction, instance, run_args, completion_id, linking_metadata, ft.duration * 1000 + ) + raise + ft.__exit__(None, None, None) + + if not response: + return response + + _create_successful_chain_run_events( + transaction, instance, run_args, completion_id, response, linking_metadata, ft.duration * 1000 + ) + return response + + +def add_nr_completion_id(run_args, completion_id): + # invoke has an argument named "config" that contains metadata and tags. + # Add the nr_completion_id into the metadata to be used as the function call + # identifier when grabbing the run_id off the transaction. + metadata = (run_args.get("config") or {}).get("metadata") or {} + metadata["nr_completion_id"] = completion_id + if not run_args.get("config"): + run_args["config"] = {"metadata": metadata} + else: + run_args["config"]["metadata"] = metadata + + +def _create_error_chain_run_events(transaction, instance, run_args, completion_id, linking_metadata, duration): + _input = run_args.get("input") + llm_metadata_dict = _get_llm_metadata(transaction) + run_id, metadata, tags = _get_run_manager_info(transaction, run_args, instance, completion_id) + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + input_message_list = [_input] + + # Make sure the builtin attributes take precedence over metadata attributes. + full_chat_completion_summary_dict = {f"metadata.{key}": value for key, value in metadata.items()} + full_chat_completion_summary_dict.update( + { + "id": completion_id, + "span_id": span_id, + "trace_id": trace_id, + "vendor": "langchain", + "ingest_source": "Python", + "virtual_llm": True, + "request_id": run_id, + "duration": duration, + "response.number_of_messages": len(input_message_list), + "tags": tags, + "error": True, + } + ) + full_chat_completion_summary_dict.update(llm_metadata_dict) + transaction.record_custom_event("LlmChatCompletionSummary", full_chat_completion_summary_dict) + create_chat_completion_message_event( + transaction, + input_message_list, + completion_id, + span_id, + trace_id, + run_id, + llm_metadata_dict, + [], + ) + + +def _get_run_manager_info(transaction, run_args, instance, completion_id): + run_id = getattr(transaction, "_nr_chain_run_ids", {}).pop(completion_id, "") + # metadata and tags are keys in the config parameter. + metadata = {} + metadata.update((run_args.get("config") or {}).get("metadata") or {}) + # Do not report internal nr_completion_id in metadata. + metadata = {key: value for key, value in metadata.items() if key != "nr_completion_id"} + tags = [] + tags.extend((run_args.get("config") or {}).get("tags") or []) + return run_id, metadata, tags or None + + +def _get_llm_metadata(transaction): + # Grab LLM-related custom attributes off of the transaction to store as metadata on LLM events + custom_attrs_dict = transaction._custom_params + llm_metadata_dict = {key: value for key, value in custom_attrs_dict.items() if key.startswith("llm.")} + llm_context_attrs = getattr(transaction, "_llm_context_attrs", None) + if llm_context_attrs: + llm_metadata_dict.update(llm_context_attrs) + + return llm_metadata_dict + + +def _create_successful_chain_run_events( + transaction, instance, run_args, completion_id, response, linking_metadata, duration +): + _input = run_args.get("input") + llm_metadata_dict = _get_llm_metadata(transaction) + run_id, metadata, tags = _get_run_manager_info(transaction, run_args, instance, completion_id) + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + input_message_list = [_input] + output_message_list = [] + try: + output_message_list = [response[0]] if response else [] + except: + try: + output_message_list = [str(response)] + except Exception as e: + _logger.warning( + f"Unable to capture response inside langchain chain instrumentation. No response message event will be captured. Report this issue to New Relic Support.\n{traceback.format_exception(*sys.exc_info())}" + ) + + # Make sure the builtin attributes take precedence over metadata attributes. + full_chat_completion_summary_dict = {f"metadata.{key}": value for key, value in metadata.items()} + full_chat_completion_summary_dict.update( + { + "id": completion_id, + "span_id": span_id, + "trace_id": trace_id, + "vendor": "langchain", + "ingest_source": "Python", + "virtual_llm": True, + "request_id": run_id, + "duration": duration, + "response.number_of_messages": len(input_message_list) + len(output_message_list), + "tags": tags, + } + ) + full_chat_completion_summary_dict.update(llm_metadata_dict) + transaction.record_custom_event("LlmChatCompletionSummary", full_chat_completion_summary_dict) + create_chat_completion_message_event( + transaction, + input_message_list, + completion_id, + span_id, + trace_id, + run_id, + llm_metadata_dict, + output_message_list, + ) + + +def create_chat_completion_message_event( + transaction, + input_message_list, + chat_completion_id, + span_id, + trace_id, + run_id, + llm_metadata_dict, + output_message_list, +): + settings = transaction.settings if transaction.settings is not None else global_settings() + + # Loop through all input messages received from the create request and emit a custom event for each one + for index, message in enumerate(input_message_list): + chat_completion_input_message_dict = { + "id": str(uuid.uuid4()), + "request_id": run_id, + "span_id": span_id, + "trace_id": trace_id, + "completion_id": chat_completion_id, + "sequence": index, + "vendor": "langchain", + "ingest_source": "Python", + "virtual_llm": True, + } + if settings.ai_monitoring.record_content.enabled: + chat_completion_input_message_dict["content"] = message + chat_completion_input_message_dict.update(llm_metadata_dict) + transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_input_message_dict) + + if output_message_list: + # Loop through all output messages received from the LLM response and emit a custom event for each one + for index, message in enumerate(output_message_list): + # Add offset of input_message_length so we don't receive any duplicate index values that match the input message IDs + index += len(input_message_list) + + chat_completion_output_message_dict = { + "id": str(uuid.uuid4()), + "request_id": run_id, + "span_id": span_id, + "trace_id": trace_id, + "completion_id": chat_completion_id, + "sequence": index, + "vendor": "langchain", + "ingest_source": "Python", + "is_response": True, + "virtual_llm": True, + } + if settings.ai_monitoring.record_content.enabled: + chat_completion_output_message_dict["content"] = message + chat_completion_output_message_dict.update(llm_metadata_dict) + transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_output_message_dict) + + +def wrap_on_chain_start(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + completion_id = _get_completion_id(instance) + run_manager = wrapped(*args, **kwargs) + _capture_chain_run_id(transaction, run_manager, completion_id) + return run_manager + + +async def wrap_async_on_chain_start(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + completion_id = _get_completion_id(instance) + run_manager = await wrapped(*args, **kwargs) + _capture_chain_run_id(transaction, run_manager, completion_id) + return run_manager + + +def _get_completion_id(instance): + return (getattr(instance, "metadata", None) or {}).pop("nr_completion_id", None) + + +def _capture_chain_run_id(transaction, run_manager, completion_id): + if completion_id: + if not hasattr(transaction, "_nr_chain_run_ids"): + transaction._nr_chain_run_ids = {} + # Only capture the first run_id. + if completion_id not in transaction._nr_chain_run_ids: + transaction._nr_chain_run_ids[completion_id] = getattr(run_manager, "run_id", "") + + +def instrument_langchain_runnables_chains_base(module): + if hasattr(getattr(module, "RunnableSequence"), "invoke"): + wrap_function_wrapper(module, "RunnableSequence.invoke", wrap_chain_sync_run) + if hasattr(getattr(module, "RunnableSequence"), "ainvoke"): + wrap_function_wrapper(module, "RunnableSequence.ainvoke", wrap_chain_async_run) + + +def instrument_langchain_chains_base(module): + if hasattr(getattr(module, "Chain"), "invoke"): + wrap_function_wrapper(module, "Chain.invoke", wrap_chain_sync_run) + if hasattr(getattr(module, "Chain"), "ainvoke"): + wrap_function_wrapper(module, "Chain.ainvoke", wrap_chain_async_run) + + +def instrument_langchain_vectorstore_similarity_search(module): + def _instrument_class(module, vector_class): + if hasattr(getattr(module, vector_class, ""), "similarity_search"): + wrap_function_wrapper(module, f"{vector_class}.similarity_search", wrap_similarity_search) + if hasattr(getattr(module, vector_class, ""), "asimilarity_search"): + wrap_function_wrapper(module, f"{vector_class}.asimilarity_search", wrap_asimilarity_search) + + vector_classes = VECTORSTORE_CLASSES.get(module.__name__) + if vector_classes is None: + return + if isinstance(vector_classes, list): + for vector_class in vector_classes: + _instrument_class(module, vector_class) + else: + _instrument_class(module, vector_classes) + + +def instrument_langchain_core_tools(module): + if hasattr(getattr(module, "BaseTool"), "run"): + wrap_function_wrapper(module, "BaseTool.run", wrap_tool_sync_run) + if hasattr(getattr(module, "BaseTool"), "arun"): + wrap_function_wrapper(module, "BaseTool.arun", wrap_tool_async_run) + + +def instrument_langchain_callbacks_manager(module): + if hasattr(getattr(module, "CallbackManager"), "on_tool_start"): + wrap_function_wrapper(module, "CallbackManager.on_tool_start", wrap_on_tool_start_sync) + if hasattr(getattr(module, "AsyncCallbackManager"), "on_tool_start"): + wrap_function_wrapper(module, "AsyncCallbackManager.on_tool_start", wrap_on_tool_start_async) + if hasattr(getattr(module, "CallbackManager"), "on_chain_start"): + wrap_function_wrapper(module, "CallbackManager.on_chain_start", wrap_on_chain_start) + if hasattr(getattr(module, "AsyncCallbackManager"), "on_chain_start"): + wrap_function_wrapper(module, "AsyncCallbackManager.on_chain_start", wrap_async_on_chain_start) + + +def instrument_langchain_core_runnables_config(module): + if hasattr(module, "ContextThreadPoolExecutor"): + wrap_function_wrapper(module, "ContextThreadPoolExecutor.submit", wrap_ContextThreadPoolExecutor_submit) diff --git a/newrelic/hooks/mlmodel_openai.py b/newrelic/hooks/mlmodel_openai.py new file mode 100644 index 0000000000..b52fbe27c6 --- /dev/null +++ b/newrelic/hooks/mlmodel_openai.py @@ -0,0 +1,1010 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import logging +import sys +import traceback +import uuid + +import openai + +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.time_trace import get_trace_linking_metadata +from newrelic.api.transaction import current_transaction +from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper +from newrelic.common.package_version_utils import get_package_version +from newrelic.common.signature import bind_args +from newrelic.core.config import global_settings + +OPENAI_VERSION = get_package_version("openai") +OPENAI_VERSION_TUPLE = tuple(map(int, OPENAI_VERSION.split("."))) +OPENAI_V1 = OPENAI_VERSION_TUPLE >= (1,) +EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE = "Exception occurred in openai instrumentation: While reporting an exception in openai, another exception occurred. Report this issue to New Relic Support.\n%s" +RECORD_EVENTS_FAILURE_LOG_MESSAGE = "Exception occurred in OpenAI instrumentation: Failed to record LLM events. Please report this issue to New Relic Support.\n%s" +STREAM_PARSING_FAILURE_LOG_MESSAGE = "Exception occurred in OpenAI instrumentation: Failed to process event stream information. Please report this issue to New Relic Support.\n%s" + +_logger = logging.getLogger(__name__) + + +def wrap_embedding_sync(wrapped, instance, args, kwargs): + transaction = current_transaction() + if ( + not transaction + or kwargs.get("stream", False) + or (kwargs.get("extra_headers") or {}).get("X-Stainless-Raw-Response") == "stream" + ): + return wrapped(*args, **kwargs) + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("OpenAI", OPENAI_VERSION) + transaction._add_agent_attribute("llm", True) + + # Obtain attributes to be stored on embedding events regardless of whether we hit an error + embedding_id = str(uuid.uuid4()) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/embedding/OpenAI") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + response = wrapped(*args, **kwargs) + except Exception as exc: + _record_embedding_error(transaction, embedding_id, linking_metadata, kwargs, ft, exc) + raise + ft.__exit__(None, None, None) + + if not response: + return response + + _record_embedding_success(transaction, embedding_id, linking_metadata, kwargs, ft, response) + return response + + +def wrap_chat_completion_sync(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + # If `.with_streaming_response.` wrapper used, switch to streaming + # For now, we will exit and instrument this later + if (kwargs.get("extra_headers") or {}).get("X-Stainless-Raw-Response") == "stream": + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("OpenAI", OPENAI_VERSION) + transaction._add_agent_attribute("llm", True) + + completion_id = str(uuid.uuid4()) + request_message_list = kwargs.get("messages", []) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/completion/OpenAI") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + return_val = wrapped(*args, **kwargs) + except Exception as exc: + _record_completion_error(transaction, linking_metadata, completion_id, kwargs, ft, exc) + raise + _handle_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val) + return return_val + + +def check_rate_limit_header(response_headers, header_name, is_int): + if not response_headers: + return None + + if header_name in response_headers: + header_value = response_headers.get(header_name) + if is_int: + try: + header_value = int(header_value) + except Exception: + pass + return header_value + else: + return None + + +def create_chat_completion_message_event( + transaction, + input_message_list, + chat_completion_id, + span_id, + trace_id, + response_model, + request_model, + response_id, + request_id, + llm_metadata, + output_message_list, +): + settings = transaction.settings if transaction.settings is not None else global_settings() + + # Loop through all input messages received from the create request and emit a custom event for each one + for index, message in enumerate(input_message_list): + message_content = message.get("content") + + # Response ID was set, append message index to it. + if response_id: + message_id = f"{response_id}-{int(index)}" + # No response IDs, use random UUID + else: + message_id = str(uuid.uuid4()) + + chat_completion_input_message_dict = { + "id": message_id, + "request_id": request_id, + "span_id": span_id, + "trace_id": trace_id, + "token_count": ( + settings.ai_monitoring.llm_token_count_callback(request_model, message_content) + if settings.ai_monitoring.llm_token_count_callback + else None + ), + "role": message.get("role"), + "completion_id": chat_completion_id, + "sequence": index, + "response.model": response_model, + "vendor": "openai", + "ingest_source": "Python", + } + + if settings.ai_monitoring.record_content.enabled: + chat_completion_input_message_dict["content"] = message_content + + chat_completion_input_message_dict.update(llm_metadata) + + transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_input_message_dict) + + if output_message_list: + # Loop through all output messages received from the LLM response and emit a custom event for each one + for index, message in enumerate(output_message_list): + message_content = message.get("content") + + # Add offset of input_message_length so we don't receive any duplicate index values that match the input message IDs + index += len(input_message_list) + + # Response ID was set, append message index to it. + if response_id: + message_id = f"{response_id}-{int(index)}" + # No response IDs, use random UUID + else: + message_id = str(uuid.uuid4()) + + chat_completion_output_message_dict = { + "id": message_id, + "request_id": request_id, + "span_id": span_id, + "trace_id": trace_id, + "token_count": ( + settings.ai_monitoring.llm_token_count_callback(response_model, message_content) + if settings.ai_monitoring.llm_token_count_callback + else None + ), + "role": message.get("role"), + "completion_id": chat_completion_id, + "sequence": index, + "response.model": response_model, + "vendor": "openai", + "ingest_source": "Python", + "is_response": True, + } + + if settings.ai_monitoring.record_content.enabled: + chat_completion_output_message_dict["content"] = message_content + + chat_completion_output_message_dict.update(llm_metadata) + + transaction.record_custom_event("LlmChatCompletionMessage", chat_completion_output_message_dict) + + +async def wrap_embedding_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if ( + not transaction + or kwargs.get("stream", False) + or (kwargs.get("extra_headers") or {}).get("X-Stainless-Raw-Response") == "stream" + ): + return await wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("OpenAI", OPENAI_VERSION) + transaction._add_agent_attribute("llm", True) + + # Obtain attributes to be stored on embedding events regardless of whether we hit an error + embedding_id = str(uuid.uuid4()) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/embedding/OpenAI") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + response = await wrapped(*args, **kwargs) + except Exception as exc: + _record_embedding_error(transaction, embedding_id, linking_metadata, kwargs, ft, exc) + raise + ft.__exit__(None, None, None) + + if not response: + return response + + _record_embedding_success(transaction, embedding_id, linking_metadata, kwargs, ft, response) + return response + + +def _record_embedding_success(transaction, embedding_id, linking_metadata, kwargs, ft, response): + settings = transaction.settings if transaction.settings is not None else global_settings() + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + try: + response_headers = getattr(response, "_nr_response_headers", {}) + input = kwargs.get("input") + + attribute_response = response + # In v1, response objects are pydantic models so this function call converts the + # object back to a dictionary for backwards compatibility. + if OPENAI_V1: + if hasattr(response, "model_dump"): + attribute_response = response.model_dump() + elif hasattr(response, "http_response") and hasattr(response.http_response, "text"): + # This is for the .with_raw_response. wrapper. This is expected + # to change, but the return type for now is the following: + # openai._legacy_response.LegacyAPIResponse + attribute_response = json.loads(response.http_response.text.strip()) + + request_id = response_headers.get("x-request-id") + response_model = attribute_response.get("model") + response_usage = attribute_response.get("usage", {}) or {} + organization = ( + response_headers.get("openai-organization") + if OPENAI_V1 + else getattr(attribute_response, "organization", None) + ) + + full_embedding_response_dict = { + "id": embedding_id, + "span_id": span_id, + "trace_id": trace_id, + "token_count": ( + settings.ai_monitoring.llm_token_count_callback(response_model, input) + if settings.ai_monitoring.llm_token_count_callback + else None + ), + "request.model": kwargs.get("model") or kwargs.get("engine"), + "request_id": request_id, + "duration": ft.duration * 1000, + "response.model": response_model, + "response.organization": organization, + "response.headers.llmVersion": response_headers.get("openai-version"), + "response.headers.ratelimitLimitRequests": check_rate_limit_header( + response_headers, "x-ratelimit-limit-requests", True + ), + "response.headers.ratelimitLimitTokens": check_rate_limit_header( + response_headers, "x-ratelimit-limit-tokens", True + ), + "response.headers.ratelimitResetTokens": check_rate_limit_header( + response_headers, "x-ratelimit-reset-tokens", False + ), + "response.headers.ratelimitResetRequests": check_rate_limit_header( + response_headers, "x-ratelimit-reset-requests", False + ), + "response.headers.ratelimitRemainingTokens": check_rate_limit_header( + response_headers, "x-ratelimit-remaining-tokens", True + ), + "response.headers.ratelimitRemainingRequests": check_rate_limit_header( + response_headers, "x-ratelimit-remaining-requests", True + ), + "vendor": "openai", + "ingest_source": "Python", + } + if settings.ai_monitoring.record_content.enabled: + full_embedding_response_dict["input"] = input + full_embedding_response_dict.update(_get_llm_attributes(transaction)) + transaction.record_custom_event("LlmEmbedding", full_embedding_response_dict) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +def _record_embedding_error(transaction, embedding_id, linking_metadata, kwargs, ft, exc): + settings = transaction.settings if transaction.settings is not None else global_settings() + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + model = kwargs.get("model") or kwargs.get("engine") + input = kwargs.get("input") + + exc_organization = None + notice_error_attributes = {} + try: + if OPENAI_V1: + response = getattr(exc, "response", None) + response_headers = getattr(response, "headers", None) or {} + exc_organization = response_headers.get("openai-organization") + # There appears to be a bug here in openai v1 where despite having code, + # param, etc in the error response, they are not populated on the exception + # object so grab them from the response body object instead. + body = getattr(exc, "body", None) or {} + notice_error_attributes = { + "http.statusCode": getattr(exc, "status_code", None), + "error.message": body.get("message"), + "error.code": body.get("code"), + "error.param": body.get("param"), + "embedding_id": embedding_id, + } + else: + exc_organization = getattr(exc, "organization", None) + notice_error_attributes = { + "http.statusCode": getattr(exc, "http_status", None), + "error.message": getattr(exc, "_message", None), + "error.code": getattr(getattr(exc, "error", None), "code", None), + "error.param": getattr(exc, "param", None), + "embedding_id": embedding_id, + } + except Exception: + _logger.warning(EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + message = notice_error_attributes.pop("error.message", None) + if message: + exc._nr_message = message + ft.notice_error( + attributes=notice_error_attributes, + ) + # Exit the trace now so that the duration is calculated. + ft.__exit__(*sys.exc_info()) + + try: + error_embedding_dict = { + "id": embedding_id, + "span_id": span_id, + "trace_id": trace_id, + "token_count": ( + settings.ai_monitoring.llm_token_count_callback(model, input) + if settings.ai_monitoring.llm_token_count_callback + else None + ), + "request.model": model, + "vendor": "openai", + "ingest_source": "Python", + "response.organization": exc_organization, + "duration": ft.duration * 1000, + "error": True, + } + if settings.ai_monitoring.record_content.enabled: + error_embedding_dict["input"] = input + error_embedding_dict.update(_get_llm_attributes(transaction)) + transaction.record_custom_event("LlmEmbedding", error_embedding_dict) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +async def wrap_chat_completion_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + # If `.with_streaming_response.` wrapper used, switch to streaming + # For now, we will exit and instrument this later + if (kwargs.get("extra_headers") or {}).get("X-Stainless-Raw-Response") == "stream": + return await wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + # Framework metric also used for entity tagging in the UI + transaction.add_ml_model_info("OpenAI", OPENAI_VERSION) + transaction._add_agent_attribute("llm", True) + + completion_id = str(uuid.uuid4()) + + ft = FunctionTrace(name=wrapped.__name__, group="Llm/completion/OpenAI") + ft.__enter__() + linking_metadata = get_trace_linking_metadata() + try: + return_val = await wrapped(*args, **kwargs) + except Exception as exc: + _record_completion_error(transaction, linking_metadata, completion_id, kwargs, ft, exc) + raise + + _handle_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val) + return return_val + + +def _handle_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, return_val): + settings = transaction.settings if transaction.settings is not None else global_settings() + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + request_message_list = kwargs.get("messages") or [] + stream = kwargs.get("stream", False) + # Only if streaming and streaming monitoring is enabled and the response is not empty + # do we not exit the function trace. + if not stream or not settings.ai_monitoring.streaming.enabled or not return_val: + ft.__exit__(None, None, None) + + # If the return value is empty or stream monitoring is disabled exit early. + if not return_val or (stream and not settings.ai_monitoring.streaming.enabled): + return + if stream: + try: + # The function trace will be exited when in the final iteration of the response + # generator. + setattr(return_val, "_nr_ft", ft) + setattr(return_val, "_nr_openai_attrs", getattr(return_val, "_nr_openai_attrs", {})) + return_val._nr_openai_attrs["messages"] = kwargs.get("messages", []) + return_val._nr_openai_attrs["temperature"] = kwargs.get("temperature") + return_val._nr_openai_attrs["max_tokens"] = kwargs.get("max_tokens") + return_val._nr_openai_attrs["model"] = kwargs.get("model") or kwargs.get("engine") + return + except Exception: + _logger.warning(STREAM_PARSING_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + try: + # If response is not a stream generator, record the event data. + # At this point, we have a response so we can grab attributes only available on the response object + response_headers = getattr(return_val, "_nr_response_headers", {}) + response = return_val + + # In v1, response objects are pydantic models so this function call converts the + # object back to a dictionary for backwards compatibility. + if OPENAI_V1: + if hasattr(response, "model_dump"): + response = response.model_dump() + elif hasattr(response, "http_response") and hasattr(response.http_response, "text"): + # This is for the .with_raw_response. wrapper. This is expected + # to change, but the return type for now is the following: + # openai._legacy_response.LegacyAPIResponse + response = json.loads(response.http_response.text.strip()) + + _record_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, response_headers, response) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +def _record_completion_success(transaction, linking_metadata, completion_id, kwargs, ft, response_headers, response): + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + try: + if response: + response_model = response.get("model") + response_id = response.get("id") + response_usage = response.get("usage") or {} + output_message_list = [] + finish_reason = None + choices = response.get("choices") or [] + if choices: + output_message_list = [ + choices[0].get("message") or {"content": choices[0].get("text"), "role": "assistant"} + ] + finish_reason = choices[0].get("finish_reason") + else: + response_model = kwargs.get("response.model") + response_id = kwargs.get("id") + response_usage = {} + output_message_list = [] + finish_reason = None + if "content" in kwargs: + output_message_list = [{"content": kwargs.get("content"), "role": kwargs.get("role")}] + finish_reason = kwargs.get("finish_reason") + request_model = kwargs.get("model") or kwargs.get("engine") + + request_id = response_headers.get("x-request-id") + organization = response_headers.get("openai-organization") or getattr(response, "organization", None) + messages = kwargs.get("messages") or [{"content": kwargs.get("prompt"), "role": "user"}] + input_message_list = list(messages) + full_chat_completion_summary_dict = { + "id": completion_id, + "span_id": span_id, + "trace_id": trace_id, + "request.model": request_model, + "request.temperature": kwargs.get("temperature"), + "request.max_tokens": kwargs.get("max_tokens"), + "vendor": "openai", + "ingest_source": "Python", + "request_id": request_id, + "duration": ft.duration * 1000, + "response.model": response_model, + "response.organization": organization, + "response.choices.finish_reason": finish_reason, + "response.headers.llmVersion": response_headers.get("openai-version"), + "response.headers.ratelimitLimitRequests": check_rate_limit_header( + response_headers, "x-ratelimit-limit-requests", True + ), + "response.headers.ratelimitLimitTokens": check_rate_limit_header( + response_headers, "x-ratelimit-limit-tokens", True + ), + "response.headers.ratelimitResetTokens": check_rate_limit_header( + response_headers, "x-ratelimit-reset-tokens", False + ), + "response.headers.ratelimitResetRequests": check_rate_limit_header( + response_headers, "x-ratelimit-reset-requests", False + ), + "response.headers.ratelimitRemainingTokens": check_rate_limit_header( + response_headers, "x-ratelimit-remaining-tokens", True + ), + "response.headers.ratelimitRemainingRequests": check_rate_limit_header( + response_headers, "x-ratelimit-remaining-requests", True + ), + "response.headers.ratelimitLimitTokensUsageBased": check_rate_limit_header( + response_headers, "x-ratelimit-limit-tokens_usage_based", True + ), + "response.headers.ratelimitResetTokensUsageBased": check_rate_limit_header( + response_headers, "x-ratelimit-reset-tokens_usage_based", False + ), + "response.headers.ratelimitRemainingTokensUsageBased": check_rate_limit_header( + response_headers, "x-ratelimit-remaining-tokens_usage_based", True + ), + "response.number_of_messages": len(input_message_list) + len(output_message_list), + } + llm_metadata = _get_llm_attributes(transaction) + full_chat_completion_summary_dict.update(llm_metadata) + transaction.record_custom_event("LlmChatCompletionSummary", full_chat_completion_summary_dict) + + create_chat_completion_message_event( + transaction, + input_message_list, + completion_id, + span_id, + trace_id, + response_model, + request_model, + response_id, + request_id, + llm_metadata, + output_message_list, + ) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +def _record_completion_error(transaction, linking_metadata, completion_id, kwargs, ft, exc): + span_id = linking_metadata.get("span.id") + trace_id = linking_metadata.get("trace.id") + request_message_list = kwargs.get("messages", None) or [] + notice_error_attributes = {} + try: + if OPENAI_V1: + response = getattr(exc, "response", None) + response_headers = getattr(response, "headers", None) or {} + exc_organization = response_headers.get("openai-organization") + # There appears to be a bug here in openai v1 where despite having code, + # param, etc in the error response, they are not populated on the exception + # object so grab them from the response body object instead. + body = getattr(exc, "body", None) or {} + notice_error_attributes = { + "http.statusCode": getattr(exc, "status_code", None), + "error.message": body.get("message"), + "error.code": body.get("code"), + "error.param": body.get("param"), + "completion_id": completion_id, + } + else: + exc_organization = getattr(exc, "organization", None) + notice_error_attributes = { + "http.statusCode": getattr(exc, "http_status", None), + "error.message": getattr(exc, "_message", None), + "error.code": getattr(getattr(exc, "error", None), "code", None), + "error.param": getattr(exc, "param", None), + "completion_id": completion_id, + } + except Exception: + _logger.warning(EXCEPTION_HANDLING_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + # Override the default message if it is not empty. + message = notice_error_attributes.pop("error.message", None) + if message: + exc._nr_message = message + + ft.notice_error( + attributes=notice_error_attributes, + ) + # Stop the span now so we compute the duration before we create the events. + ft.__exit__(*sys.exc_info()) + + try: + # In a rare case where we are streaming the response and we do get back a request + # and response id, even though an error was encountered, record them. + response_headers = kwargs.get("response_headers") or {} + request_id = response_headers.get("x-request-id") + response_id = kwargs.get("id") + request_model = kwargs.get("model") or kwargs.get("engine") + error_chat_completion_dict = { + "id": completion_id, + "span_id": span_id, + "trace_id": trace_id, + "response.number_of_messages": len(request_message_list), + "request.model": request_model, + "request.temperature": kwargs.get("temperature"), + "request.max_tokens": kwargs.get("max_tokens"), + "vendor": "openai", + "ingest_source": "Python", + "response.organization": exc_organization, + "duration": ft.duration * 1000, + "error": True, + } + llm_metadata = _get_llm_attributes(transaction) + error_chat_completion_dict.update(llm_metadata) + transaction.record_custom_event("LlmChatCompletionSummary", error_chat_completion_dict) + + output_message_list = [] + if "content" in kwargs: + output_message_list = [{"content": kwargs.get("content"), "role": kwargs.get("role")}] + create_chat_completion_message_event( + transaction, + request_message_list, + completion_id, + span_id, + trace_id, + kwargs.get("response.model"), + request_model, + response_id, + request_id, + llm_metadata, + output_message_list, + ) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +def wrap_convert_to_openai_object(wrapped, instance, args, kwargs): + """Obtain reponse headers for v0.""" + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + resp = args[0] + returned_response = wrapped(*args, **kwargs) + + if isinstance(returned_response, openai.openai_object.OpenAIObject) and isinstance( + resp, openai.openai_response.OpenAIResponse + ): + setattr(returned_response, "_nr_response_headers", getattr(resp, "_headers", {})) + + return returned_response + + +def wrap_base_client_process_response_sync(wrapped, instance, args, kwargs): + """Obtain response headers for v1.""" + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + bound_args = bind_args(wrapped, args, kwargs) + nr_response_headers = getattr(bound_args["response"], "headers", None) or {} + + return_val = wrapped(*args, **kwargs) + return_val._nr_response_headers = nr_response_headers + return return_val + + +async def wrap_base_client_process_response_async(wrapped, instance, args, kwargs): + """Obtain response headers for v1.""" + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + bound_args = bind_args(wrapped, args, kwargs) + nr_response_headers = getattr(bound_args["response"], "headers", None) or {} + return_val = await wrapped(*args, **kwargs) + return_val._nr_response_headers = nr_response_headers + return return_val + + +class GeneratorProxy(ObjectProxy): + def __init__(self, wrapped): + super(GeneratorProxy, self).__init__(wrapped) + + def __iter__(self): + return self + + def __next__(self): + transaction = current_transaction() + if not transaction: + return self.__wrapped__.__next__() + + return_val = None + try: + return_val = self.__wrapped__.__next__() + _record_stream_chunk(self, return_val) + except StopIteration as e: + _record_events_on_stop_iteration(self, transaction) + raise + except Exception as exc: + _handle_streaming_completion_error(self, transaction, exc) + raise + return return_val + + def close(self): + return super(GeneratorProxy, self).close() + + +def _record_stream_chunk(self, return_val): + if return_val: + try: + if OPENAI_V1: + if getattr(return_val, "data", "").startswith("[DONE]"): + return + return_val = return_val.json() + self._nr_openai_attrs["response_headers"] = getattr(self, "_nr_response_headers", {}) + else: + self._nr_openai_attrs["response_headers"] = getattr(return_val, "_nr_response_headers", {}) + choices = return_val.get("choices") or [] + self._nr_openai_attrs["response.model"] = return_val.get("model") + self._nr_openai_attrs["id"] = return_val.get("id") + self._nr_openai_attrs["response.organization"] = return_val.get("organization") + if choices: + delta = choices[0].get("delta") or {} + if delta: + self._nr_openai_attrs["content"] = self._nr_openai_attrs.get("content", "") + ( + delta.get("content") or "" + ) + self._nr_openai_attrs["role"] = self._nr_openai_attrs.get("role") or delta.get("role") + self._nr_openai_attrs["finish_reason"] = choices[0].get("finish_reason") + except Exception: + _logger.warning(STREAM_PARSING_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + + +def _record_events_on_stop_iteration(self, transaction): + if hasattr(self, "_nr_ft"): + linking_metadata = get_trace_linking_metadata() + self._nr_ft.__exit__(None, None, None) + try: + openai_attrs = getattr(self, "_nr_openai_attrs", {}) + + # If there are no openai attrs exit early as there's no data to record. + if not openai_attrs: + return + + completion_id = str(uuid.uuid4()) + response_headers = openai_attrs.get("response_headers") or {} + _record_completion_success( + transaction, linking_metadata, completion_id, openai_attrs, self._nr_ft, response_headers, None + ) + except Exception: + _logger.warning(RECORD_EVENTS_FAILURE_LOG_MESSAGE % traceback.format_exception(*sys.exc_info())) + finally: + # Clear cached data as this can be very large. + # Note this is also important for not reporting the events twice. In openai v1 + # there are two loops around the iterator, the second is meant to clear the + # stream since there is a condition where the iterator may exit before all the + # stream contents is read. This results in StopIteration being raised twice + # instead of once at the end of the loop. + if hasattr(self, "_nr_openai_attrs"): + self._nr_openai_attrs.clear() + + +def _handle_streaming_completion_error(self, transaction, exc): + if hasattr(self, "_nr_ft"): + openai_attrs = getattr(self, "_nr_openai_attrs", {}) + + # If there are no openai attrs exit early as there's no data to record. + if not openai_attrs: + self._nr_ft.__exit__(*sys.exc_info()) + return + linking_metadata = get_trace_linking_metadata() + completion_id = str(uuid.uuid4()) + _record_completion_error(transaction, linking_metadata, completion_id, openai_attrs, self._nr_ft, exc) + + +class AsyncGeneratorProxy(ObjectProxy): + def __init__(self, wrapped): + super(AsyncGeneratorProxy, self).__init__(wrapped) + + def __aiter__(self): + self._nr_wrapped_iter = self.__wrapped__.__aiter__() + return self + + async def __anext__(self): + transaction = current_transaction() + if not transaction: + return await self._nr_wrapped_iter.__anext__() + + return_val = None + try: + return_val = await self._nr_wrapped_iter.__anext__() + _record_stream_chunk(self, return_val) + except StopAsyncIteration as e: + _record_events_on_stop_iteration(self, transaction) + raise + except Exception as exc: + _handle_streaming_completion_error(self, transaction, exc) + raise + return return_val + + async def aclose(self): + return await super(AsyncGeneratorProxy, self).aclose() + + +def wrap_stream_iter_events_sync(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled or not settings.ai_monitoring.streaming.enabled: + return wrapped(*args, **kwargs) + + return_val = wrapped(*args, **kwargs) + proxied_return_val = GeneratorProxy(return_val) + set_attrs_on_generator_proxy(proxied_return_val, instance) + return proxied_return_val + + +def wrap_stream_iter_events_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled or not settings.ai_monitoring.streaming.enabled: + return wrapped(*args, **kwargs) + + return_val = wrapped(*args, **kwargs) + proxied_return_val = AsyncGeneratorProxy(return_val) + set_attrs_on_generator_proxy(proxied_return_val, instance) + return proxied_return_val + + +def set_attrs_on_generator_proxy(proxy, instance): + """Pass the nr attributes to the generator proxy.""" + if hasattr(instance, "_nr_ft"): + proxy._nr_ft = instance._nr_ft + if hasattr(instance, "_nr_response_headers"): + proxy._nr_response_headers = instance._nr_response_headers + if hasattr(instance, "_nr_openai_attrs"): + proxy._nr_openai_attrs = instance._nr_openai_attrs + + +def wrap_engine_api_resource_create_sync(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return wrapped(*args, **kwargs) + + stream = is_stream(wrapped, args, kwargs) + return_val = wrapped(*args, **kwargs) + if stream and settings.ai_monitoring.streaming.enabled: + return GeneratorProxy(return_val) + else: + return return_val + + +async def wrap_engine_api_resource_create_async(wrapped, instance, args, kwargs): + transaction = current_transaction() + if not transaction: + return await wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + if not settings.ai_monitoring.enabled: + return await wrapped(*args, **kwargs) + + stream = is_stream(wrapped, args, kwargs) + return_val = await wrapped(*args, **kwargs) + if stream and settings.ai_monitoring.streaming.enabled: + return AsyncGeneratorProxy(return_val) + else: + return return_val + + +def is_stream(wrapped, args, kwargs): + bound_args = bind_args(wrapped, args, kwargs) + return bound_args["params"].get("stream", False) + + +def _get_llm_attributes(transaction): + """Returns llm.* custom attributes off of the transaction.""" + custom_attrs_dict = transaction._custom_params + llm_metadata_dict = {key: value for key, value in custom_attrs_dict.items() if key.startswith("llm.")} + + llm_context_attrs = getattr(transaction, "_llm_context_attrs", None) + if llm_context_attrs: + llm_metadata_dict.update(llm_context_attrs) + + return llm_metadata_dict + + +def instrument_openai_api_resources_embedding(module): + if hasattr(module, "Embedding"): + if hasattr(module.Embedding, "create"): + wrap_function_wrapper(module, "Embedding.create", wrap_embedding_sync) + if hasattr(module.Embedding, "acreate"): + wrap_function_wrapper(module, "Embedding.acreate", wrap_embedding_async) + # This is to mark where we instrument so the SDK knows not to instrument them + # again. + setattr(module.Embedding, "_nr_wrapped", True) + + +def instrument_openai_api_resources_chat_completion(module): + if hasattr(module, "ChatCompletion"): + if hasattr(module.ChatCompletion, "create"): + wrap_function_wrapper(module, "ChatCompletion.create", wrap_chat_completion_sync) + if hasattr(module.ChatCompletion, "acreate"): + wrap_function_wrapper(module, "ChatCompletion.acreate", wrap_chat_completion_async) + # This is to mark where we instrument so the SDK knows not to instrument them + # again. + setattr(module.ChatCompletion, "_nr_wrapped", True) + + +def instrument_openai_resources_chat_completions(module): + if hasattr(module.Completions, "create"): + wrap_function_wrapper(module, "Completions.create", wrap_chat_completion_sync) + if hasattr(module.AsyncCompletions, "create"): + wrap_function_wrapper(module, "AsyncCompletions.create", wrap_chat_completion_async) + + +def instrument_openai_resources_embeddings(module): + if hasattr(module, "Embeddings"): + if hasattr(module.Embeddings, "create"): + wrap_function_wrapper(module, "Embeddings.create", wrap_embedding_sync) + + if hasattr(module, "AsyncEmbeddings"): + if hasattr(module.AsyncEmbeddings, "create"): + wrap_function_wrapper(module, "AsyncEmbeddings.create", wrap_embedding_async) + + +def instrument_openai_util(module): + if hasattr(module, "convert_to_openai_object"): + wrap_function_wrapper(module, "convert_to_openai_object", wrap_convert_to_openai_object) + # This is to mark where we instrument so the SDK knows not to instrument them + # again. + setattr(module.convert_to_openai_object, "_nr_wrapped", True) + + +def instrument_openai_base_client(module): + if hasattr(module, "BaseClient") and hasattr(module.BaseClient, "_process_response"): + wrap_function_wrapper(module, "BaseClient._process_response", wrap_base_client_process_response_sync) + else: + if hasattr(module, "SyncAPIClient") and hasattr(module.SyncAPIClient, "_process_response"): + wrap_function_wrapper(module, "SyncAPIClient._process_response", wrap_base_client_process_response_sync) + if hasattr(module, "AsyncAPIClient") and hasattr(module.AsyncAPIClient, "_process_response"): + wrap_function_wrapper(module, "AsyncAPIClient._process_response", wrap_base_client_process_response_async) + + +def instrument_openai_api_resources_abstract_engine_api_resource(module): + if hasattr(module, "EngineAPIResource"): + if hasattr(module.EngineAPIResource, "create"): + wrap_function_wrapper(module, "EngineAPIResource.create", wrap_engine_api_resource_create_sync) + if hasattr(module.EngineAPIResource, "acreate"): + wrap_function_wrapper(module, "EngineAPIResource.acreate", wrap_engine_api_resource_create_async) + + +def instrument_openai__streaming(module): + if hasattr(module, "Stream"): + if hasattr(module.Stream, "_iter_events"): + wrap_function_wrapper(module, "Stream._iter_events", wrap_stream_iter_events_sync) + if hasattr(module, "AsyncStream"): + if hasattr(module.AsyncStream, "_iter_events"): + wrap_function_wrapper(module, "AsyncStream._iter_events", wrap_stream_iter_events_async) diff --git a/newrelic/hooks/mlmodel_sklearn.py b/newrelic/hooks/mlmodel_sklearn.py new file mode 100644 index 0000000000..5ce3b32d85 --- /dev/null +++ b/newrelic/hooks/mlmodel_sklearn.py @@ -0,0 +1,781 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import sys +import uuid + +from newrelic.api.function_trace import FunctionTrace +from newrelic.api.time_trace import current_trace +from newrelic.api.transaction import current_transaction +from newrelic.common.object_wrapper import ObjectProxy, wrap_function_wrapper +from newrelic.core.config import global_settings + +METHODS_TO_WRAP = ("predict", "fit", "fit_predict", "predict_log_proba", "predict_proba", "transform", "score") +METRIC_SCORERS = ( + "accuracy_score", + "balanced_accuracy_score", + "f1_score", + "precision_score", + "recall_score", + "roc_auc_score", + "r2_score", +) +PY2 = sys.version_info[0] == 2 +_logger = logging.getLogger(__name__) + + +def isnumeric(column): + import numpy as np + + try: + column.astype(np.float64) + return [True] * len(column) + except: + pass + return [False] * len(column) + + +class PredictReturnTypeProxy(ObjectProxy): + def __init__(self, wrapped, model_name, training_step): + super(ObjectProxy, self).__init__(wrapped) + self._nr_model_name = model_name + self._nr_training_step = training_step + + +def _wrap_method_trace(module, class_, method, name=None, group=None): + def _nr_wrapper_method(wrapped, instance, args, kwargs): + transaction = current_transaction() + trace = current_trace() + + if transaction is None: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + + if settings and not settings.machine_learning.enabled: + return wrapped(*args, **kwargs) + + wrapped_attr_name = f"_nr_wrapped_{method}" + + # If the method has already been wrapped do not wrap it again. This happens + # when one class inherits from another and they both implement the method. + if getattr(trace, wrapped_attr_name, False): + return wrapped(*args, **kwargs) + + trace = FunctionTrace(name=name, group=group, source=wrapped) + + try: + # Set the _nr_wrapped attribute to denote that this method is being wrapped. + setattr(trace, wrapped_attr_name, True) + + with trace: + return_val = wrapped(*args, **kwargs) + finally: + # Set the _nr_wrapped attribute to denote that this method is no longer wrapped. + setattr(trace, wrapped_attr_name, False) + + # If this is the fit method, increment the training_step counter. + if method in ("fit", "fit_predict"): + training_step = getattr(instance, "_nr_wrapped_training_step", -1) + setattr(instance, "_nr_wrapped_training_step", training_step + 1) + + # If this is the predict method, wrap the return type in an nr type with + # _nr_wrapped attrs that will attach model info to the data. + if method in ("predict", "fit_predict"): + training_step = getattr(instance, "_nr_wrapped_training_step", "Unknown") + create_prediction_event(transaction, class_, instance, args, kwargs, return_val) + return PredictReturnTypeProxy(return_val, model_name=class_, training_step=training_step) + return return_val + + wrap_function_wrapper(module, f"{class_}.{method}", _nr_wrapper_method) + + +def _calc_prediction_feature_stats(prediction_input, class_, feature_column_names, tags): + import numpy as np + + # Drop any feature columns that are not numeric since we can't compute stats + # on non-numeric columns. + x = np.array(prediction_input) + isnumeric_features = np.apply_along_axis(isnumeric, 0, x) + numeric_features = x[isnumeric_features] + + # Drop any feature column names that are not numeric since we can't compute stats + # on non-numeric columns. + feature_column_names = feature_column_names[isnumeric_features[0]] + + # Only compute stats for features if we have any feature columns left after dropping + # non-numeric columns. + num_cols = len(feature_column_names) + if num_cols > 0: + # Boolean selection of numpy array values reshapes the array to a single + # dimension so we have to reshape it back into a 2D array. + features = np.reshape(numeric_features, (len(numeric_features) // num_cols, num_cols)) + features = features.astype(dtype=np.float64) + + _record_stats(features, feature_column_names, class_, "Feature", tags) + + +def _record_stats(data, column_names, class_, column_type, tags): + import numpy as np + + mean = np.mean(data, axis=0) + percentile25 = np.percentile(data, q=0.25, axis=0) + percentile50 = np.percentile(data, q=0.50, axis=0) + percentile75 = np.percentile(data, q=0.75, axis=0) + standard_deviation = np.std(data, axis=0) + _min = np.min(data, axis=0) + _max = np.max(data, axis=0) + _count = data.shape[0] + + transaction = current_transaction() + + # Currently record_metric only supports a subset of these stats so we have + # to upload them one at a time instead of as a dictionary of stats per + # feature column. + for index, col_name in enumerate(column_names): + metric_name = f"MLModel/Sklearn/Named/{class_}/Predict/{column_type}/{col_name}" + + transaction.record_dimensional_metrics( + [ + (f"{metric_name}/Mean", float(mean[index]), tags), + (f"{metric_name}/Percentile25", float(percentile25[index]), tags), + (f"{metric_name}/Percentile50", float(percentile50[index]), tags), + (f"{metric_name}/Percentile75", float(percentile75[index]), tags), + (f"{metric_name}/StandardDeviation", float(standard_deviation[index]), tags), + (f"{metric_name}/Min", float(_min[index]), tags), + (f"{metric_name}/Max", float(_max[index]), tags), + (f"{metric_name}/Count", _count, tags), + ] + ) + + +def _calc_prediction_label_stats(labels, class_, label_column_names, tags): + import numpy as np + + labels = np.array(labels, dtype=np.float64) + _record_stats(labels, label_column_names, class_, "Label", tags) + + +def _get_label_names(user_defined_label_names, prediction_array): + import numpy as np + + if user_defined_label_names is None: + return np.array(range(prediction_array.shape[1])) + if user_defined_label_names and len(user_defined_label_names) != prediction_array.shape[1]: + _logger.warning( + "The number of label names passed to the ml_model wrapper function is not equal to the number of predictions in the data set. Please supply the correct number of label names." + ) + return np.array(range(prediction_array.shape[1])) + else: + return user_defined_label_names + + +def find_type_category(data_set, row_index, column_index): + # If pandas DataFrame, return type of column. + pd = sys.modules.get("pandas", None) + if pd and isinstance(data_set, pd.DataFrame): + value_type = data_set.iloc[:, column_index].dtype.name + if value_type == "category": + return "categorical" + categorized_value_type = categorize_data_type(value_type) + return categorized_value_type + # If it's not a pandas DataFrame then it is a list or numpy array. + python_type = str(type(data_set[column_index][row_index])) + return categorize_data_type(python_type) + + +def categorize_data_type(python_type): + if "int" in python_type or "float" in python_type or "complex" in python_type: + return "numeric" + if "bool" in python_type: + return "bool" + if "str" in python_type or "unicode" in python_type: + return "str" + else: + return python_type + + +def _get_feature_column_names(user_provided_feature_names, features): + import numpy as np + + num_feature_columns = np.array(features).shape[1] + + # If the user provided feature names are the correct size, return the user provided feature + # names. + if user_provided_feature_names and len(user_provided_feature_names) == num_feature_columns: + return np.array(user_provided_feature_names) + + # If the user provided feature names aren't the correct size, log a warning and do not use the user provided feature names. + if user_provided_feature_names: + _logger.warning( + "The number of feature names passed to the ml_model wrapper function is not equal to the number of columns in the data set. Please supply the correct number of feature names." + ) + + # If the user doesn't provide the feature names or they were provided but the size was incorrect and the features are a pandas data frame, return the column names from the pandas data frame. + pd = sys.modules.get("pandas", None) + if pd and isinstance(features, pd.DataFrame): + return features.columns + + # If the user doesn't provide the feature names or they were provided but the size was incorrect and the features are not a pandas data frame, return the column indexes as the feature names. + return np.array(range(num_feature_columns)) + + +def bind_predict(X, *args, **kwargs): + return X + + +def create_prediction_event(transaction, class_, instance, args, kwargs, return_val): + import numpy as np + + data_set = bind_predict(*args, **kwargs) + model_name = getattr(instance, "_nr_wrapped_name", class_) + model_version = getattr(instance, "_nr_wrapped_version", "0.0.0") + user_provided_feature_names = getattr(instance, "_nr_wrapped_feature_names", None) + label_names = getattr(instance, "_nr_wrapped_label_names", None) + metadata = getattr(instance, "_nr_wrapped_metadata", {}) + settings = transaction.settings if transaction.settings is not None else global_settings() + + prediction_id = uuid.uuid4() + + labels = [] + if return_val is not None: + if not hasattr(return_val, "__iter__"): + labels = np.array([return_val]) + else: + labels = np.array(return_val) + if len(labels.shape) == 1: + labels = np.reshape(labels, (len(labels) // 1, 1)) + + label_names_list = _get_label_names(label_names, labels) + _calc_prediction_label_stats( + labels, + class_, + label_names_list, + tags={ + "prediction_id": prediction_id, + "model_version": model_version, + # The following are used for entity synthesis. + "modelName": model_name, + }, + ) + + final_feature_names = _get_feature_column_names(user_provided_feature_names, data_set) + np_casted_data_set = np.array(data_set) + _calc_prediction_feature_stats( + data_set, + class_, + final_feature_names, + tags={ + "prediction_id": prediction_id, + "model_version": model_version, + # The following are used for entity synthesis. + "modelName": model_name, + }, + ) + features, predictions = np_casted_data_set.shape + for prediction_index, prediction in enumerate(np_casted_data_set): + inference_id = uuid.uuid4() + + event = { + "inference_id": inference_id, + "prediction_id": prediction_id, + "model_version": model_version, + "new_relic_data_schema_version": 2, + # The following are used for entity synthesis. + "modelName": model_name, + } + if metadata and isinstance(metadata, dict): + event.update(metadata) + # Don't include the raw value when inference_event_value is disabled. + if settings and settings.machine_learning and settings.machine_learning.inference_events_value.enabled: + event.update( + { + f"feature.{str(final_feature_names[feature_col_index])}": value + for feature_col_index, value in enumerate(prediction) + } + ) + event.update( + { + f"label.{str(label_names_list[index])}": str(value) + for index, value in enumerate(labels[prediction_index]) + } + ) + transaction.record_ml_event("InferenceData", event) + + +def _nr_instrument_model(module, model_class): + for method_name in METHODS_TO_WRAP: + if hasattr(getattr(module, model_class), method_name): + # Function/MLModel/Sklearn/Named/. + name = f"MLModel/Sklearn/Named/{model_class}.{method_name}" + _wrap_method_trace(module, model_class, method_name, name=name) + + +def _instrument_sklearn_models(module, model_classes): + for model_cls in model_classes: + if hasattr(module, model_cls): + _nr_instrument_model(module, model_cls) + + +def _bind_scorer(y_true, y_pred, *args, **kwargs): + return y_true, y_pred, args, kwargs + + +def wrap_metric_scorer(wrapped, instance, args, kwargs): + transaction = current_transaction() + # If there is no transaction, do not wrap anything. + if not transaction: + return wrapped(*args, **kwargs) + + settings = transaction.settings if transaction.settings is not None else global_settings() + + if settings and not settings.machine_learning.enabled: + return wrapped(*args, **kwargs) + + score = wrapped(*args, **kwargs) + + y_true, y_pred, args, kwargs = _bind_scorer(*args, **kwargs) + model_name = "Unknown" + training_step = "Unknown" + if hasattr(y_pred, "_nr_model_name"): + model_name = y_pred._nr_model_name + if hasattr(y_pred, "_nr_training_step"): + training_step = y_pred._nr_training_step + # Attribute values must be int, float, str, or boolean. If it's not one of these + # types and an iterable add the values as separate attributes. + if not isinstance(score, (str, int, float, bool)): + if hasattr(score, "__iter__"): + for i, s in enumerate(score): + transaction._add_agent_attribute( + f"{model_name}/TrainingStep/{training_step}/{wrapped.__name__}[{i}]", s + ) + else: + transaction._add_agent_attribute(f"{model_name}/TrainingStep/{training_step}/{wrapped.__name__}", score) + return score + + +def instrument_sklearn_tree_models(module): + model_classes = ( + "DecisionTreeClassifier", + "DecisionTreeRegressor", + "ExtraTreeClassifier", + "ExtraTreeRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_bagging_models(module): + model_classes = ( + "BaggingClassifier", + "BaggingRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_forest_models(module): + model_classes = ( + "ExtraTreesClassifier", + "ExtraTreesRegressor", + "RandomForestClassifier", + "RandomForestRegressor", + "RandomTreesEmbedding", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_iforest_models(module): + model_classes = ("IsolationForest",) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_weight_boosting_models(module): + model_classes = ( + "AdaBoostClassifier", + "AdaBoostRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_gradient_boosting_models(module): + model_classes = ( + "GradientBoostingClassifier", + "GradientBoostingRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_voting_models(module): + model_classes = ( + "VotingClassifier", + "VotingRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_ensemble_stacking_models(module): + module_classes = ( + "StackingClassifier", + "StackingRegressor", + ) + _instrument_sklearn_models(module, module_classes) + + +def instrument_sklearn_ensemble_hist_models(module): + model_classes = ( + "HistGradientBoostingClassifier", + "HistGradientBoostingRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_coordinate_descent_models(module): + model_classes = ( + "Lasso", + "LassoCV", + "ElasticNet", + "ElasticNetCV", + "MultiTaskLasso", + "MultiTaskLassoCV", + "MultiTaskElasticNet", + "MultiTaskElasticNetCV", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_compose_models(module): + model_classes = ( + "ColumnTransformer", + "TransformedTargetRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_covariance_shrunk_models(module): + model_classes = ( + "ShrunkCovariance", + "LedoitWolf", + "OAS", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_cross_decomposition_models(module): + model_classes = ( + "PLSRegression", + "PLSSVD", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_covariance_graph_models(module): + model_classes = ( + "GraphicalLasso", + "GraphicalLassoCV", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_discriminant_analysis_models(module): + model_classes = ( + "LinearDiscriminantAnalysis", + "QuadraticDiscriminantAnalysis", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_covariance_models(module): + model_classes = ( + "EmpiricalCovariance", + "MinCovDet", + "EllipticEnvelope", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_gaussian_process_models(module): + model_classes = ( + "GaussianProcessClassifier", + "GaussianProcessRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_dummy_models(module): + model_classes = ( + "DummyClassifier", + "DummyRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_feature_selection_rfe_models(module): + model_classes = ( + "RFE", + "RFECV", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_kernel_ridge_models(module): + model_classes = ("KernelRidge",) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_calibration_models(module): + model_classes = ("CalibratedClassifierCV",) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_cluster_models(module): + model_classes = ( + "AffinityPropagation", + "Birch", + "DBSCAN", + "MeanShift", + "OPTICS", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_least_angle_models(module): + model_classes = ( + "Lars", + "LarsCV", + "LassoLars", + "LassoLarsCV", + "LassoLarsIC", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_feature_selection_models(module): + model_classes = ( + "VarianceThreshold", + "SelectFromModel", + "SequentialFeatureSelector", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_cluster_agglomerative_models(module): + model_classes = ( + "AgglomerativeClustering", + "FeatureAgglomeration", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_GLM_models(module): + model_classes = ( + "PoissonRegressor", + "GammaRegressor", + "TweedieRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_cluster_clustering_models(module): + model_classes = ( + "SpectralBiclustering", + "SpectralCoclustering", + "SpectralClustering", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_stochastic_gradient_models(module): + model_classes = ( + "SGDClassifier", + "SGDRegressor", + "SGDOneClassSVM", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_ridge_models(module): + model_classes = ( + "Ridge", + "RidgeCV", + "RidgeClassifier", + "RidgeClassifierCV", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_logistic_models(module): + model_classes = ( + "LogisticRegression", + "LogisticRegressionCV", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_OMP_models(module): + model_classes = ( + "OrthogonalMatchingPursuit", + "OrthogonalMatchingPursuitCV", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_passive_aggressive_models(module): + model_classes = ( + "PassiveAggressiveClassifier", + "PassiveAggressiveRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_bayes_models(module): + model_classes = ( + "ARDRegression", + "BayesianRidge", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_linear_models(module): + model_classes = ( + "HuberRegressor", + "LinearRegression", + "Perceptron", + "QuantileRegressor", + "TheilSenRegressor", + "RANSACRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_cluster_kmeans_models(module): + model_classes = ( + "BisectingKMeans", + "KMeans", + "MiniBatchKMeans", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_multiclass_models(module): + model_classes = ( + "OneVsRestClassifier", + "OneVsOneClassifier", + "OutputCodeClassifier", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_multioutput_models(module): + model_classes = ( + "MultiOutputEstimator", + "MultiOutputClassifier", + "ClassifierChain", + "RegressorChain", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_naive_bayes_models(module): + model_classes = ( + "GaussianNB", + "MultinomialNB", + "ComplementNB", + "BernoulliNB", + "CategoricalNB", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_model_selection_models(module): + model_classes = ( + "GridSearchCV", + "RandomizedSearchCV", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_mixture_models(module): + model_classes = ( + "GaussianMixture", + "BayesianGaussianMixture", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_neural_network_models(module): + model_classes = ( + "BernoulliRBM", + "MLPClassifier", + "MLPRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_neighbors_KRadius_models(module): + model_classes = ( + "KNeighborsClassifier", + "RadiusNeighborsClassifier", + "KNeighborsTransformer", + "RadiusNeighborsTransformer", + "KNeighborsRegressor", + "RadiusNeighborsRegressor", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_svm_models(module): + model_classes = ( + "LinearSVC", + "LinearSVR", + "SVC", + "NuSVC", + "SVR", + "NuSVR", + "OneClassSVM", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_semi_supervised_models(module): + model_classes = ( + "LabelPropagation", + "LabelSpreading", + "SelfTrainingClassifier", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_pipeline_models(module): + model_classes = ( + "Pipeline", + "FeatureUnion", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_neighbors_models(module): + model_classes = ( + "KernelDensity", + "LocalOutlierFactor", + "NeighborhoodComponentsAnalysis", + "NearestCentroid", + "NearestNeighbors", + ) + _instrument_sklearn_models(module, model_classes) + + +def instrument_sklearn_metrics(module): + for scorer in METRIC_SCORERS: + if hasattr(module, scorer): + wrap_function_wrapper(module, scorer, wrap_metric_scorer) diff --git a/newrelic/hooks/template_genshi.py b/newrelic/hooks/template_genshi.py index abea1e485a..253e06cc95 100644 --- a/newrelic/hooks/template_genshi.py +++ b/newrelic/hooks/template_genshi.py @@ -12,34 +12,41 @@ # See the License for the specific language governing permissions and # limitations under the License. -import types - import newrelic.api.transaction -import newrelic.api.object_wrapper +import newrelic.common.object_wrapper import newrelic.api.function_trace -class stream_wrapper(object): + +class stream_wrapper(): def __init__(self, stream, filepath): self.__stream = stream self.__filepath = filepath + def render(self, *args, **kwargs): return newrelic.api.function_trace.FunctionTraceWrapper( - self.__stream.render, self.__filepath, - 'Template/Render')(*args, **kwargs) + self.__stream.render, self.__filepath, "Template/Render" + )(*args, **kwargs) + def __getattr__(self, name): return getattr(self.__stream, name) + def __iter__(self): return iter(self.__stream) + def __or__(self, function): return self.__stream.__or__(function) + def __str__(self): return self.__stream.__str__() + def __unicode__(self): return self.__stream.__unicode__() + def __html__(self): return self.__stream.__html__() -class wrap_template(object): + +class wrap_template(): def __init__(self, wrapped): if isinstance(wrapped, tuple): (instance, wrapped) = wrapped @@ -57,17 +64,16 @@ def __get__(self, instance, klass): def __call__(self, *args, **kwargs): current_transaction = newrelic.api.transaction.current_transaction() if current_transaction and self.__instance: - return stream_wrapper(self.__wrapped(*args, **kwargs), - self.__instance.filepath) + return stream_wrapper(self.__wrapped(*args, **kwargs), self.__instance.filepath) else: return self.__wrapped(*args, **kwargs) def __getattr__(self, name): return getattr(self.__wrapped, name) + def instrument(module): - if module.__name__ == 'genshi.template.base': + if module.__name__ == "genshi.template.base": - newrelic.api.object_wrapper.wrap_object( - module, 'Template.generate', wrap_template) + newrelic.common.object_wrapper.wrap_object(module, "Template.generate", wrap_template) diff --git a/newrelic/hooks/template_mako.py b/newrelic/hooks/template_mako.py index 2e20da7306..4faa5ba9ea 100644 --- a/newrelic/hooks/template_mako.py +++ b/newrelic/hooks/template_mako.py @@ -13,9 +13,9 @@ # limitations under the License. import newrelic.api.function_trace -import newrelic.api.object_wrapper +import newrelic.common.object_wrapper -class TemplateRenderWrapper(object): +class TemplateRenderWrapper(): def __init__(self, wrapped): self.__wrapped = wrapped @@ -42,7 +42,7 @@ def __call__(self, template, *args, **kwargs): def instrument_mako_runtime(module): - newrelic.api.object_wrapper.wrap_object(module, + newrelic.common.object_wrapper.wrap_object(module, '_render', TemplateRenderWrapper) def instrument_mako_template(module): diff --git a/newrelic/network/addresses.py b/newrelic/network/addresses.py index d26281e70c..156e2c2311 100644 --- a/newrelic/network/addresses.py +++ b/newrelic/network/addresses.py @@ -16,10 +16,7 @@ """ -try: - import urlparse -except ImportError: - import urllib.parse as urlparse +import urllib.parse as urlparse def proxy_details(proxy_scheme, proxy_host, proxy_port, proxy_user, @@ -64,20 +61,20 @@ def proxy_details(proxy_scheme, proxy_host, proxy_port, proxy_user, netloc = proxy_host if proxy_port: - netloc = '%s:%s' % (netloc, proxy_port) + netloc = f'{netloc}:{proxy_port}' if proxy_user: proxy_user = proxy_user or '' proxy_pass = proxy_pass or '' if proxy_pass: - netloc = '%s:%s@%s' % (proxy_user, proxy_pass, netloc) + netloc = f'{proxy_user}:{proxy_pass}@{netloc}' else: - netloc = '%s@%s' % (proxy_user, netloc) + netloc = f'{proxy_user}@{netloc}' if proxy_scheme is None: proxy_scheme = 'http' - proxy = '%s://%s%s' % (proxy_scheme, netloc, path) + proxy = f'{proxy_scheme}://{netloc}{path}' return {'http': proxy, 'https': proxy} diff --git a/newrelic/newrelic.ini b/newrelic/newrelic.ini index d06d8a2926..e8da2a911b 100644 --- a/newrelic/newrelic.ini +++ b/newrelic/newrelic.ini @@ -8,7 +8,7 @@ # # The configuration file follows a structure similar to what you would # find for Microsoft Windows INI files. For further information on the -# configuration file format see the Python ConfigParser documentation at: +# configuration file format see the Python configparser documentation at: # # https://docs.python.org/library/configparser.html # @@ -49,6 +49,32 @@ app_name = Python Application # NEW_RELIC_MONITOR_MODE environment variable. monitor_mode = true +# Indicates if attack detection security module is to be enabled +security.enabled = false + +# To completely disable security set flag to false If the flag is +# set to false, the security module is not loaded. This property +# is read only once at application start. +security.agent.enabled = false + + +# security module provides two modes IAST or RASP +# RASP stands for Runtime Application Self Protection +# while IAST for Interactive Application Security Testing +# Default mode is IAST +security.mode = IAST + + +# web-protect agent endpoint connection URLs +security.validator_service_url = wss://csec.nr-data.net + + +# vulnerabilty detection flags +security.detection.rci.enabled = true +security.detection.rxss.enabled = true +security.detection.deserialization.enabled = true + + # Sets the name of a file to log agent messages to. Whatever you # set this to, you must ensure that the permissions for the # containing directory and the file itself are correct, and @@ -251,5 +277,4 @@ monitor_mode = true [newrelic:production] monitor_mode = true - # --------------------------------------------------------------------------- diff --git a/newrelic/packages/isort/LICENSE b/newrelic/packages/isort/LICENSE new file mode 100644 index 0000000000..b5083a50d8 --- /dev/null +++ b/newrelic/packages/isort/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013 Timothy Edmund Crosley + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/newrelic/packages/isort/__init__.py b/newrelic/packages/isort/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/newrelic/packages/isort/stdlibs/__init__.py b/newrelic/packages/isort/stdlibs/__init__.py new file mode 100644 index 0000000000..3394a7eda8 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/__init__.py @@ -0,0 +1,2 @@ +from . import all as _all +from . import py2, py3, py27, py36, py37, py38, py39, py310, py311 diff --git a/newrelic/packages/isort/stdlibs/all.py b/newrelic/packages/isort/stdlibs/all.py new file mode 100644 index 0000000000..08a365e19e --- /dev/null +++ b/newrelic/packages/isort/stdlibs/all.py @@ -0,0 +1,3 @@ +from . import py2, py3 + +stdlib = py2.stdlib | py3.stdlib diff --git a/newrelic/packages/isort/stdlibs/py2.py b/newrelic/packages/isort/stdlibs/py2.py new file mode 100644 index 0000000000..74af019e49 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py2.py @@ -0,0 +1,3 @@ +from . import py27 + +stdlib = py27.stdlib diff --git a/newrelic/packages/isort/stdlibs/py27.py b/newrelic/packages/isort/stdlibs/py27.py new file mode 100644 index 0000000000..a9bc99d0c7 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py27.py @@ -0,0 +1,301 @@ +""" +File contains the standard library of Python 2.7. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "AL", + "BaseHTTPServer", + "Bastion", + "CGIHTTPServer", + "Carbon", + "ColorPicker", + "ConfigParser", + "Cookie", + "DEVICE", + "DocXMLRPCServer", + "EasyDialogs", + "FL", + "FrameWork", + "GL", + "HTMLParser", + "MacOS", + "MimeWriter", + "MiniAEFrame", + "Nav", + "PixMapWrapper", + "Queue", + "SUNAUDIODEV", + "ScrolledText", + "SimpleHTTPServer", + "SimpleXMLRPCServer", + "SocketServer", + "StringIO", + "Tix", + "Tkinter", + "UserDict", + "UserList", + "UserString", + "W", + "__builtin__", + "_ast", + "_winreg", + "abc", + "aepack", + "aetools", + "aetypes", + "aifc", + "al", + "anydbm", + "applesingle", + "argparse", + "array", + "ast", + "asynchat", + "asyncore", + "atexit", + "audioop", + "autoGIL", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "bsddb", + "buildtools", + "bz2", + "cPickle", + "cProfile", + "cStringIO", + "calendar", + "cd", + "cfmfile", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "commands", + "compileall", + "compiler", + "contextlib", + "cookielib", + "copy", + "copy_reg", + "crypt", + "csv", + "ctypes", + "curses", + "datetime", + "dbhash", + "dbm", + "decimal", + "difflib", + "dircache", + "dis", + "distutils", + "dl", + "doctest", + "dumbdbm", + "dummy_thread", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "errno", + "exceptions", + "fcntl", + "filecmp", + "fileinput", + "findertools", + "fl", + "flp", + "fm", + "fnmatch", + "formatter", + "fpectl", + "fpformat", + "fractions", + "ftplib", + "functools", + "future_builtins", + "gc", + "gdbm", + "gensuitemodule", + "getopt", + "getpass", + "gettext", + "gl", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "hotshot", + "htmlentitydefs", + "htmllib", + "httplib", + "ic", + "icopen", + "imageop", + "imaplib", + "imgfile", + "imghdr", + "imp", + "importlib", + "imputil", + "inspect", + "io", + "itertools", + "jpeg", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "macerrors", + "macostools", + "macpath", + "macresource", + "mailbox", + "mailcap", + "marshal", + "math", + "md5", + "mhlib", + "mimetools", + "mimetypes", + "mimify", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multifile", + "multiprocessing", + "mutex", + "netrc", + "new", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "popen2", + "poplib", + "posix", + "posixfile", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "quopri", + "random", + "re", + "readline", + "resource", + "rexec", + "rfc822", + "rlcompleter", + "robotparser", + "runpy", + "sched", + "select", + "sets", + "sgmllib", + "sha", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statvfs", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "sunaudiodev", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "thread", + "threading", + "time", + "timeit", + "token", + "tokenize", + "trace", + "traceback", + "ttk", + "tty", + "turtle", + "types", + "unicodedata", + "unittest", + "urllib", + "urllib2", + "urlparse", + "user", + "uu", + "uuid", + "videoreader", + "warnings", + "wave", + "weakref", + "webbrowser", + "whichdb", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpclib", + "zipfile", + "zipimport", + "zlib", +} diff --git a/newrelic/packages/isort/stdlibs/py3.py b/newrelic/packages/isort/stdlibs/py3.py new file mode 100644 index 0000000000..9882543853 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py3.py @@ -0,0 +1,3 @@ +from . import py36, py37, py38, py39, py310, py311 + +stdlib = py36.stdlib | py37.stdlib | py38.stdlib | py39.stdlib | py310.stdlib | py311.stdlib diff --git a/newrelic/packages/isort/stdlibs/py310.py b/newrelic/packages/isort/stdlibs/py310.py new file mode 100644 index 0000000000..f45cf50a38 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py310.py @@ -0,0 +1,222 @@ +""" +File contains the standard library of Python 3.10. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_ast", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "graphlib", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "idlelib", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", + "zoneinfo", +} diff --git a/newrelic/packages/isort/stdlibs/py311.py b/newrelic/packages/isort/stdlibs/py311.py new file mode 100644 index 0000000000..6fa42e9952 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py311.py @@ -0,0 +1,222 @@ +""" +File contains the standard library of Python 3.11. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_ast", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "graphlib", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "idlelib", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "tomllib", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", + "zoneinfo", +} diff --git a/newrelic/packages/isort/stdlibs/py36.py b/newrelic/packages/isort/stdlibs/py36.py new file mode 100644 index 0000000000..59ebd24cb4 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py36.py @@ -0,0 +1,224 @@ +""" +File contains the standard library of Python 3.6. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_ast", + "_dummy_thread", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fpectl", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "macpath", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", +} diff --git a/newrelic/packages/isort/stdlibs/py37.py b/newrelic/packages/isort/stdlibs/py37.py new file mode 100644 index 0000000000..e0ad1228a8 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py37.py @@ -0,0 +1,225 @@ +""" +File contains the standard library of Python 3.7. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_ast", + "_dummy_thread", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "macpath", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", +} diff --git a/newrelic/packages/isort/stdlibs/py38.py b/newrelic/packages/isort/stdlibs/py38.py new file mode 100644 index 0000000000..3d89fd26b3 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py38.py @@ -0,0 +1,224 @@ +""" +File contains the standard library of Python 3.8. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_ast", + "_dummy_thread", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "dummy_threading", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", +} diff --git a/newrelic/packages/isort/stdlibs/py39.py b/newrelic/packages/isort/stdlibs/py39.py new file mode 100644 index 0000000000..4b7dd59543 --- /dev/null +++ b/newrelic/packages/isort/stdlibs/py39.py @@ -0,0 +1,224 @@ +""" +File contains the standard library of Python 3.9. + +DO NOT EDIT. If the standard library changes, a new list should be created +using the mkstdlibs.py script. +""" + +stdlib = { + "_ast", + "_thread", + "abc", + "aifc", + "argparse", + "array", + "ast", + "asynchat", + "asyncio", + "asyncore", + "atexit", + "audioop", + "base64", + "bdb", + "binascii", + "binhex", + "bisect", + "builtins", + "bz2", + "cProfile", + "calendar", + "cgi", + "cgitb", + "chunk", + "cmath", + "cmd", + "code", + "codecs", + "codeop", + "collections", + "colorsys", + "compileall", + "concurrent", + "configparser", + "contextlib", + "contextvars", + "copy", + "copyreg", + "crypt", + "csv", + "ctypes", + "curses", + "dataclasses", + "datetime", + "dbm", + "decimal", + "difflib", + "dis", + "distutils", + "doctest", + "email", + "encodings", + "ensurepip", + "enum", + "errno", + "faulthandler", + "fcntl", + "filecmp", + "fileinput", + "fnmatch", + "formatter", + "fractions", + "ftplib", + "functools", + "gc", + "getopt", + "getpass", + "gettext", + "glob", + "graphlib", + "grp", + "gzip", + "hashlib", + "heapq", + "hmac", + "html", + "http", + "imaplib", + "imghdr", + "imp", + "importlib", + "inspect", + "io", + "ipaddress", + "itertools", + "json", + "keyword", + "lib2to3", + "linecache", + "locale", + "logging", + "lzma", + "mailbox", + "mailcap", + "marshal", + "math", + "mimetypes", + "mmap", + "modulefinder", + "msilib", + "msvcrt", + "multiprocessing", + "netrc", + "nis", + "nntplib", + "ntpath", + "numbers", + "operator", + "optparse", + "os", + "ossaudiodev", + "parser", + "pathlib", + "pdb", + "pickle", + "pickletools", + "pipes", + "pkgutil", + "platform", + "plistlib", + "poplib", + "posix", + "posixpath", + "pprint", + "profile", + "pstats", + "pty", + "pwd", + "py_compile", + "pyclbr", + "pydoc", + "queue", + "quopri", + "random", + "re", + "readline", + "reprlib", + "resource", + "rlcompleter", + "runpy", + "sched", + "secrets", + "select", + "selectors", + "shelve", + "shlex", + "shutil", + "signal", + "site", + "smtpd", + "smtplib", + "sndhdr", + "socket", + "socketserver", + "spwd", + "sqlite3", + "sre", + "sre_compile", + "sre_constants", + "sre_parse", + "ssl", + "stat", + "statistics", + "string", + "stringprep", + "struct", + "subprocess", + "sunau", + "symbol", + "symtable", + "sys", + "sysconfig", + "syslog", + "tabnanny", + "tarfile", + "telnetlib", + "tempfile", + "termios", + "test", + "textwrap", + "threading", + "time", + "timeit", + "tkinter", + "token", + "tokenize", + "trace", + "traceback", + "tracemalloc", + "tty", + "turtle", + "turtledemo", + "types", + "typing", + "unicodedata", + "unittest", + "urllib", + "uu", + "uuid", + "venv", + "warnings", + "wave", + "weakref", + "webbrowser", + "winreg", + "winsound", + "wsgiref", + "xdrlib", + "xml", + "xmlrpc", + "zipapp", + "zipfile", + "zipimport", + "zlib", + "zoneinfo", +} diff --git a/newrelic/packages/opentelemetry_proto/LICENSE.txt b/newrelic/packages/opentelemetry_proto/LICENSE.txt new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/newrelic/packages/opentelemetry_proto/LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/newrelic/packages/opentelemetry_proto/__init__.py b/newrelic/packages/opentelemetry_proto/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/newrelic/packages/opentelemetry_proto/common_pb2.py b/newrelic/packages/opentelemetry_proto/common_pb2.py new file mode 100644 index 0000000000..a38431a589 --- /dev/null +++ b/newrelic/packages/opentelemetry_proto/common_pb2.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: opentelemetry/proto/common/v1/common.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n*opentelemetry/proto/common/v1/common.proto\x12\x1dopentelemetry.proto.common.v1\"\x8c\x02\n\x08\x41nyValue\x12\x16\n\x0cstring_value\x18\x01 \x01(\tH\x00\x12\x14\n\nbool_value\x18\x02 \x01(\x08H\x00\x12\x13\n\tint_value\x18\x03 \x01(\x03H\x00\x12\x16\n\x0c\x64ouble_value\x18\x04 \x01(\x01H\x00\x12@\n\x0b\x61rray_value\x18\x05 \x01(\x0b\x32).opentelemetry.proto.common.v1.ArrayValueH\x00\x12\x43\n\x0ckvlist_value\x18\x06 \x01(\x0b\x32+.opentelemetry.proto.common.v1.KeyValueListH\x00\x12\x15\n\x0b\x62ytes_value\x18\x07 \x01(\x0cH\x00\x42\x07\n\x05value\"E\n\nArrayValue\x12\x37\n\x06values\x18\x01 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.AnyValue\"G\n\x0cKeyValueList\x12\x37\n\x06values\x18\x01 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\"O\n\x08KeyValue\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\x36\n\x05value\x18\x02 \x01(\x0b\x32\'.opentelemetry.proto.common.v1.AnyValue\";\n\x16InstrumentationLibrary\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\t:\x02\x18\x01\"5\n\x14InstrumentationScope\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07version\x18\x02 \x01(\tB[\n io.opentelemetry.proto.common.v1B\x0b\x43ommonProtoP\x01Z(go.opentelemetry.io/proto/otlp/common/v1b\x06proto3') + + + +_ANYVALUE = DESCRIPTOR.message_types_by_name['AnyValue'] +_ARRAYVALUE = DESCRIPTOR.message_types_by_name['ArrayValue'] +_KEYVALUELIST = DESCRIPTOR.message_types_by_name['KeyValueList'] +_KEYVALUE = DESCRIPTOR.message_types_by_name['KeyValue'] +_INSTRUMENTATIONLIBRARY = DESCRIPTOR.message_types_by_name['InstrumentationLibrary'] +_INSTRUMENTATIONSCOPE = DESCRIPTOR.message_types_by_name['InstrumentationScope'] +AnyValue = _reflection.GeneratedProtocolMessageType('AnyValue', (_message.Message,), { + 'DESCRIPTOR' : _ANYVALUE, + '__module__' : 'opentelemetry.proto.common.v1.common_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.common.v1.AnyValue) + }) +_sym_db.RegisterMessage(AnyValue) + +ArrayValue = _reflection.GeneratedProtocolMessageType('ArrayValue', (_message.Message,), { + 'DESCRIPTOR' : _ARRAYVALUE, + '__module__' : 'opentelemetry.proto.common.v1.common_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.common.v1.ArrayValue) + }) +_sym_db.RegisterMessage(ArrayValue) + +KeyValueList = _reflection.GeneratedProtocolMessageType('KeyValueList', (_message.Message,), { + 'DESCRIPTOR' : _KEYVALUELIST, + '__module__' : 'opentelemetry.proto.common.v1.common_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.common.v1.KeyValueList) + }) +_sym_db.RegisterMessage(KeyValueList) + +KeyValue = _reflection.GeneratedProtocolMessageType('KeyValue', (_message.Message,), { + 'DESCRIPTOR' : _KEYVALUE, + '__module__' : 'opentelemetry.proto.common.v1.common_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.common.v1.KeyValue) + }) +_sym_db.RegisterMessage(KeyValue) + +InstrumentationLibrary = _reflection.GeneratedProtocolMessageType('InstrumentationLibrary', (_message.Message,), { + 'DESCRIPTOR' : _INSTRUMENTATIONLIBRARY, + '__module__' : 'opentelemetry.proto.common.v1.common_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.common.v1.InstrumentationLibrary) + }) +_sym_db.RegisterMessage(InstrumentationLibrary) + +InstrumentationScope = _reflection.GeneratedProtocolMessageType('InstrumentationScope', (_message.Message,), { + 'DESCRIPTOR' : _INSTRUMENTATIONSCOPE, + '__module__' : 'opentelemetry.proto.common.v1.common_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.common.v1.InstrumentationScope) + }) +_sym_db.RegisterMessage(InstrumentationScope) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n io.opentelemetry.proto.common.v1B\013CommonProtoP\001Z(go.opentelemetry.io/proto/otlp/common/v1' + _INSTRUMENTATIONLIBRARY._options = None + _INSTRUMENTATIONLIBRARY._serialized_options = b'\030\001' + _ANYVALUE._serialized_start=78 + _ANYVALUE._serialized_end=346 + _ARRAYVALUE._serialized_start=348 + _ARRAYVALUE._serialized_end=417 + _KEYVALUELIST._serialized_start=419 + _KEYVALUELIST._serialized_end=490 + _KEYVALUE._serialized_start=492 + _KEYVALUE._serialized_end=571 + _INSTRUMENTATIONLIBRARY._serialized_start=573 + _INSTRUMENTATIONLIBRARY._serialized_end=632 + _INSTRUMENTATIONSCOPE._serialized_start=634 + _INSTRUMENTATIONSCOPE._serialized_end=687 +# @@protoc_insertion_point(module_scope) diff --git a/newrelic/packages/opentelemetry_proto/logs_pb2.py b/newrelic/packages/opentelemetry_proto/logs_pb2.py new file mode 100644 index 0000000000..bb6a55d669 --- /dev/null +++ b/newrelic/packages/opentelemetry_proto/logs_pb2.py @@ -0,0 +1,117 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: opentelemetry/proto/logs/v1/logs.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from . import common_pb2 as opentelemetry_dot_proto_dot_common_dot_v1_dot_common__pb2 +from . import resource_pb2 as opentelemetry_dot_proto_dot_resource_dot_v1_dot_resource__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n&opentelemetry/proto/logs/v1/logs.proto\x12\x1bopentelemetry.proto.logs.v1\x1a*opentelemetry/proto/common/v1/common.proto\x1a.opentelemetry/proto/resource/v1/resource.proto\"L\n\x08LogsData\x12@\n\rresource_logs\x18\x01 \x03(\x0b\x32).opentelemetry.proto.logs.v1.ResourceLogs\"\xff\x01\n\x0cResourceLogs\x12;\n\x08resource\x18\x01 \x01(\x0b\x32).opentelemetry.proto.resource.v1.Resource\x12:\n\nscope_logs\x18\x02 \x03(\x0b\x32&.opentelemetry.proto.logs.v1.ScopeLogs\x12\x62\n\x1cinstrumentation_library_logs\x18\xe8\x07 \x03(\x0b\x32\x37.opentelemetry.proto.logs.v1.InstrumentationLibraryLogsB\x02\x18\x01\x12\x12\n\nschema_url\x18\x03 \x01(\t\"\xa0\x01\n\tScopeLogs\x12\x42\n\x05scope\x18\x01 \x01(\x0b\x32\x33.opentelemetry.proto.common.v1.InstrumentationScope\x12;\n\x0blog_records\x18\x02 \x03(\x0b\x32&.opentelemetry.proto.logs.v1.LogRecord\x12\x12\n\nschema_url\x18\x03 \x01(\t\"\xc9\x01\n\x1aInstrumentationLibraryLogs\x12V\n\x17instrumentation_library\x18\x01 \x01(\x0b\x32\x35.opentelemetry.proto.common.v1.InstrumentationLibrary\x12;\n\x0blog_records\x18\x02 \x03(\x0b\x32&.opentelemetry.proto.logs.v1.LogRecord\x12\x12\n\nschema_url\x18\x03 \x01(\t:\x02\x18\x01\"\xef\x02\n\tLogRecord\x12\x16\n\x0etime_unix_nano\x18\x01 \x01(\x06\x12\x1f\n\x17observed_time_unix_nano\x18\x0b \x01(\x06\x12\x44\n\x0fseverity_number\x18\x02 \x01(\x0e\x32+.opentelemetry.proto.logs.v1.SeverityNumber\x12\x15\n\rseverity_text\x18\x03 \x01(\t\x12\x35\n\x04\x62ody\x18\x05 \x01(\x0b\x32\'.opentelemetry.proto.common.v1.AnyValue\x12;\n\nattributes\x18\x06 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12 \n\x18\x64ropped_attributes_count\x18\x07 \x01(\r\x12\r\n\x05\x66lags\x18\x08 \x01(\x07\x12\x10\n\x08trace_id\x18\t \x01(\x0c\x12\x0f\n\x07span_id\x18\n \x01(\x0cJ\x04\x08\x04\x10\x05*\xc3\x05\n\x0eSeverityNumber\x12\x1f\n\x1bSEVERITY_NUMBER_UNSPECIFIED\x10\x00\x12\x19\n\x15SEVERITY_NUMBER_TRACE\x10\x01\x12\x1a\n\x16SEVERITY_NUMBER_TRACE2\x10\x02\x12\x1a\n\x16SEVERITY_NUMBER_TRACE3\x10\x03\x12\x1a\n\x16SEVERITY_NUMBER_TRACE4\x10\x04\x12\x19\n\x15SEVERITY_NUMBER_DEBUG\x10\x05\x12\x1a\n\x16SEVERITY_NUMBER_DEBUG2\x10\x06\x12\x1a\n\x16SEVERITY_NUMBER_DEBUG3\x10\x07\x12\x1a\n\x16SEVERITY_NUMBER_DEBUG4\x10\x08\x12\x18\n\x14SEVERITY_NUMBER_INFO\x10\t\x12\x19\n\x15SEVERITY_NUMBER_INFO2\x10\n\x12\x19\n\x15SEVERITY_NUMBER_INFO3\x10\x0b\x12\x19\n\x15SEVERITY_NUMBER_INFO4\x10\x0c\x12\x18\n\x14SEVERITY_NUMBER_WARN\x10\r\x12\x19\n\x15SEVERITY_NUMBER_WARN2\x10\x0e\x12\x19\n\x15SEVERITY_NUMBER_WARN3\x10\x0f\x12\x19\n\x15SEVERITY_NUMBER_WARN4\x10\x10\x12\x19\n\x15SEVERITY_NUMBER_ERROR\x10\x11\x12\x1a\n\x16SEVERITY_NUMBER_ERROR2\x10\x12\x12\x1a\n\x16SEVERITY_NUMBER_ERROR3\x10\x13\x12\x1a\n\x16SEVERITY_NUMBER_ERROR4\x10\x14\x12\x19\n\x15SEVERITY_NUMBER_FATAL\x10\x15\x12\x1a\n\x16SEVERITY_NUMBER_FATAL2\x10\x16\x12\x1a\n\x16SEVERITY_NUMBER_FATAL3\x10\x17\x12\x1a\n\x16SEVERITY_NUMBER_FATAL4\x10\x18*X\n\x0eLogRecordFlags\x12\x1f\n\x1bLOG_RECORD_FLAG_UNSPECIFIED\x10\x00\x12%\n LOG_RECORD_FLAG_TRACE_FLAGS_MASK\x10\xff\x01\x42U\n\x1eio.opentelemetry.proto.logs.v1B\tLogsProtoP\x01Z&go.opentelemetry.io/proto/otlp/logs/v1b\x06proto3') + +_SEVERITYNUMBER = DESCRIPTOR.enum_types_by_name['SeverityNumber'] +SeverityNumber = enum_type_wrapper.EnumTypeWrapper(_SEVERITYNUMBER) +_LOGRECORDFLAGS = DESCRIPTOR.enum_types_by_name['LogRecordFlags'] +LogRecordFlags = enum_type_wrapper.EnumTypeWrapper(_LOGRECORDFLAGS) +SEVERITY_NUMBER_UNSPECIFIED = 0 +SEVERITY_NUMBER_TRACE = 1 +SEVERITY_NUMBER_TRACE2 = 2 +SEVERITY_NUMBER_TRACE3 = 3 +SEVERITY_NUMBER_TRACE4 = 4 +SEVERITY_NUMBER_DEBUG = 5 +SEVERITY_NUMBER_DEBUG2 = 6 +SEVERITY_NUMBER_DEBUG3 = 7 +SEVERITY_NUMBER_DEBUG4 = 8 +SEVERITY_NUMBER_INFO = 9 +SEVERITY_NUMBER_INFO2 = 10 +SEVERITY_NUMBER_INFO3 = 11 +SEVERITY_NUMBER_INFO4 = 12 +SEVERITY_NUMBER_WARN = 13 +SEVERITY_NUMBER_WARN2 = 14 +SEVERITY_NUMBER_WARN3 = 15 +SEVERITY_NUMBER_WARN4 = 16 +SEVERITY_NUMBER_ERROR = 17 +SEVERITY_NUMBER_ERROR2 = 18 +SEVERITY_NUMBER_ERROR3 = 19 +SEVERITY_NUMBER_ERROR4 = 20 +SEVERITY_NUMBER_FATAL = 21 +SEVERITY_NUMBER_FATAL2 = 22 +SEVERITY_NUMBER_FATAL3 = 23 +SEVERITY_NUMBER_FATAL4 = 24 +LOG_RECORD_FLAG_UNSPECIFIED = 0 +LOG_RECORD_FLAG_TRACE_FLAGS_MASK = 255 + + +_LOGSDATA = DESCRIPTOR.message_types_by_name['LogsData'] +_RESOURCELOGS = DESCRIPTOR.message_types_by_name['ResourceLogs'] +_SCOPELOGS = DESCRIPTOR.message_types_by_name['ScopeLogs'] +_INSTRUMENTATIONLIBRARYLOGS = DESCRIPTOR.message_types_by_name['InstrumentationLibraryLogs'] +_LOGRECORD = DESCRIPTOR.message_types_by_name['LogRecord'] +LogsData = _reflection.GeneratedProtocolMessageType('LogsData', (_message.Message,), { + 'DESCRIPTOR' : _LOGSDATA, + '__module__' : 'opentelemetry.proto.logs.v1.logs_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.logs.v1.LogsData) + }) +_sym_db.RegisterMessage(LogsData) + +ResourceLogs = _reflection.GeneratedProtocolMessageType('ResourceLogs', (_message.Message,), { + 'DESCRIPTOR' : _RESOURCELOGS, + '__module__' : 'opentelemetry.proto.logs.v1.logs_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.logs.v1.ResourceLogs) + }) +_sym_db.RegisterMessage(ResourceLogs) + +ScopeLogs = _reflection.GeneratedProtocolMessageType('ScopeLogs', (_message.Message,), { + 'DESCRIPTOR' : _SCOPELOGS, + '__module__' : 'opentelemetry.proto.logs.v1.logs_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.logs.v1.ScopeLogs) + }) +_sym_db.RegisterMessage(ScopeLogs) + +InstrumentationLibraryLogs = _reflection.GeneratedProtocolMessageType('InstrumentationLibraryLogs', (_message.Message,), { + 'DESCRIPTOR' : _INSTRUMENTATIONLIBRARYLOGS, + '__module__' : 'opentelemetry.proto.logs.v1.logs_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.logs.v1.InstrumentationLibraryLogs) + }) +_sym_db.RegisterMessage(InstrumentationLibraryLogs) + +LogRecord = _reflection.GeneratedProtocolMessageType('LogRecord', (_message.Message,), { + 'DESCRIPTOR' : _LOGRECORD, + '__module__' : 'opentelemetry.proto.logs.v1.logs_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.logs.v1.LogRecord) + }) +_sym_db.RegisterMessage(LogRecord) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\036io.opentelemetry.proto.logs.v1B\tLogsProtoP\001Z&go.opentelemetry.io/proto/otlp/logs/v1' + _RESOURCELOGS.fields_by_name['instrumentation_library_logs']._options = None + _RESOURCELOGS.fields_by_name['instrumentation_library_logs']._serialized_options = b'\030\001' + _INSTRUMENTATIONLIBRARYLOGS._options = None + _INSTRUMENTATIONLIBRARYLOGS._serialized_options = b'\030\001' + _SEVERITYNUMBER._serialized_start=1237 + _SEVERITYNUMBER._serialized_end=1944 + _LOGRECORDFLAGS._serialized_start=1946 + _LOGRECORDFLAGS._serialized_end=2034 + _LOGSDATA._serialized_start=163 + _LOGSDATA._serialized_end=239 + _RESOURCELOGS._serialized_start=242 + _RESOURCELOGS._serialized_end=497 + _SCOPELOGS._serialized_start=500 + _SCOPELOGS._serialized_end=660 + _INSTRUMENTATIONLIBRARYLOGS._serialized_start=663 + _INSTRUMENTATIONLIBRARYLOGS._serialized_end=864 + _LOGRECORD._serialized_start=867 + _LOGRECORD._serialized_end=1234 +# @@protoc_insertion_point(module_scope) diff --git a/newrelic/packages/opentelemetry_proto/metrics_pb2.py b/newrelic/packages/opentelemetry_proto/metrics_pb2.py new file mode 100644 index 0000000000..dea77c7de9 --- /dev/null +++ b/newrelic/packages/opentelemetry_proto/metrics_pb2.py @@ -0,0 +1,217 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: opentelemetry/proto/metrics/v1/metrics.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import enum_type_wrapper +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from . import common_pb2 as opentelemetry_dot_proto_dot_common_dot_v1_dot_common__pb2 +from . import resource_pb2 as opentelemetry_dot_proto_dot_resource_dot_v1_dot_resource__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n,opentelemetry/proto/metrics/v1/metrics.proto\x12\x1eopentelemetry.proto.metrics.v1\x1a*opentelemetry/proto/common/v1/common.proto\x1a.opentelemetry/proto/resource/v1/resource.proto\"X\n\x0bMetricsData\x12I\n\x10resource_metrics\x18\x01 \x03(\x0b\x32/.opentelemetry.proto.metrics.v1.ResourceMetrics\"\x94\x02\n\x0fResourceMetrics\x12;\n\x08resource\x18\x01 \x01(\x0b\x32).opentelemetry.proto.resource.v1.Resource\x12\x43\n\rscope_metrics\x18\x02 \x03(\x0b\x32,.opentelemetry.proto.metrics.v1.ScopeMetrics\x12k\n\x1finstrumentation_library_metrics\x18\xe8\x07 \x03(\x0b\x32=.opentelemetry.proto.metrics.v1.InstrumentationLibraryMetricsB\x02\x18\x01\x12\x12\n\nschema_url\x18\x03 \x01(\t\"\x9f\x01\n\x0cScopeMetrics\x12\x42\n\x05scope\x18\x01 \x01(\x0b\x32\x33.opentelemetry.proto.common.v1.InstrumentationScope\x12\x37\n\x07metrics\x18\x02 \x03(\x0b\x32&.opentelemetry.proto.metrics.v1.Metric\x12\x12\n\nschema_url\x18\x03 \x01(\t\"\xc8\x01\n\x1dInstrumentationLibraryMetrics\x12V\n\x17instrumentation_library\x18\x01 \x01(\x0b\x32\x35.opentelemetry.proto.common.v1.InstrumentationLibrary\x12\x37\n\x07metrics\x18\x02 \x03(\x0b\x32&.opentelemetry.proto.metrics.v1.Metric\x12\x12\n\nschema_url\x18\x03 \x01(\t:\x02\x18\x01\"\x92\x03\n\x06Metric\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x02 \x01(\t\x12\x0c\n\x04unit\x18\x03 \x01(\t\x12\x36\n\x05gauge\x18\x05 \x01(\x0b\x32%.opentelemetry.proto.metrics.v1.GaugeH\x00\x12\x32\n\x03sum\x18\x07 \x01(\x0b\x32#.opentelemetry.proto.metrics.v1.SumH\x00\x12>\n\thistogram\x18\t \x01(\x0b\x32).opentelemetry.proto.metrics.v1.HistogramH\x00\x12U\n\x15\x65xponential_histogram\x18\n \x01(\x0b\x32\x34.opentelemetry.proto.metrics.v1.ExponentialHistogramH\x00\x12:\n\x07summary\x18\x0b \x01(\x0b\x32\'.opentelemetry.proto.metrics.v1.SummaryH\x00\x42\x06\n\x04\x64\x61taJ\x04\x08\x04\x10\x05J\x04\x08\x06\x10\x07J\x04\x08\x08\x10\t\"M\n\x05Gauge\x12\x44\n\x0b\x64\x61ta_points\x18\x01 \x03(\x0b\x32/.opentelemetry.proto.metrics.v1.NumberDataPoint\"\xba\x01\n\x03Sum\x12\x44\n\x0b\x64\x61ta_points\x18\x01 \x03(\x0b\x32/.opentelemetry.proto.metrics.v1.NumberDataPoint\x12W\n\x17\x61ggregation_temporality\x18\x02 \x01(\x0e\x32\x36.opentelemetry.proto.metrics.v1.AggregationTemporality\x12\x14\n\x0cis_monotonic\x18\x03 \x01(\x08\"\xad\x01\n\tHistogram\x12G\n\x0b\x64\x61ta_points\x18\x01 \x03(\x0b\x32\x32.opentelemetry.proto.metrics.v1.HistogramDataPoint\x12W\n\x17\x61ggregation_temporality\x18\x02 \x01(\x0e\x32\x36.opentelemetry.proto.metrics.v1.AggregationTemporality\"\xc3\x01\n\x14\x45xponentialHistogram\x12R\n\x0b\x64\x61ta_points\x18\x01 \x03(\x0b\x32=.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint\x12W\n\x17\x61ggregation_temporality\x18\x02 \x01(\x0e\x32\x36.opentelemetry.proto.metrics.v1.AggregationTemporality\"P\n\x07Summary\x12\x45\n\x0b\x64\x61ta_points\x18\x01 \x03(\x0b\x32\x30.opentelemetry.proto.metrics.v1.SummaryDataPoint\"\x86\x02\n\x0fNumberDataPoint\x12;\n\nattributes\x18\x07 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12\x1c\n\x14start_time_unix_nano\x18\x02 \x01(\x06\x12\x16\n\x0etime_unix_nano\x18\x03 \x01(\x06\x12\x13\n\tas_double\x18\x04 \x01(\x01H\x00\x12\x10\n\x06\x61s_int\x18\x06 \x01(\x10H\x00\x12;\n\texemplars\x18\x05 \x03(\x0b\x32(.opentelemetry.proto.metrics.v1.Exemplar\x12\r\n\x05\x66lags\x18\x08 \x01(\rB\x07\n\x05valueJ\x04\x08\x01\x10\x02\"\xe6\x02\n\x12HistogramDataPoint\x12;\n\nattributes\x18\t \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12\x1c\n\x14start_time_unix_nano\x18\x02 \x01(\x06\x12\x16\n\x0etime_unix_nano\x18\x03 \x01(\x06\x12\r\n\x05\x63ount\x18\x04 \x01(\x06\x12\x10\n\x03sum\x18\x05 \x01(\x01H\x00\x88\x01\x01\x12\x15\n\rbucket_counts\x18\x06 \x03(\x06\x12\x17\n\x0f\x65xplicit_bounds\x18\x07 \x03(\x01\x12;\n\texemplars\x18\x08 \x03(\x0b\x32(.opentelemetry.proto.metrics.v1.Exemplar\x12\r\n\x05\x66lags\x18\n \x01(\r\x12\x10\n\x03min\x18\x0b \x01(\x01H\x01\x88\x01\x01\x12\x10\n\x03max\x18\x0c \x01(\x01H\x02\x88\x01\x01\x42\x06\n\x04_sumB\x06\n\x04_minB\x06\n\x04_maxJ\x04\x08\x01\x10\x02\"\xb5\x04\n\x1d\x45xponentialHistogramDataPoint\x12;\n\nattributes\x18\x01 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12\x1c\n\x14start_time_unix_nano\x18\x02 \x01(\x06\x12\x16\n\x0etime_unix_nano\x18\x03 \x01(\x06\x12\r\n\x05\x63ount\x18\x04 \x01(\x06\x12\x0b\n\x03sum\x18\x05 \x01(\x01\x12\r\n\x05scale\x18\x06 \x01(\x11\x12\x12\n\nzero_count\x18\x07 \x01(\x06\x12W\n\x08positive\x18\x08 \x01(\x0b\x32\x45.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint.Buckets\x12W\n\x08negative\x18\t \x01(\x0b\x32\x45.opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint.Buckets\x12\r\n\x05\x66lags\x18\n \x01(\r\x12;\n\texemplars\x18\x0b \x03(\x0b\x32(.opentelemetry.proto.metrics.v1.Exemplar\x12\x10\n\x03min\x18\x0c \x01(\x01H\x00\x88\x01\x01\x12\x10\n\x03max\x18\r \x01(\x01H\x01\x88\x01\x01\x1a\x30\n\x07\x42uckets\x12\x0e\n\x06offset\x18\x01 \x01(\x11\x12\x15\n\rbucket_counts\x18\x02 \x03(\x04\x42\x06\n\x04_minB\x06\n\x04_max\"\xc5\x02\n\x10SummaryDataPoint\x12;\n\nattributes\x18\x07 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12\x1c\n\x14start_time_unix_nano\x18\x02 \x01(\x06\x12\x16\n\x0etime_unix_nano\x18\x03 \x01(\x06\x12\r\n\x05\x63ount\x18\x04 \x01(\x06\x12\x0b\n\x03sum\x18\x05 \x01(\x01\x12Y\n\x0fquantile_values\x18\x06 \x03(\x0b\x32@.opentelemetry.proto.metrics.v1.SummaryDataPoint.ValueAtQuantile\x12\r\n\x05\x66lags\x18\x08 \x01(\r\x1a\x32\n\x0fValueAtQuantile\x12\x10\n\x08quantile\x18\x01 \x01(\x01\x12\r\n\x05value\x18\x02 \x01(\x01J\x04\x08\x01\x10\x02\"\xc1\x01\n\x08\x45xemplar\x12\x44\n\x13\x66iltered_attributes\x18\x07 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12\x16\n\x0etime_unix_nano\x18\x02 \x01(\x06\x12\x13\n\tas_double\x18\x03 \x01(\x01H\x00\x12\x10\n\x06\x61s_int\x18\x06 \x01(\x10H\x00\x12\x0f\n\x07span_id\x18\x04 \x01(\x0c\x12\x10\n\x08trace_id\x18\x05 \x01(\x0c\x42\x07\n\x05valueJ\x04\x08\x01\x10\x02*\x8c\x01\n\x16\x41ggregationTemporality\x12\'\n#AGGREGATION_TEMPORALITY_UNSPECIFIED\x10\x00\x12!\n\x1d\x41GGREGATION_TEMPORALITY_DELTA\x10\x01\x12&\n\"AGGREGATION_TEMPORALITY_CUMULATIVE\x10\x02*;\n\x0e\x44\x61taPointFlags\x12\r\n\tFLAG_NONE\x10\x00\x12\x1a\n\x16\x46LAG_NO_RECORDED_VALUE\x10\x01\x42^\n!io.opentelemetry.proto.metrics.v1B\x0cMetricsProtoP\x01Z)go.opentelemetry.io/proto/otlp/metrics/v1b\x06proto3') + +_AGGREGATIONTEMPORALITY = DESCRIPTOR.enum_types_by_name['AggregationTemporality'] +AggregationTemporality = enum_type_wrapper.EnumTypeWrapper(_AGGREGATIONTEMPORALITY) +_DATAPOINTFLAGS = DESCRIPTOR.enum_types_by_name['DataPointFlags'] +DataPointFlags = enum_type_wrapper.EnumTypeWrapper(_DATAPOINTFLAGS) +AGGREGATION_TEMPORALITY_UNSPECIFIED = 0 +AGGREGATION_TEMPORALITY_DELTA = 1 +AGGREGATION_TEMPORALITY_CUMULATIVE = 2 +FLAG_NONE = 0 +FLAG_NO_RECORDED_VALUE = 1 + + +_METRICSDATA = DESCRIPTOR.message_types_by_name['MetricsData'] +_RESOURCEMETRICS = DESCRIPTOR.message_types_by_name['ResourceMetrics'] +_SCOPEMETRICS = DESCRIPTOR.message_types_by_name['ScopeMetrics'] +_INSTRUMENTATIONLIBRARYMETRICS = DESCRIPTOR.message_types_by_name['InstrumentationLibraryMetrics'] +_METRIC = DESCRIPTOR.message_types_by_name['Metric'] +_GAUGE = DESCRIPTOR.message_types_by_name['Gauge'] +_SUM = DESCRIPTOR.message_types_by_name['Sum'] +_HISTOGRAM = DESCRIPTOR.message_types_by_name['Histogram'] +_EXPONENTIALHISTOGRAM = DESCRIPTOR.message_types_by_name['ExponentialHistogram'] +_SUMMARY = DESCRIPTOR.message_types_by_name['Summary'] +_NUMBERDATAPOINT = DESCRIPTOR.message_types_by_name['NumberDataPoint'] +_HISTOGRAMDATAPOINT = DESCRIPTOR.message_types_by_name['HistogramDataPoint'] +_EXPONENTIALHISTOGRAMDATAPOINT = DESCRIPTOR.message_types_by_name['ExponentialHistogramDataPoint'] +_EXPONENTIALHISTOGRAMDATAPOINT_BUCKETS = _EXPONENTIALHISTOGRAMDATAPOINT.nested_types_by_name['Buckets'] +_SUMMARYDATAPOINT = DESCRIPTOR.message_types_by_name['SummaryDataPoint'] +_SUMMARYDATAPOINT_VALUEATQUANTILE = _SUMMARYDATAPOINT.nested_types_by_name['ValueAtQuantile'] +_EXEMPLAR = DESCRIPTOR.message_types_by_name['Exemplar'] +MetricsData = _reflection.GeneratedProtocolMessageType('MetricsData', (_message.Message,), { + 'DESCRIPTOR' : _METRICSDATA, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.MetricsData) + }) +_sym_db.RegisterMessage(MetricsData) + +ResourceMetrics = _reflection.GeneratedProtocolMessageType('ResourceMetrics', (_message.Message,), { + 'DESCRIPTOR' : _RESOURCEMETRICS, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.ResourceMetrics) + }) +_sym_db.RegisterMessage(ResourceMetrics) + +ScopeMetrics = _reflection.GeneratedProtocolMessageType('ScopeMetrics', (_message.Message,), { + 'DESCRIPTOR' : _SCOPEMETRICS, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.ScopeMetrics) + }) +_sym_db.RegisterMessage(ScopeMetrics) + +InstrumentationLibraryMetrics = _reflection.GeneratedProtocolMessageType('InstrumentationLibraryMetrics', (_message.Message,), { + 'DESCRIPTOR' : _INSTRUMENTATIONLIBRARYMETRICS, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.InstrumentationLibraryMetrics) + }) +_sym_db.RegisterMessage(InstrumentationLibraryMetrics) + +Metric = _reflection.GeneratedProtocolMessageType('Metric', (_message.Message,), { + 'DESCRIPTOR' : _METRIC, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.Metric) + }) +_sym_db.RegisterMessage(Metric) + +Gauge = _reflection.GeneratedProtocolMessageType('Gauge', (_message.Message,), { + 'DESCRIPTOR' : _GAUGE, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.Gauge) + }) +_sym_db.RegisterMessage(Gauge) + +Sum = _reflection.GeneratedProtocolMessageType('Sum', (_message.Message,), { + 'DESCRIPTOR' : _SUM, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.Sum) + }) +_sym_db.RegisterMessage(Sum) + +Histogram = _reflection.GeneratedProtocolMessageType('Histogram', (_message.Message,), { + 'DESCRIPTOR' : _HISTOGRAM, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.Histogram) + }) +_sym_db.RegisterMessage(Histogram) + +ExponentialHistogram = _reflection.GeneratedProtocolMessageType('ExponentialHistogram', (_message.Message,), { + 'DESCRIPTOR' : _EXPONENTIALHISTOGRAM, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.ExponentialHistogram) + }) +_sym_db.RegisterMessage(ExponentialHistogram) + +Summary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), { + 'DESCRIPTOR' : _SUMMARY, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.Summary) + }) +_sym_db.RegisterMessage(Summary) + +NumberDataPoint = _reflection.GeneratedProtocolMessageType('NumberDataPoint', (_message.Message,), { + 'DESCRIPTOR' : _NUMBERDATAPOINT, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.NumberDataPoint) + }) +_sym_db.RegisterMessage(NumberDataPoint) + +HistogramDataPoint = _reflection.GeneratedProtocolMessageType('HistogramDataPoint', (_message.Message,), { + 'DESCRIPTOR' : _HISTOGRAMDATAPOINT, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.HistogramDataPoint) + }) +_sym_db.RegisterMessage(HistogramDataPoint) + +ExponentialHistogramDataPoint = _reflection.GeneratedProtocolMessageType('ExponentialHistogramDataPoint', (_message.Message,), { + + 'Buckets' : _reflection.GeneratedProtocolMessageType('Buckets', (_message.Message,), { + 'DESCRIPTOR' : _EXPONENTIALHISTOGRAMDATAPOINT_BUCKETS, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint.Buckets) + }) + , + 'DESCRIPTOR' : _EXPONENTIALHISTOGRAMDATAPOINT, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.ExponentialHistogramDataPoint) + }) +_sym_db.RegisterMessage(ExponentialHistogramDataPoint) +_sym_db.RegisterMessage(ExponentialHistogramDataPoint.Buckets) + +SummaryDataPoint = _reflection.GeneratedProtocolMessageType('SummaryDataPoint', (_message.Message,), { + + 'ValueAtQuantile' : _reflection.GeneratedProtocolMessageType('ValueAtQuantile', (_message.Message,), { + 'DESCRIPTOR' : _SUMMARYDATAPOINT_VALUEATQUANTILE, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.SummaryDataPoint.ValueAtQuantile) + }) + , + 'DESCRIPTOR' : _SUMMARYDATAPOINT, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.SummaryDataPoint) + }) +_sym_db.RegisterMessage(SummaryDataPoint) +_sym_db.RegisterMessage(SummaryDataPoint.ValueAtQuantile) + +Exemplar = _reflection.GeneratedProtocolMessageType('Exemplar', (_message.Message,), { + 'DESCRIPTOR' : _EXEMPLAR, + '__module__' : 'opentelemetry.proto.metrics.v1.metrics_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.metrics.v1.Exemplar) + }) +_sym_db.RegisterMessage(Exemplar) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n!io.opentelemetry.proto.metrics.v1B\014MetricsProtoP\001Z)go.opentelemetry.io/proto/otlp/metrics/v1' + _RESOURCEMETRICS.fields_by_name['instrumentation_library_metrics']._options = None + _RESOURCEMETRICS.fields_by_name['instrumentation_library_metrics']._serialized_options = b'\030\001' + _INSTRUMENTATIONLIBRARYMETRICS._options = None + _INSTRUMENTATIONLIBRARYMETRICS._serialized_options = b'\030\001' + _AGGREGATIONTEMPORALITY._serialized_start=3754 + _AGGREGATIONTEMPORALITY._serialized_end=3894 + _DATAPOINTFLAGS._serialized_start=3896 + _DATAPOINTFLAGS._serialized_end=3955 + _METRICSDATA._serialized_start=172 + _METRICSDATA._serialized_end=260 + _RESOURCEMETRICS._serialized_start=263 + _RESOURCEMETRICS._serialized_end=539 + _SCOPEMETRICS._serialized_start=542 + _SCOPEMETRICS._serialized_end=701 + _INSTRUMENTATIONLIBRARYMETRICS._serialized_start=704 + _INSTRUMENTATIONLIBRARYMETRICS._serialized_end=904 + _METRIC._serialized_start=907 + _METRIC._serialized_end=1309 + _GAUGE._serialized_start=1311 + _GAUGE._serialized_end=1388 + _SUM._serialized_start=1391 + _SUM._serialized_end=1577 + _HISTOGRAM._serialized_start=1580 + _HISTOGRAM._serialized_end=1753 + _EXPONENTIALHISTOGRAM._serialized_start=1756 + _EXPONENTIALHISTOGRAM._serialized_end=1951 + _SUMMARY._serialized_start=1953 + _SUMMARY._serialized_end=2033 + _NUMBERDATAPOINT._serialized_start=2036 + _NUMBERDATAPOINT._serialized_end=2298 + _HISTOGRAMDATAPOINT._serialized_start=2301 + _HISTOGRAMDATAPOINT._serialized_end=2659 + _EXPONENTIALHISTOGRAMDATAPOINT._serialized_start=2662 + _EXPONENTIALHISTOGRAMDATAPOINT._serialized_end=3227 + _EXPONENTIALHISTOGRAMDATAPOINT_BUCKETS._serialized_start=3163 + _EXPONENTIALHISTOGRAMDATAPOINT_BUCKETS._serialized_end=3211 + _SUMMARYDATAPOINT._serialized_start=3230 + _SUMMARYDATAPOINT._serialized_end=3555 + _SUMMARYDATAPOINT_VALUEATQUANTILE._serialized_start=3499 + _SUMMARYDATAPOINT_VALUEATQUANTILE._serialized_end=3549 + _EXEMPLAR._serialized_start=3558 + _EXEMPLAR._serialized_end=3751 +# @@protoc_insertion_point(module_scope) \ No newline at end of file diff --git a/newrelic/packages/opentelemetry_proto/resource_pb2.py b/newrelic/packages/opentelemetry_proto/resource_pb2.py new file mode 100644 index 0000000000..8cc64e3524 --- /dev/null +++ b/newrelic/packages/opentelemetry_proto/resource_pb2.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: opentelemetry/proto/resource/v1/resource.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from . import common_pb2 as opentelemetry_dot_proto_dot_common_dot_v1_dot_common__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.opentelemetry/proto/resource/v1/resource.proto\x12\x1fopentelemetry.proto.resource.v1\x1a*opentelemetry/proto/common/v1/common.proto\"i\n\x08Resource\x12;\n\nattributes\x18\x01 \x03(\x0b\x32\'.opentelemetry.proto.common.v1.KeyValue\x12 \n\x18\x64ropped_attributes_count\x18\x02 \x01(\rBa\n\"io.opentelemetry.proto.resource.v1B\rResourceProtoP\x01Z*go.opentelemetry.io/proto/otlp/resource/v1b\x06proto3') + + + +_RESOURCE = DESCRIPTOR.message_types_by_name['Resource'] +Resource = _reflection.GeneratedProtocolMessageType('Resource', (_message.Message,), { + 'DESCRIPTOR' : _RESOURCE, + '__module__' : 'opentelemetry.proto.resource.v1.resource_pb2' + # @@protoc_insertion_point(class_scope:opentelemetry.proto.resource.v1.Resource) + }) +_sym_db.RegisterMessage(Resource) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'\n\"io.opentelemetry.proto.resource.v1B\rResourceProtoP\001Z*go.opentelemetry.io/proto/otlp/resource/v1' + _RESOURCE._serialized_start=127 + _RESOURCE._serialized_end=232 +# @@protoc_insertion_point(module_scope) diff --git a/newrelic/packages/requirements.txt b/newrelic/packages/requirements.txt new file mode 100644 index 0000000000..38a1391611 --- /dev/null +++ b/newrelic/packages/requirements.txt @@ -0,0 +1,8 @@ +# Note: The following dependencies are vendored within the Python +# Agent code in newrelic/packages/. +# This file is used by dependabot to keep track of and recommend updates +# to the New Relic Python Agent's dependencies in newrelic/packages/. +opentelemetry_proto==1.0.0 +urllib3==1.26.19 +wrapt==1.16.0 +asgiref==3.6.0 # We only vendor asgiref.compatibility.py diff --git a/newrelic/packages/six.py b/newrelic/packages/six.py deleted file mode 100644 index 4e15675d8b..0000000000 --- a/newrelic/packages/six.py +++ /dev/null @@ -1,998 +0,0 @@ -# Copyright (c) 2010-2020 Benjamin Peterson -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all -# copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. - -"""Utilities for writing code that runs on Python 2 and 3""" - -from __future__ import absolute_import - -import functools -import itertools -import operator -import sys -import types - -__author__ = "Benjamin Peterson " -__version__ = "1.16.0" - - -# Useful for very coarse version differentiation. -PY2 = sys.version_info[0] == 2 -PY3 = sys.version_info[0] == 3 -PY34 = sys.version_info[0:2] >= (3, 4) - -if PY3: - string_types = str, - integer_types = int, - class_types = type, - text_type = str - binary_type = bytes - - MAXSIZE = sys.maxsize -else: - string_types = basestring, - integer_types = (int, long) - class_types = (type, types.ClassType) - text_type = unicode - binary_type = str - - if sys.platform.startswith("java"): - # Jython always uses 32 bits. - MAXSIZE = int((1 << 31) - 1) - else: - # It's possible to have sizeof(long) != sizeof(Py_ssize_t). - class X(object): - - def __len__(self): - return 1 << 31 - try: - len(X()) - except OverflowError: - # 32-bit - MAXSIZE = int((1 << 31) - 1) - else: - # 64-bit - MAXSIZE = int((1 << 63) - 1) - del X - -if PY34: - from importlib.util import spec_from_loader -else: - spec_from_loader = None - - -def _add_doc(func, doc): - """Add documentation to a function.""" - func.__doc__ = doc - - -def _import_module(name): - """Import module, returning the module after the last dot.""" - __import__(name) - return sys.modules[name] - - -class _LazyDescr(object): - - def __init__(self, name): - self.name = name - - def __get__(self, obj, tp): - result = self._resolve() - setattr(obj, self.name, result) # Invokes __set__. - try: - # This is a bit ugly, but it avoids running this again by - # removing this descriptor. - delattr(obj.__class__, self.name) - except AttributeError: - pass - return result - - -class MovedModule(_LazyDescr): - - def __init__(self, name, old, new=None): - super(MovedModule, self).__init__(name) - if PY3: - if new is None: - new = name - self.mod = new - else: - self.mod = old - - def _resolve(self): - return _import_module(self.mod) - - def __getattr__(self, attr): - _module = self._resolve() - value = getattr(_module, attr) - setattr(self, attr, value) - return value - - -class _LazyModule(types.ModuleType): - - def __init__(self, name): - super(_LazyModule, self).__init__(name) - self.__doc__ = self.__class__.__doc__ - - def __dir__(self): - attrs = ["__doc__", "__name__"] - attrs += [attr.name for attr in self._moved_attributes] - return attrs - - # Subclasses should override this - _moved_attributes = [] - - -class MovedAttribute(_LazyDescr): - - def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): - super(MovedAttribute, self).__init__(name) - if PY3: - if new_mod is None: - new_mod = name - self.mod = new_mod - if new_attr is None: - if old_attr is None: - new_attr = name - else: - new_attr = old_attr - self.attr = new_attr - else: - self.mod = old_mod - if old_attr is None: - old_attr = name - self.attr = old_attr - - def _resolve(self): - module = _import_module(self.mod) - return getattr(module, self.attr) - - -class _SixMetaPathImporter(object): - - """ - A meta path importer to import six.moves and its submodules. - - This class implements a PEP302 finder and loader. It should be compatible - with Python 2.5 and all existing versions of Python3 - """ - - def __init__(self, six_module_name): - self.name = six_module_name - self.known_modules = {} - - def _add_module(self, mod, *fullnames): - for fullname in fullnames: - self.known_modules[self.name + "." + fullname] = mod - - def _get_module(self, fullname): - return self.known_modules[self.name + "." + fullname] - - def find_module(self, fullname, path=None): - if fullname in self.known_modules: - return self - return None - - def find_spec(self, fullname, path, target=None): - if fullname in self.known_modules: - return spec_from_loader(fullname, self) - return None - - def __get_module(self, fullname): - try: - return self.known_modules[fullname] - except KeyError: - raise ImportError("This loader does not know module " + fullname) - - def load_module(self, fullname): - try: - # in case of a reload - return sys.modules[fullname] - except KeyError: - pass - mod = self.__get_module(fullname) - if isinstance(mod, MovedModule): - mod = mod._resolve() - else: - mod.__loader__ = self - sys.modules[fullname] = mod - return mod - - def is_package(self, fullname): - """ - Return true, if the named module is a package. - - We need this method to get correct spec objects with - Python 3.4 (see PEP451) - """ - return hasattr(self.__get_module(fullname), "__path__") - - def get_code(self, fullname): - """Return None - - Required, if is_package is implemented""" - self.__get_module(fullname) # eventually raises ImportError - return None - get_source = get_code # same as get_code - - def create_module(self, spec): - return self.load_module(spec.name) - - def exec_module(self, module): - pass - -_importer = _SixMetaPathImporter(__name__) - - -class _MovedItems(_LazyModule): - - """Lazy loading of moved objects""" - __path__ = [] # mark as package - - -_moved_attributes = [ - MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), - MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), - MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), - MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), - MovedAttribute("intern", "__builtin__", "sys"), - MovedAttribute("map", "itertools", "builtins", "imap", "map"), - MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), - MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), - MovedAttribute("getoutput", "commands", "subprocess"), - MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), - MovedAttribute("reduce", "__builtin__", "functools"), - MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), - MovedAttribute("StringIO", "StringIO", "io"), - MovedAttribute("UserDict", "UserDict", "collections"), - MovedAttribute("UserList", "UserList", "collections"), - MovedAttribute("UserString", "UserString", "collections"), - MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), - MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), - MovedModule("builtins", "__builtin__"), - MovedModule("configparser", "ConfigParser"), - MovedModule("collections_abc", "collections", "collections.abc" if sys.version_info >= (3, 3) else "collections"), - MovedModule("copyreg", "copy_reg"), - MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), - MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), - MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread" if sys.version_info < (3, 9) else "_thread"), - MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), - MovedModule("http_cookies", "Cookie", "http.cookies"), - MovedModule("html_entities", "htmlentitydefs", "html.entities"), - MovedModule("html_parser", "HTMLParser", "html.parser"), - MovedModule("http_client", "httplib", "http.client"), - MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), - MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), - MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), - MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), - MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), - MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), - MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), - MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), - MovedModule("cPickle", "cPickle", "pickle"), - MovedModule("queue", "Queue"), - MovedModule("reprlib", "repr"), - MovedModule("socketserver", "SocketServer"), - MovedModule("_thread", "thread", "_thread"), - MovedModule("tkinter", "Tkinter"), - MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"), - MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"), - MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"), - MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"), - MovedModule("tkinter_tix", "Tix", "tkinter.tix"), - MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), - MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), - MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), - MovedModule("tkinter_colorchooser", "tkColorChooser", - "tkinter.colorchooser"), - MovedModule("tkinter_commondialog", "tkCommonDialog", - "tkinter.commondialog"), - MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), - MovedModule("tkinter_font", "tkFont", "tkinter.font"), - MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), - MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", - "tkinter.simpledialog"), - MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), - MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), - MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), - MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"), - MovedModule("xmlrpc_client", "xmlrpclib", "xmlrpc.client"), - MovedModule("xmlrpc_server", "SimpleXMLRPCServer", "xmlrpc.server"), -] -# Add windows specific modules. -if sys.platform == "win32": - _moved_attributes += [ - MovedModule("winreg", "_winreg"), - ] - -for attr in _moved_attributes: - setattr(_MovedItems, attr.name, attr) - if isinstance(attr, MovedModule): - _importer._add_module(attr, "moves." + attr.name) -del attr - -_MovedItems._moved_attributes = _moved_attributes - -moves = _MovedItems(__name__ + ".moves") -_importer._add_module(moves, "moves") - - -class Module_six_moves_urllib_parse(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_parse""" - - -_urllib_parse_moved_attributes = [ - MovedAttribute("ParseResult", "urlparse", "urllib.parse"), - MovedAttribute("SplitResult", "urlparse", "urllib.parse"), - MovedAttribute("parse_qs", "urlparse", "urllib.parse"), - MovedAttribute("parse_qsl", "urlparse", "urllib.parse"), - MovedAttribute("urldefrag", "urlparse", "urllib.parse"), - MovedAttribute("urljoin", "urlparse", "urllib.parse"), - MovedAttribute("urlparse", "urlparse", "urllib.parse"), - MovedAttribute("urlsplit", "urlparse", "urllib.parse"), - MovedAttribute("urlunparse", "urlparse", "urllib.parse"), - MovedAttribute("urlunsplit", "urlparse", "urllib.parse"), - MovedAttribute("quote", "urllib", "urllib.parse"), - MovedAttribute("quote_plus", "urllib", "urllib.parse"), - MovedAttribute("unquote", "urllib", "urllib.parse"), - MovedAttribute("unquote_plus", "urllib", "urllib.parse"), - MovedAttribute("unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes"), - MovedAttribute("urlencode", "urllib", "urllib.parse"), - MovedAttribute("splitquery", "urllib", "urllib.parse"), - MovedAttribute("splittag", "urllib", "urllib.parse"), - MovedAttribute("splituser", "urllib", "urllib.parse"), - MovedAttribute("splitvalue", "urllib", "urllib.parse"), - MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), - MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), - MovedAttribute("uses_params", "urlparse", "urllib.parse"), - MovedAttribute("uses_query", "urlparse", "urllib.parse"), - MovedAttribute("uses_relative", "urlparse", "urllib.parse"), -] -for attr in _urllib_parse_moved_attributes: - setattr(Module_six_moves_urllib_parse, attr.name, attr) -del attr - -Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes - -_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), - "moves.urllib_parse", "moves.urllib.parse") - - -class Module_six_moves_urllib_error(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_error""" - - -_urllib_error_moved_attributes = [ - MovedAttribute("URLError", "urllib2", "urllib.error"), - MovedAttribute("HTTPError", "urllib2", "urllib.error"), - MovedAttribute("ContentTooShortError", "urllib", "urllib.error"), -] -for attr in _urllib_error_moved_attributes: - setattr(Module_six_moves_urllib_error, attr.name, attr) -del attr - -Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes - -_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), - "moves.urllib_error", "moves.urllib.error") - - -class Module_six_moves_urllib_request(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_request""" - - -_urllib_request_moved_attributes = [ - MovedAttribute("urlopen", "urllib2", "urllib.request"), - MovedAttribute("install_opener", "urllib2", "urllib.request"), - MovedAttribute("build_opener", "urllib2", "urllib.request"), - MovedAttribute("pathname2url", "urllib", "urllib.request"), - MovedAttribute("url2pathname", "urllib", "urllib.request"), - MovedAttribute("getproxies", "urllib", "urllib.request"), - MovedAttribute("Request", "urllib2", "urllib.request"), - MovedAttribute("OpenerDirector", "urllib2", "urllib.request"), - MovedAttribute("HTTPDefaultErrorHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPRedirectHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPCookieProcessor", "urllib2", "urllib.request"), - MovedAttribute("ProxyHandler", "urllib2", "urllib.request"), - MovedAttribute("BaseHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPPasswordMgr", "urllib2", "urllib.request"), - MovedAttribute("HTTPPasswordMgrWithDefaultRealm", "urllib2", "urllib.request"), - MovedAttribute("AbstractBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("ProxyBasicAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("AbstractDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("ProxyDigestAuthHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPSHandler", "urllib2", "urllib.request"), - MovedAttribute("FileHandler", "urllib2", "urllib.request"), - MovedAttribute("FTPHandler", "urllib2", "urllib.request"), - MovedAttribute("CacheFTPHandler", "urllib2", "urllib.request"), - MovedAttribute("UnknownHandler", "urllib2", "urllib.request"), - MovedAttribute("HTTPErrorProcessor", "urllib2", "urllib.request"), - MovedAttribute("urlretrieve", "urllib", "urllib.request"), - MovedAttribute("urlcleanup", "urllib", "urllib.request"), - MovedAttribute("URLopener", "urllib", "urllib.request"), - MovedAttribute("FancyURLopener", "urllib", "urllib.request"), - MovedAttribute("proxy_bypass", "urllib", "urllib.request"), - MovedAttribute("parse_http_list", "urllib2", "urllib.request"), - MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), -] -for attr in _urllib_request_moved_attributes: - setattr(Module_six_moves_urllib_request, attr.name, attr) -del attr - -Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes - -_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), - "moves.urllib_request", "moves.urllib.request") - - -class Module_six_moves_urllib_response(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_response""" - - -_urllib_response_moved_attributes = [ - MovedAttribute("addbase", "urllib", "urllib.response"), - MovedAttribute("addclosehook", "urllib", "urllib.response"), - MovedAttribute("addinfo", "urllib", "urllib.response"), - MovedAttribute("addinfourl", "urllib", "urllib.response"), -] -for attr in _urllib_response_moved_attributes: - setattr(Module_six_moves_urllib_response, attr.name, attr) -del attr - -Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes - -_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), - "moves.urllib_response", "moves.urllib.response") - - -class Module_six_moves_urllib_robotparser(_LazyModule): - - """Lazy loading of moved objects in six.moves.urllib_robotparser""" - - -_urllib_robotparser_moved_attributes = [ - MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), -] -for attr in _urllib_robotparser_moved_attributes: - setattr(Module_six_moves_urllib_robotparser, attr.name, attr) -del attr - -Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes - -_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), - "moves.urllib_robotparser", "moves.urllib.robotparser") - - -class Module_six_moves_urllib(types.ModuleType): - - """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" - __path__ = [] # mark as package - parse = _importer._get_module("moves.urllib_parse") - error = _importer._get_module("moves.urllib_error") - request = _importer._get_module("moves.urllib_request") - response = _importer._get_module("moves.urllib_response") - robotparser = _importer._get_module("moves.urllib_robotparser") - - def __dir__(self): - return ['parse', 'error', 'request', 'response', 'robotparser'] - -_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), - "moves.urllib") - - -def add_move(move): - """Add an item to six.moves.""" - setattr(_MovedItems, move.name, move) - - -def remove_move(name): - """Remove item from six.moves.""" - try: - delattr(_MovedItems, name) - except AttributeError: - try: - del moves.__dict__[name] - except KeyError: - raise AttributeError("no such move, %r" % (name,)) - - -if PY3: - _meth_func = "__func__" - _meth_self = "__self__" - - _func_closure = "__closure__" - _func_code = "__code__" - _func_defaults = "__defaults__" - _func_globals = "__globals__" -else: - _meth_func = "im_func" - _meth_self = "im_self" - - _func_closure = "func_closure" - _func_code = "func_code" - _func_defaults = "func_defaults" - _func_globals = "func_globals" - - -try: - advance_iterator = next -except NameError: - def advance_iterator(it): - return it.next() -next = advance_iterator - - -try: - callable = callable -except NameError: - def callable(obj): - return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) - - -if PY3: - def get_unbound_function(unbound): - return unbound - - create_bound_method = types.MethodType - - def create_unbound_method(func, cls): - return func - - Iterator = object -else: - def get_unbound_function(unbound): - return unbound.im_func - - def create_bound_method(func, obj): - return types.MethodType(func, obj, obj.__class__) - - def create_unbound_method(func, cls): - return types.MethodType(func, None, cls) - - class Iterator(object): - - def next(self): - return type(self).__next__(self) - - callable = callable -_add_doc(get_unbound_function, - """Get the function out of a possibly unbound function""") - - -get_method_function = operator.attrgetter(_meth_func) -get_method_self = operator.attrgetter(_meth_self) -get_function_closure = operator.attrgetter(_func_closure) -get_function_code = operator.attrgetter(_func_code) -get_function_defaults = operator.attrgetter(_func_defaults) -get_function_globals = operator.attrgetter(_func_globals) - - -if PY3: - def iterkeys(d, **kw): - return iter(d.keys(**kw)) - - def itervalues(d, **kw): - return iter(d.values(**kw)) - - def iteritems(d, **kw): - return iter(d.items(**kw)) - - def iterlists(d, **kw): - return iter(d.lists(**kw)) - - viewkeys = operator.methodcaller("keys") - - viewvalues = operator.methodcaller("values") - - viewitems = operator.methodcaller("items") -else: - def iterkeys(d, **kw): - return d.iterkeys(**kw) - - def itervalues(d, **kw): - return d.itervalues(**kw) - - def iteritems(d, **kw): - return d.iteritems(**kw) - - def iterlists(d, **kw): - return d.iterlists(**kw) - - viewkeys = operator.methodcaller("viewkeys") - - viewvalues = operator.methodcaller("viewvalues") - - viewitems = operator.methodcaller("viewitems") - -_add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") -_add_doc(itervalues, "Return an iterator over the values of a dictionary.") -_add_doc(iteritems, - "Return an iterator over the (key, value) pairs of a dictionary.") -_add_doc(iterlists, - "Return an iterator over the (key, [values]) pairs of a dictionary.") - - -if PY3: - def b(s): - return s.encode("latin-1") - - def u(s): - return s - unichr = chr - import struct - int2byte = struct.Struct(">B").pack - del struct - byte2int = operator.itemgetter(0) - indexbytes = operator.getitem - iterbytes = iter - import io - StringIO = io.StringIO - BytesIO = io.BytesIO - del io - _assertCountEqual = "assertCountEqual" - if sys.version_info[1] <= 1: - _assertRaisesRegex = "assertRaisesRegexp" - _assertRegex = "assertRegexpMatches" - _assertNotRegex = "assertNotRegexpMatches" - else: - _assertRaisesRegex = "assertRaisesRegex" - _assertRegex = "assertRegex" - _assertNotRegex = "assertNotRegex" -else: - def b(s): - return s - # Workaround for standalone backslash - - def u(s): - return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") - unichr = unichr - int2byte = chr - - def byte2int(bs): - return ord(bs[0]) - - def indexbytes(buf, i): - return ord(buf[i]) - iterbytes = functools.partial(itertools.imap, ord) - import StringIO - StringIO = BytesIO = StringIO.StringIO - _assertCountEqual = "assertItemsEqual" - _assertRaisesRegex = "assertRaisesRegexp" - _assertRegex = "assertRegexpMatches" - _assertNotRegex = "assertNotRegexpMatches" -_add_doc(b, """Byte literal""") -_add_doc(u, """Text literal""") - - -def assertCountEqual(self, *args, **kwargs): - return getattr(self, _assertCountEqual)(*args, **kwargs) - - -def assertRaisesRegex(self, *args, **kwargs): - return getattr(self, _assertRaisesRegex)(*args, **kwargs) - - -def assertRegex(self, *args, **kwargs): - return getattr(self, _assertRegex)(*args, **kwargs) - - -def assertNotRegex(self, *args, **kwargs): - return getattr(self, _assertNotRegex)(*args, **kwargs) - - -if PY3: - exec_ = getattr(moves.builtins, "exec") - - def reraise(tp, value, tb=None): - try: - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value - finally: - value = None - tb = None - -else: - def exec_(_code_, _globs_=None, _locs_=None): - """Execute code in a namespace.""" - if _globs_ is None: - frame = sys._getframe(1) - _globs_ = frame.f_globals - if _locs_ is None: - _locs_ = frame.f_locals - del frame - elif _locs_ is None: - _locs_ = _globs_ - exec("""exec _code_ in _globs_, _locs_""") - - exec_("""def reraise(tp, value, tb=None): - try: - raise tp, value, tb - finally: - tb = None -""") - - -if sys.version_info[:2] > (3,): - exec_("""def raise_from(value, from_value): - try: - raise value from from_value - finally: - value = None -""") -else: - def raise_from(value, from_value): - raise value - - -print_ = getattr(moves.builtins, "print", None) -if print_ is None: - def print_(*args, **kwargs): - """The new-style print function for Python 2.4 and 2.5.""" - fp = kwargs.pop("file", sys.stdout) - if fp is None: - return - - def write(data): - if not isinstance(data, basestring): - data = str(data) - # If the file has an encoding, encode unicode with it. - if (isinstance(fp, file) and - isinstance(data, unicode) and - fp.encoding is not None): - errors = getattr(fp, "errors", None) - if errors is None: - errors = "strict" - data = data.encode(fp.encoding, errors) - fp.write(data) - want_unicode = False - sep = kwargs.pop("sep", None) - if sep is not None: - if isinstance(sep, unicode): - want_unicode = True - elif not isinstance(sep, str): - raise TypeError("sep must be None or a string") - end = kwargs.pop("end", None) - if end is not None: - if isinstance(end, unicode): - want_unicode = True - elif not isinstance(end, str): - raise TypeError("end must be None or a string") - if kwargs: - raise TypeError("invalid keyword arguments to print()") - if not want_unicode: - for arg in args: - if isinstance(arg, unicode): - want_unicode = True - break - if want_unicode: - newline = unicode("\n") - space = unicode(" ") - else: - newline = "\n" - space = " " - if sep is None: - sep = space - if end is None: - end = newline - for i, arg in enumerate(args): - if i: - write(sep) - write(arg) - write(end) -if sys.version_info[:2] < (3, 3): - _print = print_ - - def print_(*args, **kwargs): - fp = kwargs.get("file", sys.stdout) - flush = kwargs.pop("flush", False) - _print(*args, **kwargs) - if flush and fp is not None: - fp.flush() - -_add_doc(reraise, """Reraise an exception.""") - -if sys.version_info[0:2] < (3, 4): - # This does exactly the same what the :func:`py3:functools.update_wrapper` - # function does on Python versions after 3.2. It sets the ``__wrapped__`` - # attribute on ``wrapper`` object and it doesn't raise an error if any of - # the attributes mentioned in ``assigned`` and ``updated`` are missing on - # ``wrapped`` object. - def _update_wrapper(wrapper, wrapped, - assigned=functools.WRAPPER_ASSIGNMENTS, - updated=functools.WRAPPER_UPDATES): - for attr in assigned: - try: - value = getattr(wrapped, attr) - except AttributeError: - continue - else: - setattr(wrapper, attr, value) - for attr in updated: - getattr(wrapper, attr).update(getattr(wrapped, attr, {})) - wrapper.__wrapped__ = wrapped - return wrapper - _update_wrapper.__doc__ = functools.update_wrapper.__doc__ - - def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, - updated=functools.WRAPPER_UPDATES): - return functools.partial(_update_wrapper, wrapped=wrapped, - assigned=assigned, updated=updated) - wraps.__doc__ = functools.wraps.__doc__ - -else: - wraps = functools.wraps - - -def with_metaclass(meta, *bases): - """Create a base class with a metaclass.""" - # This requires a bit of explanation: the basic idea is to make a dummy - # metaclass for one level of class instantiation that replaces itself with - # the actual metaclass. - class metaclass(type): - - def __new__(cls, name, this_bases, d): - if sys.version_info[:2] >= (3, 7): - # This version introduced PEP 560 that requires a bit - # of extra care (we mimic what is done by __build_class__). - resolved_bases = types.resolve_bases(bases) - if resolved_bases is not bases: - d['__orig_bases__'] = bases - else: - resolved_bases = bases - return meta(name, resolved_bases, d) - - @classmethod - def __prepare__(cls, name, this_bases): - return meta.__prepare__(name, bases) - return type.__new__(metaclass, 'temporary_class', (), {}) - - -def add_metaclass(metaclass): - """Class decorator for creating a class with a metaclass.""" - def wrapper(cls): - orig_vars = cls.__dict__.copy() - slots = orig_vars.get('__slots__') - if slots is not None: - if isinstance(slots, str): - slots = [slots] - for slots_var in slots: - orig_vars.pop(slots_var) - orig_vars.pop('__dict__', None) - orig_vars.pop('__weakref__', None) - if hasattr(cls, '__qualname__'): - orig_vars['__qualname__'] = cls.__qualname__ - return metaclass(cls.__name__, cls.__bases__, orig_vars) - return wrapper - - -def ensure_binary(s, encoding='utf-8', errors='strict'): - """Coerce **s** to six.binary_type. - - For Python 2: - - `unicode` -> encoded to `str` - - `str` -> `str` - - For Python 3: - - `str` -> encoded to `bytes` - - `bytes` -> `bytes` - """ - if isinstance(s, binary_type): - return s - if isinstance(s, text_type): - return s.encode(encoding, errors) - raise TypeError("not expecting type '%s'" % type(s)) - - -def ensure_str(s, encoding='utf-8', errors='strict'): - """Coerce *s* to `str`. - - For Python 2: - - `unicode` -> encoded to `str` - - `str` -> `str` - - For Python 3: - - `str` -> `str` - - `bytes` -> decoded to `str` - """ - # Optimization: Fast return for the common case. - if type(s) is str: - return s - if PY2 and isinstance(s, text_type): - return s.encode(encoding, errors) - elif PY3 and isinstance(s, binary_type): - return s.decode(encoding, errors) - elif not isinstance(s, (text_type, binary_type)): - raise TypeError("not expecting type '%s'" % type(s)) - return s - - -def ensure_text(s, encoding='utf-8', errors='strict'): - """Coerce *s* to six.text_type. - - For Python 2: - - `unicode` -> `unicode` - - `str` -> `unicode` - - For Python 3: - - `str` -> `str` - - `bytes` -> decoded to `str` - """ - if isinstance(s, binary_type): - return s.decode(encoding, errors) - elif isinstance(s, text_type): - return s - else: - raise TypeError("not expecting type '%s'" % type(s)) - - -def python_2_unicode_compatible(klass): - """ - A class decorator that defines __unicode__ and __str__ methods under Python 2. - Under Python 3 it does nothing. - - To support Python 2 and 3 with a single code base, define a __str__ method - returning text and apply this decorator to the class. - """ - if PY2: - if '__str__' not in klass.__dict__: - raise ValueError("@python_2_unicode_compatible cannot be applied " - "to %s because it doesn't define __str__()." % - klass.__name__) - klass.__unicode__ = klass.__str__ - klass.__str__ = lambda self: self.__unicode__().encode('utf-8') - return klass - - -# Complete the moves implementation. -# This code is at the end of this module to speed up module loading. -# Turn this module into a package. -__path__ = [] # required for PEP 302 and PEP 451 -__package__ = __name__ # see PEP 366 @ReservedAssignment -if globals().get("__spec__") is not None: - __spec__.submodule_search_locations = [] # PEP 451 @UndefinedVariable -# Remove other six meta path importers, since they cause problems. This can -# happen if six is removed from sys.modules and then reloaded. (Setuptools does -# this for some reason.) -if sys.meta_path: - for i, importer in enumerate(sys.meta_path): - # Here's some real nastiness: Another "instance" of the six module might - # be floating around. Therefore, we can't use isinstance() to check for - # the six meta path importer, since the other six instance will have - # inserted an importer with different class. - if (type(importer).__name__ == "_SixMetaPathImporter" and - importer.name == __name__): - del sys.meta_path[i] - break - del i, importer -# Finally, add the importer to the meta path import hook. -sys.meta_path.append(_importer) diff --git a/newrelic/packages/urllib3/__init__.py b/newrelic/packages/urllib3/__init__.py index c8c7ce691e..c6fa38212f 100644 --- a/newrelic/packages/urllib3/__init__.py +++ b/newrelic/packages/urllib3/__init__.py @@ -19,6 +19,22 @@ from .util.timeout import Timeout from .util.url import get_host +# === NOTE TO REPACKAGERS AND VENDORS === +# Please delete this block, this logic is only +# for urllib3 being distributed via PyPI. +# See: https://github.com/urllib3/urllib3/issues/2680 +try: + import urllib3_secure_extra # type: ignore # noqa: F401 +except ImportError: + pass +else: + warnings.warn( + "'urllib3[secure]' extra is deprecated and will be removed " + "in a future release of urllib3 2.x. Read more in this issue: " + "https://github.com/urllib3/urllib3/issues/2680", + category=DeprecationWarning, + stacklevel=2, + ) __author__ = "Andrey Petrov (andrey.petrov@shazow.net)" __license__ = "MIT" diff --git a/newrelic/packages/urllib3/_collections.py b/newrelic/packages/urllib3/_collections.py index da9857e986..bceb8451f0 100644 --- a/newrelic/packages/urllib3/_collections.py +++ b/newrelic/packages/urllib3/_collections.py @@ -268,6 +268,24 @@ def getlist(self, key, default=__marker): else: return vals[1:] + def _prepare_for_method_change(self): + """ + Remove content-specific header fields before changing the request + method to GET or HEAD according to RFC 9110, Section 15.4. + """ + content_specific_headers = [ + "Content-Encoding", + "Content-Language", + "Content-Location", + "Content-Type", + "Content-Length", + "Digest", + "Last-Modified", + ] + for header in content_specific_headers: + self.discard(header) + return self + # Backwards compatibility for httplib getheaders = getlist getallmatchingheaders = getlist diff --git a/newrelic/packages/urllib3/_version.py b/newrelic/packages/urllib3/_version.py index e12dd0e785..c40db86d0a 100644 --- a/newrelic/packages/urllib3/_version.py +++ b/newrelic/packages/urllib3/_version.py @@ -1,2 +1,2 @@ # This file is protected via CODEOWNERS -__version__ = "1.26.15" +__version__ = "1.26.19" diff --git a/newrelic/packages/urllib3/connection.py b/newrelic/packages/urllib3/connection.py index 54b96b1915..de35b63d67 100644 --- a/newrelic/packages/urllib3/connection.py +++ b/newrelic/packages/urllib3/connection.py @@ -68,7 +68,7 @@ class BrokenPipeError(Exception): # When it comes time to update this value as a part of regular maintenance # (ie test_recent_date is failing) update it to ~6 months before the current date. -RECENT_DATE = datetime.date(2022, 1, 1) +RECENT_DATE = datetime.date(2024, 1, 1) _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") @@ -437,7 +437,7 @@ def connect(self): and self.ssl_version is None and hasattr(self.sock, "version") and self.sock.version() in {"TLSv1", "TLSv1.1"} - ): + ): # Defensive: warnings.warn( "Negotiating TLSv1/TLSv1.1 by default is deprecated " "and will be disabled in urllib3 v2.0.0. Connecting to " diff --git a/newrelic/packages/urllib3/connectionpool.py b/newrelic/packages/urllib3/connectionpool.py index c23d736b18..402bf670da 100644 --- a/newrelic/packages/urllib3/connectionpool.py +++ b/newrelic/packages/urllib3/connectionpool.py @@ -9,6 +9,7 @@ from socket import error as SocketError from socket import timeout as SocketTimeout +from ._collections import HTTPHeaderDict from .connection import ( BaseSSLError, BrokenPipeError, @@ -50,6 +51,13 @@ from .util.url import _normalize_host as normalize_host from .util.url import get_host, parse_url +try: # Platform-specific: Python 3 + import weakref + + weakref_finalize = weakref.finalize +except AttributeError: # Platform-specific: Python 2 + from .packages.backports.weakref_finalize import weakref_finalize + xrange = six.moves.xrange log = logging.getLogger(__name__) @@ -220,6 +228,16 @@ def __init__( self.conn_kw["proxy"] = self.proxy self.conn_kw["proxy_config"] = self.proxy_config + # Do not pass 'self' as callback to 'finalize'. + # Then the 'finalize' would keep an endless living (leak) to self. + # By just passing a reference to the pool allows the garbage collector + # to free self if nobody else has a reference to it. + pool = self.pool + + # Close all the HTTPConnections in the pool before the + # HTTPConnectionPool object is garbage collected. + weakref_finalize(self, _close_pool_connections, pool) + def _new_conn(self): """ Return a fresh :class:`HTTPConnection`. @@ -489,14 +507,8 @@ def close(self): # Disable access to the pool old_pool, self.pool = self.pool, None - try: - while True: - conn = old_pool.get(block=False) - if conn: - conn.close() - - except queue.Empty: - pass # Done. + # Close all the HTTPConnections in the pool. + _close_pool_connections(old_pool) def is_same_host(self, url): """ @@ -756,7 +768,9 @@ def _is_ssl_error_message_from_http_proxy(ssl_error): # so we try to cover our bases here! message = " ".join(re.split("[^a-z]", str(ssl_error).lower())) return ( - "wrong version number" in message or "unknown protocol" in message + "wrong version number" in message + or "unknown protocol" in message + or "record layer failure" in message ) # Try to detect a common user error with proxies which is to @@ -832,7 +846,11 @@ def _is_ssl_error_message_from_http_proxy(ssl_error): redirect_location = redirect and response.get_redirect_location() if redirect_location: if response.status == 303: + # Change the method according to RFC 9110, Section 15.4.4. method = "GET" + # And lose the body not to transfer anything sensitive. + body = None + headers = HTTPHeaderDict(headers)._prepare_for_method_change() try: retries = retries.increment(method, url, response=response, _pool=self) @@ -1108,3 +1126,14 @@ def _normalize_host(host, scheme): if host.startswith("[") and host.endswith("]"): host = host[1:-1] return host + + +def _close_pool_connections(pool): + """Drains a queue of connections and closes each one.""" + try: + while True: + conn = pool.get(block=False) + if conn: + conn.close() + except queue.Empty: + pass # Done. diff --git a/newrelic/packages/urllib3/contrib/securetransport.py b/newrelic/packages/urllib3/contrib/securetransport.py index 6c46a3b9f0..e311c0c899 100644 --- a/newrelic/packages/urllib3/contrib/securetransport.py +++ b/newrelic/packages/urllib3/contrib/securetransport.py @@ -64,9 +64,8 @@ import threading import weakref -import six - from .. import util +from ..packages import six from ..util.ssl_ import PROTOCOL_TLS_CLIENT from ._securetransport.bindings import CoreFoundation, Security, SecurityConst from ._securetransport.low_level import ( diff --git a/newrelic/packages/urllib3/packages/backports/weakref_finalize.py b/newrelic/packages/urllib3/packages/backports/weakref_finalize.py new file mode 100644 index 0000000000..a2f2966e54 --- /dev/null +++ b/newrelic/packages/urllib3/packages/backports/weakref_finalize.py @@ -0,0 +1,155 @@ +# -*- coding: utf-8 -*- +""" +backports.weakref_finalize +~~~~~~~~~~~~~~~~~~ + +Backports the Python 3 ``weakref.finalize`` method. +""" +from __future__ import absolute_import + +import itertools +import sys +from weakref import ref + +__all__ = ["weakref_finalize"] + + +class weakref_finalize(object): + """Class for finalization of weakrefable objects + finalize(obj, func, *args, **kwargs) returns a callable finalizer + object which will be called when obj is garbage collected. The + first time the finalizer is called it evaluates func(*arg, **kwargs) + and returns the result. After this the finalizer is dead, and + calling it just returns None. + When the program exits any remaining finalizers for which the + atexit attribute is true will be run in reverse order of creation. + By default atexit is true. + """ + + # Finalizer objects don't have any state of their own. They are + # just used as keys to lookup _Info objects in the registry. This + # ensures that they cannot be part of a ref-cycle. + + __slots__ = () + _registry = {} + _shutdown = False + _index_iter = itertools.count() + _dirty = False + _registered_with_atexit = False + + class _Info(object): + __slots__ = ("weakref", "func", "args", "kwargs", "atexit", "index") + + def __init__(self, obj, func, *args, **kwargs): + if not self._registered_with_atexit: + # We may register the exit function more than once because + # of a thread race, but that is harmless + import atexit + + atexit.register(self._exitfunc) + weakref_finalize._registered_with_atexit = True + info = self._Info() + info.weakref = ref(obj, self) + info.func = func + info.args = args + info.kwargs = kwargs or None + info.atexit = True + info.index = next(self._index_iter) + self._registry[self] = info + weakref_finalize._dirty = True + + def __call__(self, _=None): + """If alive then mark as dead and return func(*args, **kwargs); + otherwise return None""" + info = self._registry.pop(self, None) + if info and not self._shutdown: + return info.func(*info.args, **(info.kwargs or {})) + + def detach(self): + """If alive then mark as dead and return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None and self._registry.pop(self, None): + return (obj, info.func, info.args, info.kwargs or {}) + + def peek(self): + """If alive then return (obj, func, args, kwargs); + otherwise return None""" + info = self._registry.get(self) + obj = info and info.weakref() + if obj is not None: + return (obj, info.func, info.args, info.kwargs or {}) + + @property + def alive(self): + """Whether finalizer is alive""" + return self in self._registry + + @property + def atexit(self): + """Whether finalizer should be called at exit""" + info = self._registry.get(self) + return bool(info) and info.atexit + + @atexit.setter + def atexit(self, value): + info = self._registry.get(self) + if info: + info.atexit = bool(value) + + def __repr__(self): + info = self._registry.get(self) + obj = info and info.weakref() + if obj is None: + return "<%s object at %#x; dead>" % (type(self).__name__, id(self)) + else: + return "<%s object at %#x; for %r at %#x>" % ( + type(self).__name__, + id(self), + type(obj).__name__, + id(obj), + ) + + @classmethod + def _select_for_exit(cls): + # Return live finalizers marked for exit, oldest first + L = [(f, i) for (f, i) in cls._registry.items() if i.atexit] + L.sort(key=lambda item: item[1].index) + return [f for (f, i) in L] + + @classmethod + def _exitfunc(cls): + # At shutdown invoke finalizers for which atexit is true. + # This is called once all other non-daemonic threads have been + # joined. + reenable_gc = False + try: + if cls._registry: + import gc + + if gc.isenabled(): + reenable_gc = True + gc.disable() + pending = None + while True: + if pending is None or weakref_finalize._dirty: + pending = cls._select_for_exit() + weakref_finalize._dirty = False + if not pending: + break + f = pending.pop() + try: + # gc is disabled, so (assuming no daemonic + # threads) the following is the only line in + # this function which might trigger creation + # of a new finalizer + f() + except Exception: + sys.excepthook(*sys.exc_info()) + assert f not in cls._registry + finally: + # prevent any more finalizers from executing during shutdown + weakref_finalize._shutdown = True + if reenable_gc: + gc.enable() diff --git a/newrelic/packages/urllib3/poolmanager.py b/newrelic/packages/urllib3/poolmanager.py index ca4ec34118..fb51bf7d96 100644 --- a/newrelic/packages/urllib3/poolmanager.py +++ b/newrelic/packages/urllib3/poolmanager.py @@ -4,7 +4,7 @@ import functools import logging -from ._collections import RecentlyUsedContainer +from ._collections import HTTPHeaderDict, RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme from .exceptions import ( LocationValueError, @@ -171,7 +171,7 @@ class PoolManager(RequestMethods): def __init__(self, num_pools=10, headers=None, **connection_pool_kw): RequestMethods.__init__(self, headers) self.connection_pool_kw = connection_pool_kw - self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) + self.pools = RecentlyUsedContainer(num_pools) # Locally set the pool classes and keys so other PoolManagers can # override them. @@ -382,9 +382,12 @@ def urlopen(self, method, url, redirect=True, **kw): # Support relative URLs for redirecting. redirect_location = urljoin(url, redirect_location) - # RFC 7231, Section 6.4.4 if response.status == 303: + # Change the method according to RFC 9110, Section 15.4.4. method = "GET" + # And lose the body not to transfer anything sensitive. + kw["body"] = None + kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change() retries = kw.get("retries") if not isinstance(retries, Retry): diff --git a/newrelic/packages/urllib3/request.py b/newrelic/packages/urllib3/request.py index 398386a5b9..3b4cf99922 100644 --- a/newrelic/packages/urllib3/request.py +++ b/newrelic/packages/urllib3/request.py @@ -1,6 +1,9 @@ from __future__ import absolute_import +import sys + from .filepost import encode_multipart_formdata +from .packages import six from .packages.six.moves.urllib.parse import urlencode __all__ = ["RequestMethods"] @@ -168,3 +171,21 @@ def request_encode_body( extra_kw.update(urlopen_kw) return self.urlopen(method, url, **extra_kw) + + +if not six.PY2: + + class RequestModule(sys.modules[__name__].__class__): + def __call__(self, *args, **kwargs): + """ + If user tries to call this module directly urllib3 v2.x style raise an error to the user + suggesting they may need urllib3 v2 + """ + raise TypeError( + "'module' object is not callable\n" + "urllib3.request() method is not supported in this release, " + "upgrade to urllib3 v2 to use it\n" + "see https://urllib3.readthedocs.io/en/stable/v2-migration-guide.html" + ) + + sys.modules[__name__].__class__ = RequestModule diff --git a/newrelic/packages/urllib3/util/retry.py b/newrelic/packages/urllib3/util/retry.py index 2490d5e5b6..9a1e90d0b2 100644 --- a/newrelic/packages/urllib3/util/retry.py +++ b/newrelic/packages/urllib3/util/retry.py @@ -235,7 +235,9 @@ class Retry(object): RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) #: Default headers to be used for ``remove_headers_on_redirect`` - DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Authorization"]) + DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset( + ["Cookie", "Authorization", "Proxy-Authorization"] + ) #: Maximum backoff time. DEFAULT_BACKOFF_MAX = 120 diff --git a/newrelic/packages/wrapt/__init__.py b/newrelic/packages/wrapt/__init__.py index ee6539b774..ed31a94313 100644 --- a/newrelic/packages/wrapt/__init__.py +++ b/newrelic/packages/wrapt/__init__.py @@ -1,12 +1,15 @@ -__version_info__ = ('1', '14', '1') +__version_info__ = ('1', '16', '0') __version__ = '.'.join(__version_info__) -from .wrappers import (ObjectProxy, CallableObjectProxy, FunctionWrapper, - BoundFunctionWrapper, WeakFunctionProxy, PartialCallableObjectProxy, - resolve_path, apply_patch, wrap_object, wrap_object_attribute, +from .__wrapt__ import (ObjectProxy, CallableObjectProxy, FunctionWrapper, + BoundFunctionWrapper, PartialCallableObjectProxy) + +from .patches import (resolve_path, apply_patch, wrap_object, wrap_object_attribute, function_wrapper, wrap_function_wrapper, patch_function_wrapper, transient_function_wrapper) +from .weakrefs import WeakFunctionProxy + from .decorators import (adapter_factory, AdapterFactory, decorator, synchronized) diff --git a/newrelic/packages/wrapt/__wrapt__.py b/newrelic/packages/wrapt/__wrapt__.py new file mode 100644 index 0000000000..9933b2c972 --- /dev/null +++ b/newrelic/packages/wrapt/__wrapt__.py @@ -0,0 +1,14 @@ +import os + +from .wrappers import (ObjectProxy, CallableObjectProxy, + PartialCallableObjectProxy, FunctionWrapper, + BoundFunctionWrapper, _FunctionWrapperBase) + +try: + if not os.environ.get('WRAPT_DISABLE_EXTENSIONS'): + from ._wrappers import (ObjectProxy, CallableObjectProxy, + PartialCallableObjectProxy, FunctionWrapper, + BoundFunctionWrapper, _FunctionWrapperBase) + +except ImportError: + pass diff --git a/newrelic/packages/wrapt/_wrappers.c b/newrelic/packages/wrapt/_wrappers.c index 67c5d5e1af..e0e1b5bc65 100644 --- a/newrelic/packages/wrapt/_wrappers.c +++ b/newrelic/packages/wrapt/_wrappers.c @@ -1139,6 +1139,30 @@ static int WraptObjectProxy_setitem(WraptObjectProxyObject *self, /* ------------------------------------------------------------------------- */ +static PyObject *WraptObjectProxy_self_setattr( + WraptObjectProxyObject *self, PyObject *args) +{ + PyObject *name = NULL; + PyObject *value = NULL; + +#if PY_MAJOR_VERSION >= 3 + if (!PyArg_ParseTuple(args, "UO:__self_setattr__", &name, &value)) + return NULL; +#else + if (!PyArg_ParseTuple(args, "SO:__self_setattr__", &name, &value)) + return NULL; +#endif + + if (PyObject_GenericSetAttr((PyObject *)self, name, value) != 0) { + return NULL; + } + + Py_INCREF(Py_None); + return Py_None; +} + +/* ------------------------------------------------------------------------- */ + static PyObject *WraptObjectProxy_dir( WraptObjectProxyObject *self, PyObject *args) { @@ -1464,6 +1488,19 @@ static PyObject *WraptObjectProxy_get_class( /* ------------------------------------------------------------------------- */ +static int WraptObjectProxy_set_class(WraptObjectProxyObject *self, + PyObject *value) +{ + if (!self->wrapped) { + PyErr_SetString(PyExc_ValueError, "wrapper has not been initialized"); + return -1; + } + + return PyObject_SetAttrString(self->wrapped, "__class__", value); +} + +/* ------------------------------------------------------------------------- */ + static PyObject *WraptObjectProxy_get_annotations( WraptObjectProxyObject *self) { @@ -1535,6 +1572,9 @@ static PyObject *WraptObjectProxy_getattro( if (object) return object; + if (!PyErr_ExceptionMatches(PyExc_AttributeError)) + return NULL; + PyErr_Clear(); if (!getattr_str) { @@ -1738,6 +1778,8 @@ static PyMappingMethods WraptObjectProxy_as_mapping = { }; static PyMethodDef WraptObjectProxy_methods[] = { + { "__self_setattr__", (PyCFunction)WraptObjectProxy_self_setattr, + METH_VARARGS , 0 }, { "__dir__", (PyCFunction)WraptObjectProxy_dir, METH_NOARGS, 0 }, { "__enter__", (PyCFunction)WraptObjectProxy_enter, METH_VARARGS | METH_KEYWORDS, 0 }, @@ -1776,7 +1818,7 @@ static PyGetSetDef WraptObjectProxy_getset[] = { { "__doc__", (getter)WraptObjectProxy_get_doc, (setter)WraptObjectProxy_set_doc, 0 }, { "__class__", (getter)WraptObjectProxy_get_class, - NULL, 0 }, + (setter)WraptObjectProxy_set_class, 0 }, { "__annotations__", (getter)WraptObjectProxy_get_annotations, (setter)WraptObjectProxy_set_annotations, 0 }, { "__wrapped__", (getter)WraptObjectProxy_get_wrapped, @@ -2547,7 +2589,6 @@ static PyObject *WraptFunctionWrapperBase_set_name( static PyObject *WraptFunctionWrapperBase_instancecheck( WraptFunctionWrapperObject *self, PyObject *instance) { - PyObject *object = NULL; PyObject *result = NULL; int check = 0; diff --git a/newrelic/packages/wrapt/decorators.py b/newrelic/packages/wrapt/decorators.py index c3f2547295..c80a4bb72e 100644 --- a/newrelic/packages/wrapt/decorators.py +++ b/newrelic/packages/wrapt/decorators.py @@ -41,7 +41,7 @@ def exec_(_code_, _globs_=None, _locs_=None): except ImportError: pass -from .wrappers import (FunctionWrapper, BoundFunctionWrapper, ObjectProxy, +from .__wrapt__ import (FunctionWrapper, BoundFunctionWrapper, ObjectProxy, CallableObjectProxy) # Adapter wrapper for the wrapped function which will overlay certain diff --git a/newrelic/packages/wrapt/importer.py b/newrelic/packages/wrapt/importer.py index 5c4d4cc663..23fcbd2f63 100644 --- a/newrelic/packages/wrapt/importer.py +++ b/newrelic/packages/wrapt/importer.py @@ -15,7 +15,7 @@ string_types = str, from importlib.util import find_spec -from .decorators import synchronized +from .__wrapt__ import ObjectProxy # The dictionary registering any post import hooks to be triggered once # the target module has been imported. Once a module has been imported @@ -45,7 +45,6 @@ def import_hook(module): return callback(module) return import_hook -@synchronized(_post_import_hooks_lock) def register_post_import_hook(hook, name): # Create a deferred import hook if hook is a string name rather than # a callable function. @@ -53,51 +52,32 @@ def register_post_import_hook(hook, name): if isinstance(hook, string_types): hook = _create_import_hook_from_string(hook) - # Automatically install the import hook finder if it has not already - # been installed. + with _post_import_hooks_lock: + # Automatically install the import hook finder if it has not already + # been installed. - global _post_import_hooks_init + global _post_import_hooks_init - if not _post_import_hooks_init: - _post_import_hooks_init = True - sys.meta_path.insert(0, ImportHookFinder()) + if not _post_import_hooks_init: + _post_import_hooks_init = True + sys.meta_path.insert(0, ImportHookFinder()) - # Determine if any prior registration of a post import hook for - # the target modules has occurred and act appropriately. - - hooks = _post_import_hooks.get(name, None) - - if hooks is None: - # No prior registration of post import hooks for the target - # module. We need to check whether the module has already been - # imported. If it has we fire the hook immediately and add an - # empty list to the registry to indicate that the module has - # already been imported and hooks have fired. Otherwise add - # the post import hook to the registry. + # Check if the module is already imported. If not, register the hook + # to be called after import. module = sys.modules.get(name, None) - if module is not None: - _post_import_hooks[name] = [] - hook(module) - - else: - _post_import_hooks[name] = [hook] + if module is None: + _post_import_hooks.setdefault(name, []).append(hook) - elif hooks == []: - # A prior registration of port import hooks for the target - # module was done and the hooks already fired. Fire the hook - # immediately. + # If the module is already imported, we fire the hook right away. Note that + # the hook is called outside of the lock to avoid deadlocks if code run as a + # consequence of calling the module import hook in turn triggers a separate + # thread which tries to register an import hook. - module = sys.modules[name] + if module is not None: hook(module) - else: - # A prior registration of port import hooks for the target - # module was done but the module has not yet been imported. - - _post_import_hooks[name].append(hook) - # Register post import hooks defined as package entry points. def _create_import_hook_from_entrypoint(entrypoint): @@ -124,16 +104,18 @@ def discover_post_import_hooks(group): # exception is raised in any of the post import hooks, that will cause # the import of the target module to fail. -@synchronized(_post_import_hooks_lock) def notify_module_loaded(module): name = getattr(module, '__name__', None) - hooks = _post_import_hooks.get(name, None) - if hooks: - _post_import_hooks[name] = [] + with _post_import_hooks_lock: + hooks = _post_import_hooks.pop(name, ()) - for hook in hooks: - hook(module) + # Note that the hook is called outside of the lock to avoid deadlocks if + # code run as a consequence of calling the module import hook in turn + # triggers a separate thread which tries to register an import hook. + + for hook in hooks: + hook(module) # A custom module import finder. This intercepts attempts to import # modules and watches out for attempts to import target modules of @@ -148,20 +130,45 @@ def load_module(self, fullname): return module -class _ImportHookChainedLoader: +class _ImportHookChainedLoader(ObjectProxy): def __init__(self, loader): - self.loader = loader + super(_ImportHookChainedLoader, self).__init__(loader) if hasattr(loader, "load_module"): - self.load_module = self._load_module + self.__self_setattr__('load_module', self._self_load_module) if hasattr(loader, "create_module"): - self.create_module = self._create_module + self.__self_setattr__('create_module', self._self_create_module) if hasattr(loader, "exec_module"): - self.exec_module = self._exec_module - - def _load_module(self, fullname): - module = self.loader.load_module(fullname) + self.__self_setattr__('exec_module', self._self_exec_module) + + def _self_set_loader(self, module): + # Set module's loader to self.__wrapped__ unless it's already set to + # something else. Import machinery will set it to spec.loader if it is + # None, so handle None as well. The module may not support attribute + # assignment, in which case we simply skip it. Note that we also deal + # with __loader__ not existing at all. This is to future proof things + # due to proposal to remove the attribue as described in the GitHub + # issue at https://github.com/python/cpython/issues/77458. Also prior + # to Python 3.3, the __loader__ attribute was only set if a custom + # module loader was used. It isn't clear whether the attribute still + # existed in that case or was set to None. + + class UNDEFINED: pass + + if getattr(module, "__loader__", UNDEFINED) in (None, self): + try: + module.__loader__ = self.__wrapped__ + except AttributeError: + pass + + if (getattr(module, "__spec__", None) is not None + and getattr(module.__spec__, "loader", None) is self): + module.__spec__.loader = self.__wrapped__ + + def _self_load_module(self, fullname): + module = self.__wrapped__.load_module(fullname) + self._self_set_loader(module) notify_module_loaded(module) return module @@ -169,11 +176,12 @@ def _load_module(self, fullname): # Python 3.4 introduced create_module() and exec_module() instead of # load_module() alone. Splitting the two steps. - def _create_module(self, spec): - return self.loader.create_module(spec) + def _self_create_module(self, spec): + return self.__wrapped__.create_module(spec) - def _exec_module(self, module): - self.loader.exec_module(module) + def _self_exec_module(self, module): + self._self_set_loader(module) + self.__wrapped__.exec_module(module) notify_module_loaded(module) class ImportHookFinder: @@ -181,14 +189,14 @@ class ImportHookFinder: def __init__(self): self.in_progress = {} - @synchronized(_post_import_hooks_lock) def find_module(self, fullname, path=None): # If the module being imported is not one we have registered # post import hooks for, we can return immediately. We will # take no further part in the importing of this module. - if not fullname in _post_import_hooks: - return None + with _post_import_hooks_lock: + if fullname not in _post_import_hooks: + return None # When we are interested in a specific module, we will call back # into the import system a second time to defer to the import @@ -244,8 +252,9 @@ def find_spec(self, fullname, path=None, target=None): # post import hooks for, we can return immediately. We will # take no further part in the importing of this module. - if not fullname in _post_import_hooks: - return None + with _post_import_hooks_lock: + if fullname not in _post_import_hooks: + return None # When we are interested in a specific module, we will call back # into the import system a second time to defer to the import diff --git a/newrelic/packages/wrapt/patches.py b/newrelic/packages/wrapt/patches.py new file mode 100644 index 0000000000..e22adf7ca8 --- /dev/null +++ b/newrelic/packages/wrapt/patches.py @@ -0,0 +1,141 @@ +import inspect +import sys + +PY2 = sys.version_info[0] == 2 + +if PY2: + string_types = basestring, +else: + string_types = str, + +from .__wrapt__ import FunctionWrapper + +# Helper functions for applying wrappers to existing functions. + +def resolve_path(module, name): + if isinstance(module, string_types): + __import__(module) + module = sys.modules[module] + + parent = module + + path = name.split('.') + attribute = path[0] + + # We can't just always use getattr() because in doing + # that on a class it will cause binding to occur which + # will complicate things later and cause some things not + # to work. For the case of a class we therefore access + # the __dict__ directly. To cope though with the wrong + # class being given to us, or a method being moved into + # a base class, we need to walk the class hierarchy to + # work out exactly which __dict__ the method was defined + # in, as accessing it from __dict__ will fail if it was + # not actually on the class given. Fallback to using + # getattr() if we can't find it. If it truly doesn't + # exist, then that will fail. + + def lookup_attribute(parent, attribute): + if inspect.isclass(parent): + for cls in inspect.getmro(parent): + if attribute in vars(cls): + return vars(cls)[attribute] + else: + return getattr(parent, attribute) + else: + return getattr(parent, attribute) + + original = lookup_attribute(parent, attribute) + + for attribute in path[1:]: + parent = original + original = lookup_attribute(parent, attribute) + + return (parent, attribute, original) + +def apply_patch(parent, attribute, replacement): + setattr(parent, attribute, replacement) + +def wrap_object(module, name, factory, args=(), kwargs={}): + (parent, attribute, original) = resolve_path(module, name) + wrapper = factory(original, *args, **kwargs) + apply_patch(parent, attribute, wrapper) + return wrapper + +# Function for applying a proxy object to an attribute of a class +# instance. The wrapper works by defining an attribute of the same name +# on the class which is a descriptor and which intercepts access to the +# instance attribute. Note that this cannot be used on attributes which +# are themselves defined by a property object. + +class AttributeWrapper(object): + + def __init__(self, attribute, factory, args, kwargs): + self.attribute = attribute + self.factory = factory + self.args = args + self.kwargs = kwargs + + def __get__(self, instance, owner): + value = instance.__dict__[self.attribute] + return self.factory(value, *self.args, **self.kwargs) + + def __set__(self, instance, value): + instance.__dict__[self.attribute] = value + + def __delete__(self, instance): + del instance.__dict__[self.attribute] + +def wrap_object_attribute(module, name, factory, args=(), kwargs={}): + path, attribute = name.rsplit('.', 1) + parent = resolve_path(module, path)[2] + wrapper = AttributeWrapper(attribute, factory, args, kwargs) + apply_patch(parent, attribute, wrapper) + return wrapper + +# Functions for creating a simple decorator using a FunctionWrapper, +# plus short cut functions for applying wrappers to functions. These are +# for use when doing monkey patching. For a more featured way of +# creating decorators see the decorator decorator instead. + +def function_wrapper(wrapper): + def _wrapper(wrapped, instance, args, kwargs): + target_wrapped = args[0] + if instance is None: + target_wrapper = wrapper + elif inspect.isclass(instance): + target_wrapper = wrapper.__get__(None, instance) + else: + target_wrapper = wrapper.__get__(instance, type(instance)) + return FunctionWrapper(target_wrapped, target_wrapper) + return FunctionWrapper(wrapper, _wrapper) + +def wrap_function_wrapper(module, name, wrapper): + return wrap_object(module, name, FunctionWrapper, (wrapper,)) + +def patch_function_wrapper(module, name, enabled=None): + def _wrapper(wrapper): + return wrap_object(module, name, FunctionWrapper, (wrapper, enabled)) + return _wrapper + +def transient_function_wrapper(module, name): + def _decorator(wrapper): + def _wrapper(wrapped, instance, args, kwargs): + target_wrapped = args[0] + if instance is None: + target_wrapper = wrapper + elif inspect.isclass(instance): + target_wrapper = wrapper.__get__(None, instance) + else: + target_wrapper = wrapper.__get__(instance, type(instance)) + def _execute(wrapped, instance, args, kwargs): + (parent, attribute, original) = resolve_path(module, name) + replacement = FunctionWrapper(original, target_wrapper) + setattr(parent, attribute, replacement) + try: + return wrapped(*args, **kwargs) + finally: + setattr(parent, attribute, original) + return FunctionWrapper(target_wrapped, _execute) + return FunctionWrapper(wrapper, _wrapper) + return _decorator diff --git a/newrelic/packages/wrapt/weakrefs.py b/newrelic/packages/wrapt/weakrefs.py new file mode 100644 index 0000000000..f931b60d5f --- /dev/null +++ b/newrelic/packages/wrapt/weakrefs.py @@ -0,0 +1,98 @@ +import functools +import weakref + +from .__wrapt__ import ObjectProxy, _FunctionWrapperBase + +# A weak function proxy. This will work on instance methods, class +# methods, static methods and regular functions. Special treatment is +# needed for the method types because the bound method is effectively a +# transient object and applying a weak reference to one will immediately +# result in it being destroyed and the weakref callback called. The weak +# reference is therefore applied to the instance the method is bound to +# and the original function. The function is then rebound at the point +# of a call via the weak function proxy. + +def _weak_function_proxy_callback(ref, proxy, callback): + if proxy._self_expired: + return + + proxy._self_expired = True + + # This could raise an exception. We let it propagate back and let + # the weakref.proxy() deal with it, at which point it generally + # prints out a short error message direct to stderr and keeps going. + + if callback is not None: + callback(proxy) + +class WeakFunctionProxy(ObjectProxy): + + __slots__ = ('_self_expired', '_self_instance') + + def __init__(self, wrapped, callback=None): + # We need to determine if the wrapped function is actually a + # bound method. In the case of a bound method, we need to keep a + # reference to the original unbound function and the instance. + # This is necessary because if we hold a reference to the bound + # function, it will be the only reference and given it is a + # temporary object, it will almost immediately expire and + # the weakref callback triggered. So what is done is that we + # hold a reference to the instance and unbound function and + # when called bind the function to the instance once again and + # then call it. Note that we avoid using a nested function for + # the callback here so as not to cause any odd reference cycles. + + _callback = callback and functools.partial( + _weak_function_proxy_callback, proxy=self, + callback=callback) + + self._self_expired = False + + if isinstance(wrapped, _FunctionWrapperBase): + self._self_instance = weakref.ref(wrapped._self_instance, + _callback) + + if wrapped._self_parent is not None: + super(WeakFunctionProxy, self).__init__( + weakref.proxy(wrapped._self_parent, _callback)) + + else: + super(WeakFunctionProxy, self).__init__( + weakref.proxy(wrapped, _callback)) + + return + + try: + self._self_instance = weakref.ref(wrapped.__self__, _callback) + + super(WeakFunctionProxy, self).__init__( + weakref.proxy(wrapped.__func__, _callback)) + + except AttributeError: + self._self_instance = None + + super(WeakFunctionProxy, self).__init__( + weakref.proxy(wrapped, _callback)) + + def __call__(*args, **kwargs): + def _unpack_self(self, *args): + return self, args + + self, args = _unpack_self(*args) + + # We perform a boolean check here on the instance and wrapped + # function as that will trigger the reference error prior to + # calling if the reference had expired. + + instance = self._self_instance and self._self_instance() + function = self.__wrapped__ and self.__wrapped__ + + # If the wrapped function was originally a bound function, for + # which we retained a reference to the instance and the unbound + # function we need to rebind the function and then call it. If + # not just called the wrapped function. + + if instance is None: + return self.__wrapped__(*args, **kwargs) + + return function.__get__(instance, type(instance))(*args, **kwargs) diff --git a/newrelic/packages/wrapt/wrappers.py b/newrelic/packages/wrapt/wrappers.py index 2716cd1da1..dfc3440db4 100644 --- a/newrelic/packages/wrapt/wrappers.py +++ b/newrelic/packages/wrapt/wrappers.py @@ -1,8 +1,5 @@ -import os import sys -import functools import operator -import weakref import inspect PY2 = sys.version_info[0] == 2 @@ -94,6 +91,9 @@ def __init__(self, wrapped): except AttributeError: pass + def __self_setattr__(self, name, value): + object.__setattr__(self, name, value) + @property def __name__(self): return self.__wrapped__.__name__ @@ -445,12 +445,22 @@ def __reduce_ex__(self, protocol): class CallableObjectProxy(ObjectProxy): - def __call__(self, *args, **kwargs): + def __call__(*args, **kwargs): + def _unpack_self(self, *args): + return self, args + + self, args = _unpack_self(*args) + return self.__wrapped__(*args, **kwargs) class PartialCallableObjectProxy(ObjectProxy): - def __init__(self, *args, **kwargs): + def __init__(*args, **kwargs): + def _unpack_self(self, *args): + return self, args + + self, args = _unpack_self(*args) + if len(args) < 1: raise TypeError('partial type takes at least one argument') @@ -464,7 +474,12 @@ def __init__(self, *args, **kwargs): self._self_args = args self._self_kwargs = kwargs - def __call__(self, *args, **kwargs): + def __call__(*args, **kwargs): + def _unpack_self(self, *args): + return self, args + + self, args = _unpack_self(*args) + _args = self._self_args + args _kwargs = dict(self._self_kwargs) @@ -544,7 +559,12 @@ def __get__(self, instance, owner): return self - def __call__(self, *args, **kwargs): + def __call__(*args, **kwargs): + def _unpack_self(self, *args): + return self, args + + self, args = _unpack_self(*args) + # If enabled has been specified, then evaluate it at this point # and if the wrapper is not to be executed, then simply return # the bound function rather than a bound wrapper for the bound @@ -607,7 +627,12 @@ def __subclasscheck__(self, subclass): class BoundFunctionWrapper(_FunctionWrapperBase): - def __call__(self, *args, **kwargs): + def __call__(*args, **kwargs): + def _unpack_self(self, *args): + return self, args + + self, args = _unpack_self(*args) + # If enabled has been specified, then evaluate it at this point # and if the wrapper is not to be executed, then simply return # the bound function rather than a bound wrapper for the bound @@ -757,230 +782,3 @@ def __init__(self, wrapped, wrapper, enabled=None): super(FunctionWrapper, self).__init__(wrapped, None, wrapper, enabled, binding) - -try: - if not os.environ.get('WRAPT_DISABLE_EXTENSIONS'): - from ._wrappers import (ObjectProxy, CallableObjectProxy, - PartialCallableObjectProxy, FunctionWrapper, - BoundFunctionWrapper, _FunctionWrapperBase) -except ImportError: - pass - -# Helper functions for applying wrappers to existing functions. - -def resolve_path(module, name): - if isinstance(module, string_types): - __import__(module) - module = sys.modules[module] - - parent = module - - path = name.split('.') - attribute = path[0] - - # We can't just always use getattr() because in doing - # that on a class it will cause binding to occur which - # will complicate things later and cause some things not - # to work. For the case of a class we therefore access - # the __dict__ directly. To cope though with the wrong - # class being given to us, or a method being moved into - # a base class, we need to walk the class hierarchy to - # work out exactly which __dict__ the method was defined - # in, as accessing it from __dict__ will fail if it was - # not actually on the class given. Fallback to using - # getattr() if we can't find it. If it truly doesn't - # exist, then that will fail. - - def lookup_attribute(parent, attribute): - if inspect.isclass(parent): - for cls in inspect.getmro(parent): - if attribute in vars(cls): - return vars(cls)[attribute] - else: - return getattr(parent, attribute) - else: - return getattr(parent, attribute) - - original = lookup_attribute(parent, attribute) - - for attribute in path[1:]: - parent = original - original = lookup_attribute(parent, attribute) - - return (parent, attribute, original) - -def apply_patch(parent, attribute, replacement): - setattr(parent, attribute, replacement) - -def wrap_object(module, name, factory, args=(), kwargs={}): - (parent, attribute, original) = resolve_path(module, name) - wrapper = factory(original, *args, **kwargs) - apply_patch(parent, attribute, wrapper) - return wrapper - -# Function for applying a proxy object to an attribute of a class -# instance. The wrapper works by defining an attribute of the same name -# on the class which is a descriptor and which intercepts access to the -# instance attribute. Note that this cannot be used on attributes which -# are themselves defined by a property object. - -class AttributeWrapper(object): - - def __init__(self, attribute, factory, args, kwargs): - self.attribute = attribute - self.factory = factory - self.args = args - self.kwargs = kwargs - - def __get__(self, instance, owner): - value = instance.__dict__[self.attribute] - return self.factory(value, *self.args, **self.kwargs) - - def __set__(self, instance, value): - instance.__dict__[self.attribute] = value - - def __delete__(self, instance): - del instance.__dict__[self.attribute] - -def wrap_object_attribute(module, name, factory, args=(), kwargs={}): - path, attribute = name.rsplit('.', 1) - parent = resolve_path(module, path)[2] - wrapper = AttributeWrapper(attribute, factory, args, kwargs) - apply_patch(parent, attribute, wrapper) - return wrapper - -# Functions for creating a simple decorator using a FunctionWrapper, -# plus short cut functions for applying wrappers to functions. These are -# for use when doing monkey patching. For a more featured way of -# creating decorators see the decorator decorator instead. - -def function_wrapper(wrapper): - def _wrapper(wrapped, instance, args, kwargs): - target_wrapped = args[0] - if instance is None: - target_wrapper = wrapper - elif inspect.isclass(instance): - target_wrapper = wrapper.__get__(None, instance) - else: - target_wrapper = wrapper.__get__(instance, type(instance)) - return FunctionWrapper(target_wrapped, target_wrapper) - return FunctionWrapper(wrapper, _wrapper) - -def wrap_function_wrapper(module, name, wrapper): - return wrap_object(module, name, FunctionWrapper, (wrapper,)) - -def patch_function_wrapper(module, name): - def _wrapper(wrapper): - return wrap_object(module, name, FunctionWrapper, (wrapper,)) - return _wrapper - -def transient_function_wrapper(module, name): - def _decorator(wrapper): - def _wrapper(wrapped, instance, args, kwargs): - target_wrapped = args[0] - if instance is None: - target_wrapper = wrapper - elif inspect.isclass(instance): - target_wrapper = wrapper.__get__(None, instance) - else: - target_wrapper = wrapper.__get__(instance, type(instance)) - def _execute(wrapped, instance, args, kwargs): - (parent, attribute, original) = resolve_path(module, name) - replacement = FunctionWrapper(original, target_wrapper) - setattr(parent, attribute, replacement) - try: - return wrapped(*args, **kwargs) - finally: - setattr(parent, attribute, original) - return FunctionWrapper(target_wrapped, _execute) - return FunctionWrapper(wrapper, _wrapper) - return _decorator - -# A weak function proxy. This will work on instance methods, class -# methods, static methods and regular functions. Special treatment is -# needed for the method types because the bound method is effectively a -# transient object and applying a weak reference to one will immediately -# result in it being destroyed and the weakref callback called. The weak -# reference is therefore applied to the instance the method is bound to -# and the original function. The function is then rebound at the point -# of a call via the weak function proxy. - -def _weak_function_proxy_callback(ref, proxy, callback): - if proxy._self_expired: - return - - proxy._self_expired = True - - # This could raise an exception. We let it propagate back and let - # the weakref.proxy() deal with it, at which point it generally - # prints out a short error message direct to stderr and keeps going. - - if callback is not None: - callback(proxy) - -class WeakFunctionProxy(ObjectProxy): - - __slots__ = ('_self_expired', '_self_instance') - - def __init__(self, wrapped, callback=None): - # We need to determine if the wrapped function is actually a - # bound method. In the case of a bound method, we need to keep a - # reference to the original unbound function and the instance. - # This is necessary because if we hold a reference to the bound - # function, it will be the only reference and given it is a - # temporary object, it will almost immediately expire and - # the weakref callback triggered. So what is done is that we - # hold a reference to the instance and unbound function and - # when called bind the function to the instance once again and - # then call it. Note that we avoid using a nested function for - # the callback here so as not to cause any odd reference cycles. - - _callback = callback and functools.partial( - _weak_function_proxy_callback, proxy=self, - callback=callback) - - self._self_expired = False - - if isinstance(wrapped, _FunctionWrapperBase): - self._self_instance = weakref.ref(wrapped._self_instance, - _callback) - - if wrapped._self_parent is not None: - super(WeakFunctionProxy, self).__init__( - weakref.proxy(wrapped._self_parent, _callback)) - - else: - super(WeakFunctionProxy, self).__init__( - weakref.proxy(wrapped, _callback)) - - return - - try: - self._self_instance = weakref.ref(wrapped.__self__, _callback) - - super(WeakFunctionProxy, self).__init__( - weakref.proxy(wrapped.__func__, _callback)) - - except AttributeError: - self._self_instance = None - - super(WeakFunctionProxy, self).__init__( - weakref.proxy(wrapped, _callback)) - - def __call__(self, *args, **kwargs): - # We perform a boolean check here on the instance and wrapped - # function as that will trigger the reference error prior to - # calling if the reference had expired. - - instance = self._self_instance and self._self_instance() - function = self.__wrapped__ and self.__wrapped__ - - # If the wrapped function was originally a bound function, for - # which we retained a reference to the instance and the unbound - # function we need to rebind the function and then call it. If - # not just called the wrapped function. - - if instance is None: - return self.__wrapped__(*args, **kwargs) - - return function.__get__(instance, type(instance))(*args, **kwargs) diff --git a/newrelic/samplers/cpu_usage.py b/newrelic/samplers/cpu_usage.py index f19ccadbfa..5dff15db35 100644 --- a/newrelic/samplers/cpu_usage.py +++ b/newrelic/samplers/cpu_usage.py @@ -25,7 +25,7 @@ from newrelic.samplers.decorators import data_source_factory @data_source_factory(name='CPU Usage') -class _CPUUsageDataSource(object): +class _CPUUsageDataSource(): def __init__(self, settings, environ): self._timer = None diff --git a/newrelic/samplers/data_sampler.py b/newrelic/samplers/data_sampler.py index a7ecb3b4e5..9abb3c5536 100644 --- a/newrelic/samplers/data_sampler.py +++ b/newrelic/samplers/data_sampler.py @@ -23,7 +23,7 @@ _logger = logging.getLogger(__name__) -class DataSampler(object): +class DataSampler(): def __init__(self, consumer, source, name, settings, **properties): self.consumer = consumer @@ -93,7 +93,7 @@ def metrics(self): return [] if self.group: - return (('%s/%s' % (self.group, key), value) + return ((f'{self.group}/{key}', value) for key, value in self.instance()) else: return self.instance() diff --git a/newrelic/samplers/gc_data.py b/newrelic/samplers/gc_data.py index 169212c05e..4aebc704d1 100644 --- a/newrelic/samplers/gc_data.py +++ b/newrelic/samplers/gc_data.py @@ -25,7 +25,7 @@ @data_source_factory(name="Garbage Collector Metrics") -class _GCDataSource(object): +class _GCDataSource(): def __init__(self, settings, environ): self.gc_time_metrics = CustomMetrics() self.start_time = 0.0 @@ -55,12 +55,12 @@ def record_gc(self, phase, info): self.start_time = time.time() elif phase == "stop": total_time = time.time() - self.start_time - self.gc_time_metrics.record_custom_metric("GC/time/%d/all" % self.pid, total_time) + self.gc_time_metrics.record_custom_metric(f"GC/time/{self.pid}/all", total_time) for gen in range(0, 3): if gen <= current_generation: - self.gc_time_metrics.record_custom_metric("GC/time/%d/%d" % (self.pid, gen), total_time) + self.gc_time_metrics.record_custom_metric(f"GC/time/{self.pid}/{gen}", total_time) else: - self.gc_time_metrics.record_custom_metric("GC/time/%d/%d" % (self.pid, gen), 0) + self.gc_time_metrics.record_custom_metric(f"GC/time/{self.pid}/{gen}", 0) def start(self): if hasattr(gc, "callbacks"): @@ -83,10 +83,10 @@ def __call__(self): # Record object count in total and per generation if hasattr(gc, "get_count"): counts = gc.get_count() - yield ("GC/objects/%d/all" % self.pid, {"count": sum(counts)}) + yield (f"GC/objects/{self.pid}/all", {"count": sum(counts)}) for gen, count in enumerate(counts): yield ( - "GC/objects/%d/generation/%d" % (self.pid, gen), + f"GC/objects/{self.pid}/generation/{gen}", {"count": count}, ) @@ -97,7 +97,7 @@ def __call__(self): highest_types = Counter(object_types).most_common(self.top_object_count_limit) for obj_type, count in highest_types: yield ( - "GC/objects/%d/type/%s" % (self.pid, callable_name(obj_type)), + f"GC/objects/{self.pid}/type/{callable_name(obj_type)}", {"count": count}, ) @@ -111,7 +111,7 @@ def __call__(self): self.previous_stats[(stat_name, "all")] = count change_in_value = count - previous_value yield ( - "GC/%s/%d/all" % (stat_name, self.pid), + f"GC/{stat_name}/{self.pid}/all", {"count": change_in_value}, ) @@ -122,7 +122,7 @@ def __call__(self): change_in_value = stats[stat_name] - previous_value yield ( - "GC/%s/%d/%d" % (stat_name, self.pid, gen), + f"GC/{stat_name}/{self.pid}/{gen}", {"count": change_in_value}, ) diff --git a/newrelic/samplers/memory_usage.py b/newrelic/samplers/memory_usage.py index 11b75eef96..f1895bdfec 100644 --- a/newrelic/samplers/memory_usage.py +++ b/newrelic/samplers/memory_usage.py @@ -18,12 +18,15 @@ """ import os +from newrelic.core.config import global_settings from newrelic.common.system_info import physical_memory_used, total_physical_memory from newrelic.samplers.decorators import data_source_generator @data_source_generator(name="Memory Usage") def memory_usage_data_source(): + settings = global_settings() + memory = physical_memory_used() total_memory = total_physical_memory() pid = os.getpid() @@ -32,7 +35,8 @@ def memory_usage_data_source(): memory_utilization = (memory / total_memory) if total_memory != 0 else 0 yield ("Memory/Physical", memory) - yield ("Memory/Physical/%d" % (pid), memory) - yield ("Memory/Physical/Utilization", memory_utilization) - yield ("Memory/Physical/Utilization/%d" % (pid), memory_utilization) + + if settings.memory_runtime_pid_metrics.enabled: + yield (f"Memory/Physical/{pid}", memory) + yield (f"Memory/Physical/Utilization/{pid}", memory_utilization) diff --git a/setup.cfg b/setup.cfg index 453a10eeb5..8a41f1534d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,4 +5,4 @@ license_files = [flake8] max-line-length = 120 -extend-ignore = E122,E126,E127,E128,E203,E501,E722,F841,W504 +extend-ignore = E122,E126,E127,E128,E203,E501,E722,F841,W504,E731,F811 diff --git a/setup.py b/setup.py index 044125a23e..3cd9386517 100644 --- a/setup.py +++ b/setup.py @@ -12,17 +12,35 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function - import os import sys python_version = sys.version_info[:2] -assert python_version in ((2, 7),) or python_version >= ( - 3, - 7, -), "The New Relic Python agent only supports Python 2.7 and 3.7+." +if python_version >= (3, 7): + pass +else: + error_msg = "The New Relic Python agent only supports Python 3.7+. We recommend upgrading to a newer version of Python." + + try: + # Lookup table for the last agent versions to support each Python version. + last_supported_version_lookup = { + (2, 6): "3.4.0.95", + (2, 7): "9.13.0", + (3, 3): "3.4.0.95", + (3, 4): "4.20.0.120", + (3, 5): "5.24.0.153", + (3, 6): "7.16.0.178", + } + last_supported_version = last_supported_version_lookup.get(python_version, None) + + if last_supported_version: + python_version_str = "%s.%s" % (python_version[0], python_version[1]) + error_msg += " The last agent version to support Python %s was v%s." % (python_version_str, last_supported_version) + except Exception: + pass + + raise RuntimeError(error_msg) with_setuptools = False @@ -43,6 +61,9 @@ def newrelic_agent_guess_next_version(tag_version): + if hasattr(tag_version, "tag"): # For setuptools_scm 7.0+ + tag_version = tag_version.tag + version, _, _ = str(tag_version).partition("+") version_info = list(map(int, version.split("."))) if len(version_info) < 3: @@ -102,6 +123,8 @@ def build_extension(self, ext): "newrelic.hooks", "newrelic.network", "newrelic/packages", + "newrelic/packages/isort", + "newrelic/packages/isort/stdlibs", "newrelic/packages/urllib3", "newrelic/packages/urllib3/util", "newrelic/packages/urllib3/contrib", @@ -109,18 +132,20 @@ def build_extension(self, ext): "newrelic/packages/urllib3/packages", "newrelic/packages/urllib3/packages/backports", "newrelic/packages/wrapt", + "newrelic/packages/opentelemetry_proto", "newrelic.samplers", ] classifiers = [ "Development Status :: 5 - Production/Stable", "License :: OSI Approved :: Apache Software License", - "Programming Language :: Python :: 2.7", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: System :: Monitoring", @@ -134,7 +159,7 @@ def build_extension(self, ext): "git_describe_command": "git describe --dirty --tags --long --match *.*.*", "write_to": "newrelic/version.txt", }, - setup_requires=["setuptools_scm>=3.2,<7"], + setup_requires=["setuptools_scm>=3.2,<9"], description="New Relic Python Agent", long_description=open(readme_file).read(), url="https://docs.newrelic.com/docs/apm/agents/python-agent/", @@ -147,10 +172,11 @@ def build_extension(self, ext): zip_safe=False, classifiers=classifiers, packages=packages, - python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*", + python_requires=">=3.7", package_data={ "newrelic": ["newrelic.ini", "version.txt", "packages/urllib3/LICENSE.txt", "common/cacert.pem"], }, + #install_requires=["newrelic-security @ git+https://github.com/newrelic/csec-python-agent.git@develop#egg=newrelic-security"], extras_require={"infinite-tracing": ["grpcio", "protobuf"]}, ) diff --git a/tests/adapter_asgiref/conftest.py b/tests/adapter_asgiref/conftest.py new file mode 100644 index 0000000000..e840a574ac --- /dev/null +++ b/tests/adapter_asgiref/conftest.py @@ -0,0 +1,33 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from testing_support.fixture.event_loop import ( # noqa: F401; pylint: disable=W0611 + event_loop as loop, +) +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (adapter_asgiref)", default_settings=_default_settings +) diff --git a/tests/adapter_asgiref/test_context_propagation.py b/tests/adapter_asgiref/test_context_propagation.py new file mode 100644 index 0000000000..f79af93251 --- /dev/null +++ b/tests/adapter_asgiref/test_context_propagation.py @@ -0,0 +1,63 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from asgiref.sync import async_to_sync, sync_to_async +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.function_trace import function_trace +from newrelic.api.transaction import current_transaction + + +@sync_to_async +@function_trace() +def sync_func(): + assert current_transaction() + return 1 + + +@validate_transaction_metrics( + "test_context_propagation:test_sync_to_async_context_propagation", + scoped_metrics=[("Function/test_context_propagation:sync_func", 1)], + rollup_metrics=[("Function/test_context_propagation:sync_func", 1)], + background_task=True, +) +@background_task() +def test_sync_to_async_context_propagation(loop): + async def _test(): + return_val = await sync_func() + assert return_val == 1, "Sync function failed to return" + + loop.run_until_complete(_test()) + + +@async_to_sync +@function_trace() +async def async_func(): + assert current_transaction() + return 1 + + +@validate_transaction_metrics( + "test_context_propagation:test_async_to_sync_context_propagation", + scoped_metrics=[("Function/test_context_propagation:async_func", 1)], + rollup_metrics=[("Function/test_context_propagation:async_func", 1)], + background_task=True, +) +@background_task() +def test_async_to_sync_context_propagation(): + return_val = async_func() + assert return_val == 1, "Async function failed to return" diff --git a/tests/adapter_cheroot/conftest.py b/tests/adapter_cheroot/conftest.py index 37d9d4df4a..891dbc5a0f 100644 --- a/tests/adapter_cheroot/conftest.py +++ b/tests/adapter_cheroot/conftest.py @@ -12,19 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (adapter_cheroot)', - default_settings=_default_settings) + app_name="Python Agent Test (adapter_cheroot)", default_settings=_default_settings +) diff --git a/tests/adapter_daphne/conftest.py b/tests/adapter_daphne/conftest.py index 3b35b2ee65..6ee940fbbb 100644 --- a/tests/adapter_daphne/conftest.py +++ b/tests/adapter_daphne/conftest.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, diff --git a/tests/adapter_daphne/test_daphne.py b/tests/adapter_daphne/test_daphne.py index e5f9dd832b..88c916954a 100644 --- a/tests/adapter_daphne/test_daphne.py +++ b/tests/adapter_daphne/test_daphne.py @@ -98,7 +98,7 @@ async def fake_app(*args, **kwargs): server = daphne.server.Server( fake_app, - endpoints=["tcp:%d:interface=127.0.0.1" % port], + endpoints=[f"tcp:{port}:interface=127.0.0.1"], ready_callable=on_ready, signal_handlers=False, verbosity=9, @@ -124,13 +124,13 @@ def test_daphne_200(port, app): @validate_transaction_metrics( callable_name(app), custom_metrics=[ - ("Python/Dispatcher/Daphne/%s" % daphne.__version__, 1), + (f"Python/Dispatcher/Daphne/{daphne.__version__}", 1), ], ) @raise_background_exceptions() @wait_for_background_threads() def response(): - return urlopen("http://localhost:%d" % port, timeout=10) # nosec + return urlopen(f"http://localhost:{port}", timeout=10) # nosec assert response().status == 200 @@ -143,7 +143,7 @@ def test_daphne_500(port, app): @wait_for_background_threads() def _test(): try: - urlopen("http://localhost:%d/exc" % port) # nosec + urlopen(f"http://localhost:{port}/exc") # nosec except HTTPError: pass diff --git a/tests/adapter_gevent/conftest.py b/tests/adapter_gevent/conftest.py index 01dacc9e69..46d4c1ee3e 100644 --- a/tests/adapter_gevent/conftest.py +++ b/tests/adapter_gevent/conftest.py @@ -14,24 +14,29 @@ import pytest import webtest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, - 'debug.disable_harvest_until_shutdown': False, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.disable_harvest_until_shutdown": False, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (adapter_gevent)', - default_settings=_default_settings) + app_name="Python Agent Test (adapter_gevent)", default_settings=_default_settings +) + -@pytest.fixture(autouse=True, scope='session') +@pytest.fixture(autouse=True, scope="session") def target_application(): import _application + port = _application.setup_application() - return webtest.TestApp('http://localhost:%d' % port) + return webtest.TestApp(f"http://localhost:{port}") diff --git a/tests/adapter_gunicorn/conftest.py b/tests/adapter_gunicorn/conftest.py index 228742c964..95bb184d3a 100644 --- a/tests/adapter_gunicorn/conftest.py +++ b/tests/adapter_gunicorn/conftest.py @@ -12,18 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (gunicorn)', - default_settings=_default_settings) + app_name="Python Agent Test (gunicorn)", default_settings=_default_settings +) diff --git a/tests/adapter_gunicorn/test_aiohttp_app_factory.py b/tests/adapter_gunicorn/test_aiohttp_app_factory.py index dc16b1231a..55eeee05b2 100644 --- a/tests/adapter_gunicorn/test_aiohttp_app_factory.py +++ b/tests/adapter_gunicorn/test_aiohttp_app_factory.py @@ -36,7 +36,7 @@ def test_aiohttp_app_factory(nr_enabled): # Restart the server if it dies during testing for _ in range(5): PORT = get_open_port() - cmd = [gunicorn, '-b', '127.0.0.1:%d' % PORT, '--worker-class', + cmd = [gunicorn, '-b', f'127.0.0.1:{PORT}', '--worker-class', 'aiohttp.GunicornWebWorker', 'async_app:app_factory'] if nr_enabled: @@ -71,7 +71,7 @@ def test_aiohttp_app_factory(nr_enabled): else: continue - with urlopen('http://127.0.0.1:%d' % PORT) as resp: + with urlopen(f'http://127.0.0.1:{PORT}') as resp: assert resp.getcode() == 200 assert resp.read() == b'PONG' diff --git a/tests/adapter_gunicorn/test_asgi_app.py b/tests/adapter_gunicorn/test_asgi_app.py index 93e3484655..6ce6882477 100644 --- a/tests/adapter_gunicorn/test_asgi_app.py +++ b/tests/adapter_gunicorn/test_asgi_app.py @@ -31,7 +31,7 @@ def test_asgi_app(nr_enabled): gunicorn = os.path.join(os.environ['TOX_ENV_DIR'], 'bin', 'gunicorn') PORT = get_open_port() - cmd = [gunicorn, '-b', '127.0.0.1:%d' % PORT, '--worker-class', + cmd = [gunicorn, '-b', f'127.0.0.1:{PORT}', '--worker-class', 'worker.AsgiWorker', 'asgi_app:Application'] if nr_enabled: @@ -66,7 +66,7 @@ def test_asgi_app(nr_enabled): time.sleep(0.1) else: continue - with urlopen('http://127.0.0.1:%d' % PORT) as resp: + with urlopen(f'http://127.0.0.1:{PORT}') as resp: assert resp.getcode() == 200 assert resp.read() == b'PONG' diff --git a/tests/adapter_gunicorn/test_gaiohttp.py b/tests/adapter_gunicorn/test_gaiohttp.py index 9f205bad94..9b12611229 100644 --- a/tests/adapter_gunicorn/test_gaiohttp.py +++ b/tests/adapter_gunicorn/test_gaiohttp.py @@ -36,7 +36,7 @@ def test_gunicorn_gaiohttp_worker(nr_enabled): # Restart the server if it dies during testing for _ in range(5): PORT = get_open_port() - cmd = [gunicorn, '-b', '127.0.0.1:%d' % PORT, '-k', 'gaiohttp', + cmd = [gunicorn, '-b', f'127.0.0.1:{PORT}', '-k', 'gaiohttp', 'app:application'] if nr_enabled: @@ -69,7 +69,7 @@ def test_gunicorn_gaiohttp_worker(nr_enabled): else: continue - with urlopen('http://127.0.0.1:%d' % PORT) as resp: + with urlopen(f'http://127.0.0.1:{PORT}') as resp: assert resp.getcode() == 200 assert resp.read() == b'PONG' diff --git a/tests/adapter_gunicorn/worker.py b/tests/adapter_gunicorn/worker.py index 5d3a984980..f7bd868820 100644 --- a/tests/adapter_gunicorn/worker.py +++ b/tests/adapter_gunicorn/worker.py @@ -17,7 +17,7 @@ from gunicorn.workers.sync import SyncWorker -class WsgiProxy(object): +class WsgiProxy(): def __init__(self, asgi): self.asgi = asgi self.status_code = None diff --git a/tests/adapter_hypercorn/conftest.py b/tests/adapter_hypercorn/conftest.py index 2276e9415f..e6311a88d6 100644 --- a/tests/adapter_hypercorn/conftest.py +++ b/tests/adapter_hypercorn/conftest.py @@ -15,10 +15,13 @@ from testing_support.fixture.event_loop import ( # noqa: F401; pylint: disable=W0611 event_loop as loop, ) -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, diff --git a/tests/adapter_hypercorn/test_hypercorn.py b/tests/adapter_hypercorn/test_hypercorn.py index 8b53eee0ac..77f1223373 100644 --- a/tests/adapter_hypercorn/test_hypercorn.py +++ b/tests/adapter_hypercorn/test_hypercorn.py @@ -17,7 +17,6 @@ import time from urllib.request import HTTPError, urlopen -import pkg_resources import pytest from testing_support.fixtures import ( override_application_settings, @@ -39,8 +38,12 @@ from newrelic.api.transaction import ignore_transaction from newrelic.common.object_names import callable_name +from newrelic.common.package_version_utils import ( + get_package_version, + get_package_version_tuple, +) -HYPERCORN_VERSION = tuple(int(v) for v in pkg_resources.get_distribution("hypercorn").version.split(".")) +HYPERCORN_VERSION = get_package_version_tuple("hypercorn") asgi_2_unsupported = HYPERCORN_VERSION >= (0, 14, 1) wsgi_unsupported = HYPERCORN_VERSION < (0, 14, 1) @@ -60,6 +63,7 @@ def wsgi_app(environ, start_response): @pytest.fixture( + scope="session", params=( pytest.param( simple_app_v2_raw, @@ -78,7 +82,7 @@ def app(request): return request.param -@pytest.fixture() +@pytest.fixture(scope="session") def port(loop, app): import hypercorn.asyncio import hypercorn.config @@ -93,7 +97,7 @@ async def shutdown_trigger(): config = hypercorn.config.Config.from_mapping( { - "bind": ["127.0.0.1:%d" % port], + "bind": [f"127.0.0.1:{port}"], } ) @@ -119,7 +123,7 @@ def wait_for_port(port, retries=10): status = None for _ in range(retries): try: - status = urlopen("http://localhost:%d/ignored" % port, timeout=1).status # nosec + status = urlopen(f"http://localhost:{port}/ignored", timeout=1).status # nosec assert status == 200 return except Exception as e: @@ -127,23 +131,23 @@ def wait_for_port(port, retries=10): time.sleep(1) - raise RuntimeError("Failed to wait for port %d. Got status %s" % (port, status)) + raise RuntimeError(f"Failed to wait for port {port}. Got status {status}") @override_application_settings({"transaction_name.naming_scheme": "framework"}) def test_hypercorn_200(port, app): - hypercorn_version = pkg_resources.get_distribution("hypercorn").version + hypercorn_version = get_package_version("hypercorn") @validate_transaction_metrics( callable_name(app), custom_metrics=[ - ("Python/Dispatcher/Hypercorn/%s" % hypercorn_version, 1), + (f"Python/Dispatcher/Hypercorn/{hypercorn_version}", 1), ], ) @raise_background_exceptions() @wait_for_background_threads() def response(): - return urlopen("http://localhost:%d" % port, timeout=10) # nosec + return urlopen(f"http://localhost:{port}", timeout=10) # nosec assert response().status == 200 @@ -156,6 +160,6 @@ def test_hypercorn_500(port, app): @wait_for_background_threads() def _test(): with pytest.raises(HTTPError): - urlopen("http://localhost:%d/exc" % port) # nosec + urlopen(f"http://localhost:{port}/exc") # nosec _test() diff --git a/tests/adapter_uvicorn/conftest.py b/tests/adapter_uvicorn/conftest.py index 4f2f7c2df4..175284ba86 100644 --- a/tests/adapter_uvicorn/conftest.py +++ b/tests/adapter_uvicorn/conftest.py @@ -12,12 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, diff --git a/tests/adapter_uvicorn/test_uvicorn.py b/tests/adapter_uvicorn/test_uvicorn.py index 93d155aa85..6a6718891b 100644 --- a/tests/adapter_uvicorn/test_uvicorn.py +++ b/tests/adapter_uvicorn/test_uvicorn.py @@ -112,7 +112,7 @@ def test_uvicorn_200(port, app): @raise_background_exceptions() @wait_for_background_threads() def response(): - return urlopen("http://localhost:%d" % port) + return urlopen(f"http://localhost:{port}") assert response().status == 200 @@ -125,7 +125,7 @@ def test_uvicorn_500(port, app): @wait_for_background_threads() def _test(): try: - urlopen("http://localhost:%d/exc" % port) + urlopen(f"http://localhost:{port}/exc") except HTTPError: pass diff --git a/tests/adapter_waitress/conftest.py b/tests/adapter_waitress/conftest.py index aecbfd86d6..680005d3d7 100644 --- a/tests/adapter_waitress/conftest.py +++ b/tests/adapter_waitress/conftest.py @@ -20,6 +20,7 @@ ) _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, @@ -37,4 +38,4 @@ def target_application(): import _application port = _application.setup_application() - return webtest.TestApp("http://localhost:%d" % port) + return webtest.TestApp(f"http://localhost:{port}") diff --git a/tests/adapter_waitress/test_wsgi.py b/tests/adapter_waitress/test_wsgi.py index c9fa427196..ba0b402290 100644 --- a/tests/adapter_waitress/test_wsgi.py +++ b/tests/adapter_waitress/test_wsgi.py @@ -35,7 +35,7 @@ def test_wsgi_application_index(target_application): @validate_transaction_metrics( "_application:sample_application", custom_metrics=[ - ("Python/Dispatcher/Waitress/%s" % WAITRESS_VERSION, 1), + (f"Python/Dispatcher/Waitress/{WAITRESS_VERSION}", 1), ], ) @raise_background_exceptions() @@ -53,7 +53,7 @@ def test_raise_exception_application(target_application): @validate_transaction_metrics( "_application:sample_application", custom_metrics=[ - ("Python/Dispatcher/Waitress/%s" % WAITRESS_VERSION, 1), + (f"Python/Dispatcher/Waitress/{WAITRESS_VERSION}", 1), ], ) @raise_background_exceptions() @@ -71,7 +71,7 @@ def test_raise_exception_response(target_application): @validate_transaction_metrics( "_application:sample_application", custom_metrics=[ - ("Python/Dispatcher/Waitress/%s" % WAITRESS_VERSION, 1), + (f"Python/Dispatcher/Waitress/{WAITRESS_VERSION}", 1), ], ) @raise_background_exceptions() @@ -89,7 +89,7 @@ def test_raise_exception_finalize(target_application): @validate_transaction_metrics( "_application:sample_application", custom_metrics=[ - ("Python/Dispatcher/Waitress/%s" % WAITRESS_VERSION, 1), + (f"Python/Dispatcher/Waitress/{WAITRESS_VERSION}", 1), ], ) @raise_background_exceptions() diff --git a/tests/agent_features/_test_async_coroutine_trace.py b/tests/agent_features/_test_async_coroutine_trace.py index 51b81f5f64..1250b8c254 100644 --- a/tests/agent_features/_test_async_coroutine_trace.py +++ b/tests/agent_features/_test_async_coroutine_trace.py @@ -28,6 +28,7 @@ from newrelic.api.datastore_trace import datastore_trace from newrelic.api.external_trace import external_trace from newrelic.api.function_trace import function_trace +from newrelic.api.graphql_trace import graphql_operation_trace, graphql_resolver_trace from newrelic.api.memcache_trace import memcache_trace from newrelic.api.message_trace import message_trace @@ -41,6 +42,8 @@ (functools.partial(datastore_trace, "lib", "foo", "bar"), "Datastore/statement/lib/foo/bar"), (functools.partial(message_trace, "lib", "op", "typ", "name"), "MessageBroker/lib/typ/op/Named/name"), (functools.partial(memcache_trace, "cmd"), "Memcache/cmd"), + (functools.partial(graphql_operation_trace), "GraphQL/operation/GraphQL///"), + (functools.partial(graphql_resolver_trace), "GraphQL/resolve/GraphQL/"), ], ) def test_awaitable_timing(event_loop, trace, metric): @@ -79,6 +82,8 @@ def _test(): (functools.partial(datastore_trace, "lib", "foo", "bar"), "Datastore/statement/lib/foo/bar"), (functools.partial(message_trace, "lib", "op", "typ", "name"), "MessageBroker/lib/typ/op/Named/name"), (functools.partial(memcache_trace, "cmd"), "Memcache/cmd"), + (functools.partial(graphql_operation_trace), "GraphQL/operation/GraphQL///"), + (functools.partial(graphql_resolver_trace), "GraphQL/resolve/GraphQL/"), ], ) @pytest.mark.parametrize("yield_from", [True, False]) diff --git a/tests/agent_features/_test_async_generator_trace.py b/tests/agent_features/_test_async_generator_trace.py new file mode 100644 index 0000000000..30b970c372 --- /dev/null +++ b/tests/agent_features/_test_async_generator_trace.py @@ -0,0 +1,548 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import sys +import time + +import pytest +from testing_support.fixtures import capture_transaction_metrics, validate_tt_parenting +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.database_trace import database_trace +from newrelic.api.datastore_trace import datastore_trace +from newrelic.api.external_trace import external_trace +from newrelic.api.function_trace import function_trace +from newrelic.api.graphql_trace import graphql_operation_trace, graphql_resolver_trace +from newrelic.api.memcache_trace import memcache_trace +from newrelic.api.message_trace import message_trace + +asyncio = pytest.importorskip("asyncio") + + +@pytest.mark.parametrize( + "trace,metric", + [ + (functools.partial(function_trace, name="simple_gen"), "Function/simple_gen"), + (functools.partial(external_trace, library="lib", url="http://foo.com"), "External/foo.com/lib/"), + (functools.partial(database_trace, "select * from foo"), "Datastore/statement/None/foo/select"), + (functools.partial(datastore_trace, "lib", "foo", "bar"), "Datastore/statement/lib/foo/bar"), + (functools.partial(message_trace, "lib", "op", "typ", "name"), "MessageBroker/lib/typ/op/Named/name"), + (functools.partial(memcache_trace, "cmd"), "Memcache/cmd"), + (functools.partial(graphql_operation_trace), "GraphQL/operation/GraphQL///"), + (functools.partial(graphql_resolver_trace), "GraphQL/resolve/GraphQL/"), + ], +) +def test_async_generator_timing(event_loop, trace, metric): + @trace() + async def simple_gen(): + time.sleep(0.1) + yield + time.sleep(0.1) + + metrics = [] + full_metrics = {} + + @capture_transaction_metrics(metrics, full_metrics) + @validate_transaction_metrics( + "test_async_generator_timing", background_task=True, scoped_metrics=[(metric, 1)], rollup_metrics=[(metric, 1)] + ) + @background_task(name="test_async_generator_timing") + def _test_async_generator_timing(): + async def _test(): + async for _ in simple_gen(): + pass + + event_loop.run_until_complete(_test()) + _test_async_generator_timing() + + # Check that coroutines time the total call time (including pauses) + metric_key = (metric, "") + assert full_metrics[metric_key].total_call_time >= 0.2 + + +class MyException(Exception): + pass + + +@validate_transaction_metrics( + "test_async_generator_error", + background_task=True, + scoped_metrics=[("Function/agen", 1)], + rollup_metrics=[("Function/agen", 1)], +) +@validate_transaction_errors(errors=["_test_async_generator_trace:MyException"]) +def test_async_generator_error(event_loop): + @function_trace(name="agen") + async def agen(): + yield + + @background_task(name="test_async_generator_error") + async def _test(): + gen = agen() + await gen.asend(None) + await gen.athrow(MyException) + + with pytest.raises(MyException): + event_loop.run_until_complete(_test()) + + +@validate_transaction_metrics( + "test_async_generator_caught_exception", + background_task=True, + scoped_metrics=[("Function/agen", 1)], + rollup_metrics=[("Function/agen", 1)], +) +@validate_transaction_errors(errors=[]) +def test_async_generator_caught_exception(event_loop): + @function_trace(name="agen") + async def agen(): + for _ in range(2): + time.sleep(0.1) + try: + yield + except ValueError: + pass + + metrics = [] + full_metrics = {} + + @capture_transaction_metrics(metrics, full_metrics) + @background_task(name="test_async_generator_caught_exception") + def _test_async_generator_caught_exception(): + async def _test(): + gen = agen() + # kickstart the generator (the try/except logic is inside the + # generator) + await gen.asend(None) + await gen.athrow(ValueError) + + # consume the generator + async for _ in gen: + pass + + # The ValueError should not be reraised + event_loop.run_until_complete(_test()) + _test_async_generator_caught_exception() + + assert full_metrics[("Function/agen", "")].total_call_time >= 0.2 + + +@validate_transaction_metrics( + "test_async_generator_handles_terminal_nodes", + background_task=True, + scoped_metrics=[("Function/parent", 1), ("Function/agen", None)], + rollup_metrics=[("Function/parent", 1), ("Function/agen", None)], +) +def test_async_generator_handles_terminal_nodes(event_loop): + # sometimes coroutines can be called underneath terminal nodes + # In this case, the trace shouldn't actually be created and we also + # shouldn't get any errors + + @function_trace(name="agen") + async def agen(): + yield + time.sleep(0.1) + + @function_trace(name="parent", terminal=True) + async def parent(): + # parent calls child + async for _ in agen(): + pass + + metrics = [] + full_metrics = {} + + @capture_transaction_metrics(metrics, full_metrics) + @background_task(name="test_async_generator_handles_terminal_nodes") + def _test_async_generator_handles_terminal_nodes(): + async def _test(): + await parent() + + event_loop.run_until_complete(_test()) + _test_async_generator_handles_terminal_nodes() + + metric_key = ("Function/parent", "") + assert full_metrics[metric_key].total_exclusive_call_time >= 0.1 + + +@validate_transaction_metrics( + "test_async_generator_close_ends_trace", + background_task=True, + scoped_metrics=[("Function/agen", 1)], + rollup_metrics=[("Function/agen", 1)], +) +def test_async_generator_close_ends_trace(event_loop): + @function_trace(name="agen") + async def agen(): + yield + + @background_task(name="test_async_generator_close_ends_trace") + async def _test(): + gen = agen() + + # kickstart the coroutine + await gen.asend(None) + + # trace should be ended/recorded by close + await gen.aclose() + + # We may call gen.close as many times as we want + await gen.aclose() + + event_loop.run_until_complete(_test()) + +@validate_tt_parenting( + ( + "TransactionNode", + [ + ( + "FunctionNode", + [ + ("FunctionNode", []), + ], + ), + ], + ) +) +@validate_transaction_metrics( + "test_async_generator_parents", + background_task=True, + scoped_metrics=[("Function/child", 1), ("Function/parent", 1)], + rollup_metrics=[("Function/child", 1), ("Function/parent", 1)], +) +def test_async_generator_parents(event_loop): + @function_trace(name="child") + async def child(): + yield + time.sleep(0.1) + yield + + @function_trace(name="parent") + async def parent(): + time.sleep(0.1) + yield + async for _ in child(): + pass + + metrics = [] + full_metrics = {} + + @capture_transaction_metrics(metrics, full_metrics) + @background_task(name="test_async_generator_parents") + def _test_async_generator_parents(): + async def _test(): + async for _ in parent(): + pass + + event_loop.run_until_complete(_test()) + _test_async_generator_parents() + + # Check that the child time is subtracted from the parent time (parenting + # relationship is correctly established) + key = ("Function/parent", "") + assert full_metrics[key].total_exclusive_call_time < 0.2 + + +@validate_transaction_metrics( + "test_asend_receives_a_value", + background_task=True, + scoped_metrics=[("Function/agen", 1)], + rollup_metrics=[("Function/agen", 1)], +) +def test_asend_receives_a_value(event_loop): + _received = [] + @function_trace(name="agen") + async def agen(): + value = yield + _received.append(value) + yield value + + @background_task(name="test_asend_receives_a_value") + async def _test(): + gen = agen() + + # kickstart the coroutine + await gen.asend(None) + + assert await gen.asend("foobar") == "foobar" + assert _received and _received[0] == "foobar" + + # finish consumption of the coroutine if necessary + async for _ in gen: + pass + + event_loop.run_until_complete(_test()) + + +@validate_transaction_metrics( + "test_athrow_yields_a_value", + background_task=True, + scoped_metrics=[("Function/agen", 1)], + rollup_metrics=[("Function/agen", 1)], +) +def test_athrow_yields_a_value(event_loop): + @function_trace(name="agen") + async def agen(): + for _ in range(2): + try: + yield + except MyException: + yield "foobar" + + @background_task(name="test_athrow_yields_a_value") + async def _test(): + gen = agen() + + # kickstart the coroutine + await gen.asend(None) + + assert await gen.athrow(MyException) == "foobar" + + # finish consumption of the coroutine if necessary + async for _ in gen: + pass + + event_loop.run_until_complete(_test()) + + +@validate_transaction_metrics( + "test_multiple_throws_yield_a_value", + background_task=True, + scoped_metrics=[("Function/agen", 1)], + rollup_metrics=[("Function/agen", 1)], +) +def test_multiple_throws_yield_a_value(event_loop): + @function_trace(name="agen") + async def agen(): + value = None + for _ in range(4): + try: + yield value + value = "bar" + except MyException: + value = "foo" + + + @background_task(name="test_multiple_throws_yield_a_value") + async def _test(): + gen = agen() + + # kickstart the coroutine + assert await gen.asend(None) is None + assert await gen.athrow(MyException) == "foo" + assert await gen.athrow(MyException) == "foo" + assert await gen.asend(None) == "bar" + + # finish consumption of the coroutine if necessary + async for _ in gen: + pass + + event_loop.run_until_complete(_test()) + + +@validate_transaction_metrics( + "test_athrow_does_not_yield_a_value", + background_task=True, + scoped_metrics=[("Function/agen", 1)], + rollup_metrics=[("Function/agen", 1)], +) +def test_athrow_does_not_yield_a_value(event_loop): + @function_trace(name="agen") + async def agen(): + for _ in range(2): + try: + yield + except MyException: + return + + @background_task(name="test_athrow_does_not_yield_a_value") + async def _test(): + gen = agen() + + # kickstart the coroutine + await gen.asend(None) + + # async generator will raise StopAsyncIteration + with pytest.raises(StopAsyncIteration): + await gen.athrow(MyException) + + + event_loop.run_until_complete(_test()) + + +@pytest.mark.parametrize( + "trace", + [ + function_trace(name="simple_gen"), + external_trace(library="lib", url="http://foo.com"), + database_trace("select * from foo"), + datastore_trace("lib", "foo", "bar"), + message_trace("lib", "op", "typ", "name"), + memcache_trace("cmd"), + ], +) +def test_async_generator_functions_outside_of_transaction(event_loop, trace): + @trace + async def agen(): + for _ in range(2): + yield "foo" + + async def _test(): + assert [_ async for _ in agen()] == ["foo", "foo"] + + event_loop.run_until_complete(_test()) + + +@validate_transaction_metrics( + "test_catching_generator_exit_causes_runtime_error", + background_task=True, + scoped_metrics=[("Function/agen", 1)], + rollup_metrics=[("Function/agen", 1)], +) +def test_catching_generator_exit_causes_runtime_error(event_loop): + @function_trace(name="agen") + async def agen(): + try: + yield + except GeneratorExit: + yield + + @background_task(name="test_catching_generator_exit_causes_runtime_error") + async def _test(): + gen = agen() + + # kickstart the coroutine (we're inside the try now) + await gen.asend(None) + + # Generators cannot catch generator exit exceptions (which are injected by + # close). This will result in a runtime error. + with pytest.raises(RuntimeError): + await gen.aclose() + + event_loop.run_until_complete(_test()) + + +@validate_transaction_metrics( + "test_async_generator_time_excludes_creation_time", + background_task=True, + scoped_metrics=[("Function/agen", 1)], + rollup_metrics=[("Function/agen", 1)], +) +def test_async_generator_time_excludes_creation_time(event_loop): + @function_trace(name="agen") + async def agen(): + yield + + metrics = [] + full_metrics = {} + + @capture_transaction_metrics(metrics, full_metrics) + @background_task(name="test_async_generator_time_excludes_creation_time") + def _test_async_generator_time_excludes_creation_time(): + async def _test(): + gen = agen() + time.sleep(0.1) + async for _ in gen: + pass + + event_loop.run_until_complete(_test()) + _test_async_generator_time_excludes_creation_time() + + # check that the trace does not include the time between creation and + # consumption + assert full_metrics[("Function/agen", "")].total_call_time < 0.1 + + +@validate_transaction_metrics( + "test_complete_async_generator", + background_task=True, + scoped_metrics=[("Function/agen", 1)], + rollup_metrics=[("Function/agen", 1)], +) +@background_task(name="test_complete_async_generator") +def test_complete_async_generator(event_loop): + @function_trace(name="agen") + async def agen(): + for i in range(5): + yield i + + async def _test(): + gen = agen() + assert [x async for x in gen] == [x for x in range(5)] + + event_loop.run_until_complete(_test()) + + +@pytest.mark.parametrize("nr_transaction", [True, False]) +def test_incomplete_async_generator(event_loop, nr_transaction): + @function_trace(name="agen") + async def agen(): + for _ in range(5): + yield + + def _test_incomplete_async_generator(): + async def _test(): + c = agen() + + async for _ in c: + break + + if nr_transaction: + _test = background_task(name="test_incomplete_async_generator")(_test) + + event_loop.run_until_complete(_test()) + + if nr_transaction: + _test_incomplete_async_generator = validate_transaction_metrics( + "test_incomplete_async_generator", + background_task=True, + scoped_metrics=[("Function/agen", 1)], + rollup_metrics=[("Function/agen", 1)], + )(_test_incomplete_async_generator) + + _test_incomplete_async_generator() + + +def test_incomplete_async_generator_transaction_exited(event_loop): + @function_trace(name="agen") + async def agen(): + for _ in range(5): + yield + + @validate_transaction_metrics( + "test_incomplete_async_generator", + background_task=True, + scoped_metrics=[("Function/agen", 1)], + rollup_metrics=[("Function/agen", 1)], + ) + def _test_incomplete_async_generator(): + c = agen() + @background_task(name="test_incomplete_async_generator") + async def _test(): + async for _ in c: + break + + event_loop.run_until_complete(_test()) + + # Remove generator after transaction completes + del c + + _test_incomplete_async_generator() diff --git a/tests/agent_features/_test_code_level_metrics.py b/tests/agent_features/_test_code_level_metrics.py index bbe3363f4c..21d193a104 100644 --- a/tests/agent_features/_test_code_level_metrics.py +++ b/tests/agent_features/_test_code_level_metrics.py @@ -18,7 +18,7 @@ def exercise_function(): return -class ExerciseClass(object): +class ExerciseClass(): def exercise_method(self): return @@ -31,7 +31,7 @@ def exercise_class_method(cls): return -class ExerciseClassCallable(object): +class ExerciseClassCallable(): def __call__(self): return diff --git a/tests/agent_features/conftest.py b/tests/agent_features/conftest.py index 57263238be..a8c6c42adb 100644 --- a/tests/agent_features/conftest.py +++ b/tests/agent_features/conftest.py @@ -12,17 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 newrelic_caplog as caplog, ) - -from newrelic.packages import six +from testing_support.fixture.event_loop import ( # noqa: F401; pylint: disable=W0611 + event_loop, +) _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, @@ -30,24 +33,9 @@ "debug.record_transaction_failure": True, "debug.log_autorum_middleware": True, "agent_limits.errors_per_harvest": 100, + "ml_insights_events.enabled": True, } collector_agent_registration = collector_agent_registration_fixture( app_name="Python Agent Test (agent_features)", default_settings=_default_settings ) - - -if six.PY2: - collect_ignore = [ - "test_async_context_propagation.py", - "test_coroutine_trace.py", - "test_coroutine_transaction.py", - "test_async_timing.py", - "test_event_loop_wait_time.py", - "test_asgi_transaction.py", - "test_asgi_browser.py", - "test_asgi_distributed_tracing.py", - "test_asgi_w3c_trace_context.py", - ] -else: - from testing_support.fixture.event_loop import event_loop diff --git a/tests/agent_features/test_apdex_metrics.py b/tests/agent_features/test_apdex_metrics.py index e32a96e312..c150fcf7e6 100644 --- a/tests/agent_features/test_apdex_metrics.py +++ b/tests/agent_features/test_apdex_metrics.py @@ -13,24 +13,41 @@ # limitations under the License. import webtest - -from testing_support.validators.validate_apdex_metrics import ( - validate_apdex_metrics) from testing_support.sample_applications import simple_app +from testing_support.validators.validate_apdex_metrics import validate_apdex_metrics +from newrelic.api.transaction import current_transaction, suppress_apdex_metric +from newrelic.api.wsgi_application import wsgi_application normal_application = webtest.TestApp(simple_app) - # NOTE: This test validates that the server-side apdex_t is set to 0.5 # If the server-side configuration changes, this test will start to fail. @validate_apdex_metrics( - name='', - group='Uri', + name="", + group="Uri", apdex_t_min=0.5, apdex_t_max=0.5, ) def test_apdex(): - normal_application.get('/') + normal_application.get("/") + + +# This has to be a Web Transaction. +# The apdex measurement only applies to Web Transactions +def test_apdex_suppression(): + @wsgi_application() + def simple_apdex_supression_app(environ, start_response): + suppress_apdex_metric() + + start_response(status="200 OK", response_headers=[]) + transaction = current_transaction() + + assert transaction.suppress_apdex + assert transaction.apdex == 0 + return [] + + apdex_suppression_app = webtest.TestApp(simple_apdex_supression_app) + apdex_suppression_app.get("/") diff --git a/tests/agent_features/test_asgi_browser.py b/tests/agent_features/test_asgi_browser.py index 1e718e1e0d..8821cb0bbe 100644 --- a/tests/agent_features/test_asgi_browser.py +++ b/tests/agent_features/test_asgi_browser.py @@ -15,7 +15,6 @@ import json import pytest -import six from bs4 import BeautifulSoup from testing_support.asgi_testing import AsgiTest from testing_support.fixtures import override_application_settings @@ -31,19 +30,18 @@ from newrelic.api.transaction import ( add_custom_attribute, disable_browser_autorum, - get_browser_timing_footer, get_browser_timing_header, ) from newrelic.common.encoding_utils import deobfuscate -_runtime_error_name = RuntimeError.__module__ + ":" + RuntimeError.__name__ +_runtime_error_name = f"{RuntimeError.__module__}:{RuntimeError.__name__}" @asgi_application() async def target_asgi_application_manual_rum(scope, receive, send): - text = "%s

RESPONSE

%s" + text = "%s

RESPONSE

" - output = (text % (get_browser_timing_header(), get_browser_timing_footer())).encode("UTF-8") + output = (text % get_browser_timing_header()).encode("UTF-8") response_headers = [ (b"content-type", b"text/html; charset=utf-8"), @@ -56,15 +54,15 @@ async def target_asgi_application_manual_rum(scope, receive, send): target_application_manual_rum = AsgiTest(target_asgi_application_manual_rum) -_test_footer_attributes = { +_test_header_attributes = { "browser_monitoring.enabled": True, "browser_monitoring.auto_instrument": False, "js_agent_loader": "", } -@override_application_settings(_test_footer_attributes) -def test_footer_attributes(): +@override_application_settings(_test_header_attributes) +def test_header_attributes(): settings = application_settings() assert settings.browser_monitoring.enabled @@ -72,19 +70,18 @@ def test_footer_attributes(): assert settings.browser_key assert settings.browser_monitoring.loader_version assert settings.js_agent_loader - assert isinstance(settings.js_agent_file, six.string_types) + assert isinstance(settings.js_agent_file, str) assert settings.beacon assert settings.error_beacon token = "0123456789ABCDEF" # nosec - headers = {"Cookie": "NRAGENT=tk=%s" % token} + headers = {"Cookie": f"NRAGENT=tk={token}"} response = target_application_manual_rum.get("/", headers=headers) html = BeautifulSoup(response.body, "html.parser") header = html.html.head.script.string content = html.html.body.p.string - footer = html.html.body.script.string # Validate actual body content. @@ -94,10 +91,10 @@ def test_footer_attributes(): assert header.find("NREUM HEADER") != -1 - # Now validate the various fields of the footer. The fields are + # Now validate the various fields of the header. The fields are # held by a JSON dictionary. - data = json.loads(footer.split("NREUM.info=")[1]) + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) assert data["licenseKey"] == settings.browser_key assert data["applicationID"] == settings.application_id @@ -111,8 +108,7 @@ def test_footer_attributes(): obfuscation_key = settings.license_key[:13] - type_transaction_data = unicode if six.PY2 else str # noqa: F821 - assert isinstance(data["transactionName"], type_transaction_data) + assert isinstance(data["transactionName"], str) txn_name = deobfuscate(data["transactionName"], obfuscation_key) @@ -137,8 +133,8 @@ def test_ssl_for_http_is_none(): response = target_application_manual_rum.get("/") html = BeautifulSoup(response.body, "html.parser") - footer = html.html.body.script.string - data = json.loads(footer.split("NREUM.info=")[1]) + header = html.html.head.script.string + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) assert "sslForHttp" not in data @@ -159,8 +155,8 @@ def test_ssl_for_http_is_true(): response = target_application_manual_rum.get("/") html = BeautifulSoup(response.body, "html.parser") - footer = html.html.body.script.string - data = json.loads(footer.split("NREUM.info=")[1]) + header = html.html.head.script.string + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) assert data["sslForHttp"] is True @@ -181,8 +177,8 @@ def test_ssl_for_http_is_false(): response = target_application_manual_rum.get("/") html = BeautifulSoup(response.body, "html.parser") - footer = html.html.body.script.string - data = json.loads(footer.split("NREUM.info=")[1]) + header = html.html.head.script.string + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) assert data["sslForHttp"] is False @@ -219,7 +215,7 @@ def test_html_insertion_yield_single_no_head(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert b"NREUM HEADER" in response.body assert b"NREUM.info" in response.body @@ -259,7 +255,7 @@ def test_html_insertion_yield_multi_no_head(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert b"NREUM HEADER" in response.body assert b"NREUM.info" in response.body @@ -299,7 +295,7 @@ def test_html_insertion_unnamed_attachment_header(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert b"NREUM HEADER" not in response.body assert b"NREUM.info" not in response.body @@ -339,7 +335,7 @@ def test_html_insertion_named_attachment_header(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert b"NREUM HEADER" not in response.body assert b"NREUM.info" not in response.body @@ -379,7 +375,7 @@ def test_html_insertion_inline_attachment_header(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert b"NREUM HEADER" in response.body assert b"NREUM.info" in response.body @@ -414,7 +410,7 @@ def test_html_insertion_empty(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert b"NREUM HEADER" not in response.body assert b"NREUM.info" not in response.body @@ -449,7 +445,7 @@ def test_html_insertion_single_empty_string(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert b"NREUM HEADER" not in response.body assert b"NREUM.info" not in response.body @@ -485,7 +481,7 @@ def test_html_insertion_multiple_empty_string(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert b"NREUM HEADER" not in response.body assert b"NREUM.info" not in response.body @@ -522,7 +518,7 @@ def test_html_insertion_single_large_prelude(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert "content-type" in response.headers assert "content-length" in response.headers @@ -566,7 +562,7 @@ def test_html_insertion_multi_large_prelude(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert "content-type" in response.headers assert "content-length" in response.headers @@ -655,6 +651,19 @@ async def target_asgi_application_invalid_content_length(scope, receive, send): target_application_invalid_content_length = AsgiTest(target_asgi_application_invalid_content_length) + +@asgi_application() +async def target_asgi_application_no_content_length(scope, receive, send): + output = b"

RESPONSE

" + + response_headers = [(b"content-type", b"text/html; charset=utf-8")] + + await send({"type": "http.response.start", "status": 200, "headers": response_headers}) + await send({"type": "http.response.body", "body": output}) + + +target_application_no_content_length = AsgiTest(target_asgi_application_no_content_length) + _test_html_insertion_invalid_content_length_settings = { "browser_monitoring.enabled": True, "browser_monitoring.auto_instrument": True, @@ -676,6 +685,17 @@ def test_html_insertion_invalid_content_length(): assert b"NREUM.info" not in response.body +@override_application_settings(_test_html_insertion_invalid_content_length_settings) +def test_html_insertion_no_content_length(): + response = target_application_no_content_length.get("/") + assert response.status == 200 + + assert "content-type" in response.headers + + assert b"NREUM HEADER" not in response.body + assert b"NREUM.info" not in response.body + + @asgi_application() async def target_asgi_application_content_encoding(scope, receive, send): output = b"

RESPONSE

" @@ -884,7 +904,7 @@ def test_html_insertion_disable_autorum_via_api(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert b"NREUM HEADER" not in response.body assert b"NREUM.info" not in response.body @@ -895,13 +915,9 @@ async def target_asgi_application_manual_rum_insertion(scope, receive, send): output = b"

RESPONSE

" header = get_browser_timing_header() - footer = get_browser_timing_footer() - header = get_browser_timing_header() - footer = get_browser_timing_footer() assert header == "" - assert footer == "" response_headers = [ (b"content-type", b"text/html; charset=utf-8"), @@ -931,7 +947,7 @@ def test_html_insertion_manual_rum_insertion(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert b"NREUM HEADER" not in response.body assert b"NREUM.info" not in response.body diff --git a/tests/agent_features/test_asgi_distributed_tracing.py b/tests/agent_features/test_asgi_distributed_tracing.py index 90f57becc2..3f9e7c0c88 100644 --- a/tests/agent_features/test_asgi_distributed_tracing.py +++ b/tests/agent_features/test_asgi_distributed_tracing.py @@ -12,80 +12,91 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import json + import pytest -import copy +from testing_support.asgi_testing import AsgiTest +from testing_support.fixtures import override_application_settings +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.application import application_instance +from newrelic.api.asgi_application import ASGIWebTransaction, asgi_application from newrelic.api.background_task import BackgroundTask from newrelic.api.transaction import current_transaction -from newrelic.api.asgi_application import asgi_application, ASGIWebTransaction -from testing_support.asgi_testing import AsgiTest -from testing_support.fixtures import override_application_settings -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics - - -distributed_trace_intrinsics = ['guid', 'traceId', 'priority', 'sampled'] -inbound_payload_intrinsics = ['parent.type', 'parent.app', 'parent.account', - 'parent.transportType', 'parent.transportDuration'] +distributed_trace_intrinsics = ["guid", "traceId", "priority", "sampled"] +inbound_payload_intrinsics = [ + "parent.type", + "parent.app", + "parent.account", + "parent.transportType", + "parent.transportDuration", +] payload = { - 'v': [0, 1], - 'd': { - 'ac': '1', - 'ap': '2827902', - 'id': '7d3efb1b173fecfa', - 'pa': '5e5733a911cfbc73', - 'pr': 10.001, - 'sa': True, - 'ti': 1518469636035, - 'tr': 'd6b4ba0c3a712ca', - 'ty': 'App', - } + "v": [0, 1], + "d": { + "ac": "1", + "ap": "2827902", + "id": "7d3efb1b173fecfa", + "pa": "5e5733a911cfbc73", + "pr": 10.001, + "sa": True, + "ti": 1518469636035, + "tr": "d6b4ba0c3a712ca", + "ty": "App", + }, } -parent_order = ['parent_type', 'parent_account', - 'parent_app', 'parent_transport_type'] +parent_order = ["parent_type", "parent_account", "parent_app", "parent_transport_type"] parent_info = { - 'parent_type': payload['d']['ty'], - 'parent_account': payload['d']['ac'], - 'parent_app': payload['d']['ap'], - 'parent_transport_type': 'HTTP' + "parent_type": payload["d"]["ty"], + "parent_account": payload["d"]["ac"], + "parent_app": payload["d"]["ap"], + "parent_transport_type": "HTTP", } @asgi_application() async def target_asgi_application(scope, receive, send): - status = '200 OK' + status = "200 OK" type = "http.response.start" - output = b'hello world' - response_headers = [(b'content-type', b'text/html; charset=utf-8'), - (b'content-length', str(len(output)).encode('utf-8'))] + output = b"hello world" + response_headers = [ + (b"content-type", b"text/html; charset=utf-8"), + (b"content-length", str(len(output)).encode("utf-8")), + ] txn = current_transaction() # Make assertions on the ASGIWebTransaction object assert txn._distributed_trace_state - assert txn.parent_type == 'App' - assert txn.parent_app == '2827902' - assert txn.parent_account == '1' - assert txn.parent_span == '7d3efb1b173fecfa' - assert txn.parent_transport_type == 'HTTP' + assert txn.parent_type == "App" + assert txn.parent_app == "2827902" + assert txn.parent_account == "1" + assert txn.parent_span == "7d3efb1b173fecfa" + assert txn.parent_transport_type == "HTTP" assert isinstance(txn.parent_transport_duration, float) - assert txn._trace_id == 'd6b4ba0c3a712ca' + assert txn._trace_id == "d6b4ba0c3a712ca" assert txn.priority == 10.001 assert txn.sampled - await send({ - "type": type, - "status": status, - "headers": response_headers, - }) - - await send({ - "type": "http.response.body", - "body": b"Hello World", - }) + await send( + { + "type": type, + "status": status, + "headers": response_headers, + } + ) + + await send( + { + "type": "http.response.body", + "body": b"Hello World", + } + ) return [output] @@ -94,41 +105,38 @@ async def target_asgi_application(scope, receive, send): _override_settings = { - 'trusted_account_key': '1', - 'distributed_tracing.enabled': True, + "trusted_account_key": "1", + "distributed_tracing.enabled": True, } _metrics = [ - ('Supportability/DistributedTrace/AcceptPayload/Success', 1), - ('Supportability/TraceContext/Accept/Success', None) + ("Supportability/DistributedTrace/AcceptPayload/Success", 1), + ("Supportability/TraceContext/Accept/Success", None), ] @override_application_settings(_override_settings) -@validate_transaction_metrics( - '', - group='Uri', - rollup_metrics=_metrics) +@validate_transaction_metrics("", group="Uri", rollup_metrics=_metrics) def test_distributed_tracing_web_transaction(): - headers = {'newrelic': json.dumps(payload)} - response = test_application.make_request('GET', '/', headers=headers) - assert 'X-NewRelic-App-Data' not in response.headers + headers = {"newrelic": json.dumps(payload)} + response = test_application.make_request("GET", "/", headers=headers) + assert "X-NewRelic-App-Data" not in response.headers -class TestAsgiRequest(object): +class TestAsgiRequest(): scope = { - 'asgi': {'spec_version': '2.1', 'version': '3.0'}, - 'client': ('127.0.0.1', 54768), - 'headers': [(b'host', b'localhost:8000')], - 'http_version': '1.1', - 'method': 'GET', - 'path': '/', - 'query_string': b'', - 'raw_path': b'/', - 'root_path': '', - 'scheme': 'http', - 'server': ('127.0.0.1', 8000), - 'type': 'http' + "asgi": {"spec_version": "2.1", "version": "3.0"}, + "client": ("127.0.0.1", 54768), + "headers": [(b"host", b"localhost:8000")], + "http_version": "1.1", + "method": "GET", + "path": "/", + "query_string": b"", + "raw_path": b"/", + "root_path": "", + "scheme": "http", + "server": ("127.0.0.1", 8000), + "type": "http", } async def receive(self): @@ -140,9 +148,9 @@ async def send(self, event): # test our distributed_trace metrics by creating a transaction and then forcing # it to process a distributed trace payload -@pytest.mark.parametrize('web_transaction', (True, False)) -@pytest.mark.parametrize('gen_error', (True, False)) -@pytest.mark.parametrize('has_parent', (True, False)) +@pytest.mark.parametrize("web_transaction", (True, False)) +@pytest.mark.parametrize("gen_error", (True, False)) +@pytest.mark.parametrize("has_parent", (True, False)) def test_distributed_tracing_metrics(web_transaction, gen_error, has_parent): def _make_dt_tag(pi): return "%s/%s/%s/%s/all" % tuple(pi[x] for x in parent_order) @@ -150,11 +158,11 @@ def _make_dt_tag(pi): # figure out which metrics we'll see based on the test params # note: we'll always see DurationByCaller if the distributed # tracing flag is turned on - metrics = ['DurationByCaller'] + metrics = ["DurationByCaller"] if gen_error: - metrics.append('ErrorsByCaller') + metrics.append("ErrorsByCaller") if has_parent: - metrics.append('TransportDuration') + metrics.append("TransportDuration") tag = None dt_payload = copy.deepcopy(payload) @@ -164,15 +172,13 @@ def _make_dt_tag(pi): if has_parent: tag = _make_dt_tag(parent_info) else: - tag = _make_dt_tag(dict((x, 'Unknown') for x in parent_info.keys())) - del dt_payload['d']['tr'] + tag = _make_dt_tag(dict((x, "Unknown") for x in parent_info.keys())) + del dt_payload["d"]["tr"] # now run the test - transaction_name = "test_dt_metrics_%s" % '_'.join(metrics) + transaction_name = f"test_dt_metrics_{'_'.join(metrics)}" _rollup_metrics = [ - ("%s/%s%s" % (x, tag, bt), 1) - for x in metrics - for bt in ['', 'Web' if web_transaction else 'Other'] + (f"{x}/{tag}{bt}", 1) for x in metrics for bt in ["", "Web" if web_transaction else "Other"] ] def _make_test_transaction(): @@ -182,16 +188,14 @@ def _make_test_transaction(): if not web_transaction: return BackgroundTask(application, transaction_name) - tn = ASGIWebTransaction(application, request.scope, - request.send, request.receive) + tn = ASGIWebTransaction(application, request.scope, request.send, request.receive) tn.set_transaction_name(transaction_name) return tn @override_application_settings(_override_settings) @validate_transaction_metrics( - transaction_name, - background_task=not(web_transaction), - rollup_metrics=_rollup_metrics) + transaction_name, background_task=not (web_transaction), rollup_metrics=_rollup_metrics + ) def _test(): with _make_test_transaction() as transaction: transaction.accept_distributed_trace_payload(dt_payload) diff --git a/tests/agent_features/test_asgi_w3c_trace_context.py b/tests/agent_features/test_asgi_w3c_trace_context.py index 8cec2eb7a1..e090a52815 100644 --- a/tests/agent_features/test_asgi_w3c_trace_context.py +++ b/tests/agent_features/test_asgi_w3c_trace_context.py @@ -78,7 +78,7 @@ async def target_asgi_application(scope, receive, send): INBOUND_TRACESTATE = \ 'rojo=f06a0ba902b7,congo=t61rcWkgMzE' LONG_TRACESTATE = \ - ','.join(["{}@rojo=f06a0ba902b7".format(x) for x in range(32)]) + ','.join([f"{x}@rojo=f06a0ba902b7" for x in range(32)]) INBOUND_UNTRUSTED_NR_TRACESTATE = \ ('2@nr=0-0-1345936-55632452-27jjj2d8890283b4-b28ce285632jjhl9-' '1-1.1273-1569367663277') @@ -155,10 +155,10 @@ def _test(): @pytest.mark.parametrize('inbound_tracestate,expected', ( ('', None), - (INBOUND_NR_TRACESTATE + "," + INBOUND_TRACESTATE, INBOUND_TRACESTATE), + (f"{INBOUND_NR_TRACESTATE},{INBOUND_TRACESTATE}", INBOUND_TRACESTATE), (INBOUND_TRACESTATE, INBOUND_TRACESTATE), - (LONG_TRACESTATE + ',' + INBOUND_NR_TRACESTATE, - ','.join("{}@rojo=f06a0ba902b7".format(x) for x in range(31))), + (f"{LONG_TRACESTATE},{INBOUND_NR_TRACESTATE}", + ','.join(f"{x}@rojo=f06a0ba902b7" for x in range(31))), ), ids=( 'empty_inbound_payload', 'nr_payload', @@ -231,7 +231,7 @@ def _test(): "parentSpanId": "00f067aa0ba902b7", "parent.transportType": "HTTP"}, [("Supportability/TraceContext/TraceParent/Accept/Success", 1)]), - (INBOUND_TRACEPARENT + ' ', { + (f"{INBOUND_TRACEPARENT} ", { "traceId": "0af7651916cd43dd8448eb211c80319c", "parentSpanId": "00f067aa0ba902b7", "parent.transportType": "HTTP"}, @@ -283,16 +283,16 @@ def _test(): (INBOUND_NR_TRACESTATE, {'trustedParentId': '27ddd2d8890283b4'}), ('garbage', {'parentId': '00f067aa0ba902b7'}), - (INBOUND_TRACESTATE + ',' + INBOUND_NR_TRACESTATE, + (f"{INBOUND_TRACESTATE},{INBOUND_NR_TRACESTATE}", {'parentId': '00f067aa0ba902b7', 'trustedParentId': '27ddd2d8890283b4', 'tracingVendors': 'rojo,congo'}), - (INBOUND_TRACESTATE + ',' + INBOUND_UNTRUSTED_NR_TRACESTATE, + (f"{INBOUND_TRACESTATE},{INBOUND_UNTRUSTED_NR_TRACESTATE}", {'parentId': '00f067aa0ba902b7', 'tracingVendors': 'rojo,congo,2@nr'}), - ('rojo=12345,' + 'v' * 257 + '=x', + (f"rojo=12345,{'v' * 257}=x", {'tracingVendors': 'rojo'}), - ('rojo=12345,k=' + 'v' * 257, + (f"rojo=12345,k={'v' * 257}", {'tracingVendors': 'rojo'}), )) @override_application_settings(_override_settings) diff --git a/tests/agent_features/test_async_generator_trace.py b/tests/agent_features/test_async_generator_trace.py new file mode 100644 index 0000000000..208cf1588a --- /dev/null +++ b/tests/agent_features/test_async_generator_trace.py @@ -0,0 +1,19 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys + +# Async Generators were introduced in Python 3.6, but some APIs weren't completely stable until Python 3.7. +if sys.version_info >= (3, 7): + from _test_async_generator_trace import * # NOQA diff --git a/tests/agent_features/test_async_wrapper_detection.py b/tests/agent_features/test_async_wrapper_detection.py new file mode 100644 index 0000000000..bb1fd3f1e3 --- /dev/null +++ b/tests/agent_features/test_async_wrapper_detection.py @@ -0,0 +1,102 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +import functools +import time + +from newrelic.api.background_task import background_task +from newrelic.api.database_trace import database_trace +from newrelic.api.datastore_trace import datastore_trace +from newrelic.api.external_trace import external_trace +from newrelic.api.function_trace import function_trace +from newrelic.api.graphql_trace import graphql_operation_trace, graphql_resolver_trace +from newrelic.api.memcache_trace import memcache_trace +from newrelic.api.message_trace import message_trace + +from newrelic.common.async_wrapper import generator_wrapper + +from testing_support.fixtures import capture_transaction_metrics +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +trace_metric_cases = [ + (functools.partial(function_trace, name="simple_gen"), "Function/simple_gen"), + (functools.partial(external_trace, library="lib", url="http://foo.com"), "External/foo.com/lib/"), + (functools.partial(database_trace, "select * from foo"), "Datastore/statement/None/foo/select"), + (functools.partial(datastore_trace, "lib", "foo", "bar"), "Datastore/statement/lib/foo/bar"), + (functools.partial(message_trace, "lib", "op", "typ", "name"), "MessageBroker/lib/typ/op/Named/name"), + (functools.partial(memcache_trace, "cmd"), "Memcache/cmd"), + (functools.partial(graphql_operation_trace), "GraphQL/operation/GraphQL///"), + (functools.partial(graphql_resolver_trace), "GraphQL/resolve/GraphQL/"), +] + + +@pytest.mark.parametrize("trace,metric", trace_metric_cases) +def test_automatic_generator_trace_wrapper(trace, metric): + metrics = [] + full_metrics = {} + + @capture_transaction_metrics(metrics, full_metrics) + @validate_transaction_metrics( + "test_automatic_generator_trace_wrapper", background_task=True, scoped_metrics=[(metric, 1)], rollup_metrics=[(metric, 1)] + ) + @background_task(name="test_automatic_generator_trace_wrapper") + def _test(): + @trace() + def gen(): + time.sleep(0.1) + yield + time.sleep(0.1) + + for _ in gen(): + pass + + _test() + + # Check that generators time the total call time (including pauses) + metric_key = (metric, "") + assert full_metrics[metric_key].total_call_time >= 0.2 + + +@pytest.mark.parametrize("trace,metric", trace_metric_cases) +def test_manual_generator_trace_wrapper(trace, metric): + metrics = [] + full_metrics = {} + + @capture_transaction_metrics(metrics, full_metrics) + @validate_transaction_metrics( + "test_automatic_generator_trace_wrapper", background_task=True, scoped_metrics=[(metric, 1)], rollup_metrics=[(metric, 1)] + ) + @background_task(name="test_automatic_generator_trace_wrapper") + def _test(): + @trace(async_wrapper=generator_wrapper) + def wrapper_func(): + """Function that returns a generator object, obscuring the automatic introspection of async_wrapper()""" + def gen(): + time.sleep(0.1) + yield + time.sleep(0.1) + return gen() + + for _ in wrapper_func(): + pass + + _test() + + # Check that generators time the total call time (including pauses) + metric_key = (metric, "") + assert full_metrics[metric_key].total_call_time >= 0.2 diff --git a/tests/agent_features/test_attribute.py b/tests/agent_features/test_attribute.py index f4b9e896fe..3f4a99d385 100644 --- a/tests/agent_features/test_attribute.py +++ b/tests/agent_features/test_attribute.py @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - import pytest import webtest from testing_support.fixtures import ( @@ -38,12 +36,7 @@ sanitize, truncate, ) -from newrelic.packages import six - -# Python 3 lacks longs -if sys.version_info >= (3, 0): - long = int try: from newrelic.core._thread_utilization import ThreadUtilization except ImportError: @@ -192,60 +185,57 @@ def test_display_host_custom(): def test_truncate_string(): s = "blahblah" result = truncate(s, maxsize=4) - assert isinstance(result, six.string_types) + assert isinstance(result, str) assert result == "blah" def test_truncate_bytes(): b = b"foobar" result = truncate(b, maxsize=3) - assert isinstance(result, six.binary_type) + assert isinstance(result, bytes) assert result == b"foo" def test_truncate_unicode_snowman(): # '\u2603' is 'SNOWMAN' - # decode("unicode-escape") is used to get Py2 unicode - u = "snow\u2603".decode("unicode-escape") if six.PY2 else "snow\u2603" + u = "snow\u2603" assert u.encode("utf-8") == b"snow\xe2\x98\x83" result = truncate(u, maxsize=5) - assert isinstance(result, six.text_type) + assert isinstance(result, str) assert result == "snow" def test_truncate_combining_characters(): # '\u0308' is 'COMBINING DIAERESIS' (AKA 'umlaut') - # decode("unicode-escape") is used to get Py2 unicode - u = "Zoe\u0308".decode("unicode-escape") if six.PY2 else "Zoe\u0308" + u = "Zoe\u0308" assert u.encode("utf-8") == b"Zoe\xcc\x88" # truncate will chop off 'COMBINING DIAERESIS', which leaves # 'LATIN SMALL LETTER E' by itself. result = truncate(u, maxsize=3) - assert isinstance(result, six.text_type) + assert isinstance(result, str) assert result == "Zoe" def test_truncate_empty_string(): s = "" result = truncate(s, maxsize=4) - assert isinstance(result, six.string_types) + assert isinstance(result, str) assert result == "" def test_truncate_empty_bytes(): b = b"" result = truncate(b, maxsize=3) - assert isinstance(result, six.binary_type) + assert isinstance(result, bytes) assert result == b"" def test_truncate_empty_unicode(): - # decode("unicode-escape") is used to get Py2 unicode - u = "".decode("unicode-escape") if six.PY2 else "" + u = "" result = truncate(u, maxsize=5) - assert isinstance(result, six.text_type) + assert isinstance(result, str) assert result == "" @@ -317,7 +307,7 @@ def test_custom_params_value_too_long(): @background_task() def test_custom_param_too_many(): for i in range(129): - result = add_custom_attribute("key-%02d" % i, "value") + result = add_custom_attribute(f"key-{i:02}", "value") if i < 128: assert result else: @@ -327,7 +317,7 @@ def test_custom_param_too_many(): @validate_custom_parameters(_required_custom_params_too_many, _forgone_custom_params_too_many) @background_task() def test_custom_params_too_many(): - item_list = [("key-%02d" % i, "value") for i in range(129)] + item_list = [(f"key-{i:02}", "value") for i in range(129)] result = add_custom_attributes(item_list) assert not result @@ -371,9 +361,9 @@ def test_custom_params_int_too_big(): OK_KEY = "*" * (255 - len("request.parameters.")) -OK_REQUEST_PARAM = "request.parameters." + OK_KEY +OK_REQUEST_PARAM = f"request.parameters.{OK_KEY}" TOO_LONG_KEY = "*" * (256 - len("request.parameters.")) -TOO_LONG_REQUEST_PARAM = "request.parameters." + TOO_LONG_KEY +TOO_LONG_REQUEST_PARAM = f"request.parameters.{TOO_LONG_KEY}" assert len(OK_REQUEST_PARAM) == 255 assert len(TOO_LONG_REQUEST_PARAM) == 256 @@ -385,7 +375,7 @@ def test_custom_params_int_too_big(): @validate_attributes("agent", _required_request_key_ok, _forgone_request_key_ok) def test_capture_request_params_key_ok(): target_application = webtest.TestApp(target_wsgi_application) - response = target_application.get("/?%s=bar" % OK_KEY) + response = target_application.get(f"/?{OK_KEY}=bar") assert response.body == b"Hello World!" @@ -396,7 +386,7 @@ def test_capture_request_params_key_ok(): @validate_attributes("agent", _required_request_key_too_long, _forgone_request_key_too_long) def test_capture_request_params_key_too_long(): target_application = webtest.TestApp(target_wsgi_application) - response = target_application.get("/?%s=bar" % TOO_LONG_KEY) + response = target_application.get(f"/?{TOO_LONG_KEY}=bar") assert response.body == b"Hello World!" @@ -407,7 +397,7 @@ def test_capture_request_params_key_too_long(): @validate_attributes("agent", _required_request_value_too_long, _forgone_request_value_too_long) def test_capture_request_params_value_too_long(): target_application = webtest.TestApp(target_wsgi_application) - response = target_application.get("/?foo=%s" % TOO_LONG) + response = target_application.get(f"/?foo={TOO_LONG}") assert response.body == b"Hello World!" @@ -418,17 +408,17 @@ def test_capture_request_params_value_too_long(): # Types are only defined in the spec for agent attributes, not intrinsics. agent_attributes = { - "request.headers.accept": six.string_types, + "request.headers.accept": str, "request.headers.contentLength": int, - "request.headers.contentType": six.string_types, - "request.headers.host": six.string_types, - "request.headers.referer": six.string_types, - "request.headers.userAgent": six.string_types, - "request.method": six.string_types, - "request.parameters.test": six.string_types, + "request.headers.contentType": str, + "request.headers.host": str, + "request.headers.referer": str, + "request.headers.userAgent": str, + "request.method": str, + "request.parameters.test": str, "response.headers.contentLength": int, - "response.headers.contentType": six.string_types, - "response.status": six.string_types, + "response.headers.contentType": str, + "response.status": str, } @@ -474,11 +464,6 @@ def test_sanitize_int(): assert sanitize(9876) == 9876 -def test_sanitize_long(): - long_int = long(123456) - assert sanitize(long_int) == long_int - - def test_sanitize_dict(): d = {1: "foo"} assert sanitize(d) == "{1: 'foo'}" @@ -494,7 +479,7 @@ def test_sanitize_tuple(): assert sanitize(t) == "('one', 'two', 'three')" -class Foo(object): +class Foo(): pass @@ -503,7 +488,7 @@ def test_sanitize_object(): assert sanitize(f) == str(f) -class TypeErrorString(object): +class TypeErrorString(): def __str__(self): return 42 @@ -513,7 +498,7 @@ def test_str_raises_type_error(): sanitize(TypeErrorString()) -class AttributeErrorString(object): +class AttributeErrorString(): def __str__(self): raise AttributeError() diff --git a/tests/agent_features/test_attributes_in_action.py b/tests/agent_features/test_attributes_in_action.py index e56994d0a1..b2a129c174 100644 --- a/tests/agent_features/test_attributes_in_action.py +++ b/tests/agent_features/test_attributes_in_action.py @@ -20,14 +20,22 @@ override_application_settings, reset_core_stats_engine, validate_attributes, +) +from testing_support.validators.validate_browser_attributes import ( validate_browser_attributes, +) +from testing_support.validators.validate_error_event_attributes import ( validate_error_event_attributes, +) +from testing_support.validators.validate_error_event_attributes_outside_transaction import ( validate_error_event_attributes_outside_transaction, - validate_error_trace_attributes_outside_transaction, ) from testing_support.validators.validate_error_trace_attributes import ( validate_error_trace_attributes, ) +from testing_support.validators.validate_error_trace_attributes_outside_transaction import ( + validate_error_trace_attributes_outside_transaction, +) from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_error_trace_attributes import ( validate_transaction_error_trace_attributes, @@ -43,7 +51,7 @@ from newrelic.api.background_task import background_task from newrelic.api.message_transaction import message_transaction from newrelic.api.time_trace import notice_error -from newrelic.api.transaction import add_custom_attribute, current_transaction, set_user_id +from newrelic.api.transaction import add_custom_attribute, set_user_id from newrelic.api.wsgi_application import wsgi_application from newrelic.common.object_names import callable_name @@ -56,7 +64,7 @@ URL_PARAM = "some_key" URL_PARAM2 = "second_key" -REQUEST_URL = "/?" + URL_PARAM + "=someval&" + URL_PARAM2 + "=anotherval" +REQUEST_URL = f"/?{URL_PARAM}=someval&{URL_PARAM2}=anotherval" REQUEST_HEADERS = [ ("Accept", "*/*"), ("Host", "foobar"), @@ -65,7 +73,7 @@ ("Content-Length", "10"), ] -REQ_PARAMS = ["request.parameters." + URL_PARAM, "request.parameters." + URL_PARAM2] +REQ_PARAMS = [f"request.parameters.{URL_PARAM}", f"request.parameters.{URL_PARAM2}"] DISTRIBUTED_TRACE_ATTRS = [ "traceId", "priority", @@ -75,7 +83,6 @@ "parent.transportType", "parent.transportDuration", "parentId", - "guid", "sampled", "parentSpanId", ] @@ -361,22 +368,22 @@ def test_browser_include_request_params(normal_application): _override_settings = { "error_collector.attributes.include": ["request.parameters.*"], - "error_collector.attributes.exclude": ["request.parameters." + URL_PARAM2], + "error_collector.attributes.exclude": [f"request.parameters.{URL_PARAM2}"], } _expected_attributes = { - "agent": TRACE_ERROR_AGENT_KEYS + ["request.parameters." + URL_PARAM], + "agent": TRACE_ERROR_AGENT_KEYS + [f"request.parameters.{URL_PARAM}"], "user": ERROR_USER_ATTRS, "intrinsic": ["trip_id"], } _expected_attributes_event = { - "agent": TRACE_ERROR_AGENT_KEYS + ["request.parameters." + URL_PARAM], + "agent": TRACE_ERROR_AGENT_KEYS + [f"request.parameters.{URL_PARAM}"], "user": ERROR_USER_ATTRS, "intrinsic": ERROR_EVENT_INTRINSICS, } -_expected_absent_attributes = {"agent": ["request.parameters." + URL_PARAM2], "user": [], "intrinsic": []} +_expected_absent_attributes = {"agent": [f"request.parameters.{URL_PARAM2}"], "user": [], "intrinsic": []} @validate_error_event_attributes(_expected_attributes_event, _expected_absent_attributes) @@ -388,11 +395,11 @@ def test_error_in_transaction_include_exclude(normal_application): _override_settings = { "transaction_tracer.attributes.include": ["request.parameters.*"], - "transaction_tracer.attributes.exclude": ["request.parameters." + URL_PARAM2], + "transaction_tracer.attributes.exclude": [f"request.parameters.{URL_PARAM2}"], } _expected_attributes = { - "agent": TRACE_ERROR_AGENT_KEYS + ["request.parameters." + URL_PARAM], + "agent": TRACE_ERROR_AGENT_KEYS + [f"request.parameters.{URL_PARAM}"], "user": USER_ATTRS, "intrinsic": ["trip_id"], } @@ -406,16 +413,16 @@ def test_transaction_trace_include_exclude(normal_application): _override_settings = { "transaction_events.attributes.include": ["request.parameters.*"], - "transaction_events.attributes.exclude": ["request.parameters." + URL_PARAM2], + "transaction_events.attributes.exclude": [f"request.parameters.{URL_PARAM2}"], } _expected_attributes = { - "agent": TRANS_EVENT_AGENT_KEYS + ["request.parameters." + URL_PARAM], + "agent": TRANS_EVENT_AGENT_KEYS + [f"request.parameters.{URL_PARAM}"], "user": USER_ATTRS, "intrinsic": TRANS_EVENT_INTRINSICS, } -_expected_absent_attributes = {"agent": ["request.parameters." + URL_PARAM2], "user": [], "intrinsic": []} +_expected_absent_attributes = {"agent": [f"request.parameters.{URL_PARAM2}"], "user": [], "intrinsic": []} @validate_transaction_event_attributes(_expected_attributes, _expected_absent_attributes) @@ -427,16 +434,16 @@ def test_transaction_event_include_exclude(normal_application): _override_settings = { "browser_monitoring.attributes.enabled": True, "browser_monitoring.attributes.include": ["request.parameters.*"], - "browser_monitoring.attributes.exclude": ["request.parameters." + URL_PARAM2], + "browser_monitoring.attributes.exclude": [f"request.parameters.{URL_PARAM2}"], } _expected_attributes = { - "agent": ["request.parameters." + URL_PARAM], + "agent": [f"request.parameters.{URL_PARAM}"], "user": USER_ATTRS, "intrinsic": BROWSER_INTRINSIC_KEYS, } -_expected_absent_attributes = {"agent": ABSENT_BROWSER_KEYS + ["request.parameters." + URL_PARAM2], "user": []} +_expected_absent_attributes = {"agent": ABSENT_BROWSER_KEYS + [f"request.parameters.{URL_PARAM2}"], "user": []} @validate_browser_attributes(_expected_attributes, _expected_absent_attributes) @@ -930,16 +937,21 @@ def test_none_type_routing_key_agent_attribute(): _forgone_agent_attributes = [] -@pytest.mark.parametrize('input_user_id, reported_user_id, high_security',( +@pytest.mark.parametrize( + "input_user_id, reported_user_id, high_security", + ( ("1234", "1234", True), - ("a" * 260, "a" * 255, False), -)) + ("a" * 260, "a" * 255, False), + ), +) def test_enduser_id_attribute_api_valid_types(input_user_id, reported_user_id, high_security): @reset_core_stats_engine() @validate_error_trace_attributes( callable_name(ValueError), exact_attrs={"user": {}, "intrinsic": {}, "agent": {"enduser.id": reported_user_id}} ) - @validate_error_event_attributes(exact_attrs={"user": {}, "intrinsic": {}, "agent": {"enduser.id": reported_user_id}}) + @validate_error_event_attributes( + exact_attrs={"user": {}, "intrinsic": {}, "agent": {"enduser.id": reported_user_id}} + ) @validate_attributes("agent", _required_agent_attributes, _forgone_agent_attributes) @background_task() @override_application_settings({"high_security": high_security}) @@ -950,10 +962,11 @@ def _test(): raise ValueError() except Exception: notice_error() + _test() -@pytest.mark.parametrize('input_user_id',(None, '', 123)) +@pytest.mark.parametrize("input_user_id", (None, "", 123)) def test_enduser_id_attribute_api_invalid_types(input_user_id): @reset_core_stats_engine() @validate_attributes("agent", [], ["enduser.id"]) @@ -965,4 +978,5 @@ def _test(): raise ValueError() except Exception: notice_error() + _test() diff --git a/tests/agent_features/test_browser.py b/tests/agent_features/test_browser.py index e0f562d1e8..5781109931 100644 --- a/tests/agent_features/test_browser.py +++ b/tests/agent_features/test_browser.py @@ -13,9 +13,9 @@ # limitations under the License. import json +import re import sys -import six import webtest from testing_support.fixtures import override_application_settings from testing_support.validators.validate_custom_parameters import ( @@ -29,22 +29,22 @@ from newrelic.api.transaction import ( add_custom_attribute, disable_browser_autorum, - get_browser_timing_footer, get_browser_timing_header, ) +from newrelic.api.web_transaction import web_transaction from newrelic.api.wsgi_application import wsgi_application from newrelic.common.encoding_utils import deobfuscate -_runtime_error_name = RuntimeError.__module__ + ":" + RuntimeError.__name__ +_runtime_error_name = f"{RuntimeError.__module__}:{RuntimeError.__name__}" @wsgi_application() def target_wsgi_application_manual_rum(environ, start_response): status = "200 OK" - text = "%s

RESPONSE

%s" + text = "%s

RESPONSE

" - output = (text % (get_browser_timing_header(), get_browser_timing_footer())).encode("UTF-8") + output = (text % get_browser_timing_header()).encode("UTF-8") response_headers = [("Content-Type", "text/html; charset=utf-8"), ("Content-Length", str(len(output)))] start_response(status, response_headers) @@ -54,15 +54,15 @@ def target_wsgi_application_manual_rum(environ, start_response): target_application_manual_rum = webtest.TestApp(target_wsgi_application_manual_rum) -_test_footer_attributes = { +_test_header_attributes = { "browser_monitoring.enabled": True, "browser_monitoring.auto_instrument": False, "js_agent_loader": "", } -@override_application_settings(_test_footer_attributes) -def test_footer_attributes(): +@override_application_settings(_test_header_attributes) +def test_header_attributes(): settings = application_settings() assert settings.browser_monitoring.enabled @@ -70,18 +70,17 @@ def test_footer_attributes(): assert settings.browser_key assert settings.browser_monitoring.loader_version assert settings.js_agent_loader - assert isinstance(settings.js_agent_file, six.string_types) + assert isinstance(settings.js_agent_file, str) assert settings.beacon assert settings.error_beacon token = "0123456789ABCDEF" # nosec - headers = {"Cookie": "NRAGENT=tk=%s" % token} + headers = {"Cookie": f"NRAGENT=tk={token}"} response = target_application_manual_rum.get("/", headers=headers) header = response.html.html.head.script.string content = response.html.html.body.p.string - footer = response.html.html.body.script.string # Validate actual body content. @@ -91,10 +90,10 @@ def test_footer_attributes(): assert header.find("NREUM HEADER") != -1 - # Now validate the various fields of the footer. The fields are + # Now validate the various fields of the header. The fields are # held by a JSON dictionary. - data = json.loads(footer.split("NREUM.info=")[1]) + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) assert data["licenseKey"] == settings.browser_key assert data["applicationID"] == settings.application_id @@ -108,8 +107,7 @@ def test_footer_attributes(): obfuscation_key = settings.license_key[:13] - type_transaction_data = unicode if six.PY2 else str # noqa: F821 - assert isinstance(data["transactionName"], type_transaction_data) + assert isinstance(data["transactionName"], str) txn_name = deobfuscate(data["transactionName"], obfuscation_key) @@ -133,8 +131,8 @@ def test_ssl_for_http_is_none(): assert settings.browser_monitoring.ssl_for_http is None response = target_application_manual_rum.get("/") - footer = response.html.html.body.script.string - data = json.loads(footer.split("NREUM.info=")[1]) + header = response.html.html.head.script.string + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) assert "sslForHttp" not in data @@ -154,8 +152,8 @@ def test_ssl_for_http_is_true(): assert settings.browser_monitoring.ssl_for_http is True response = target_application_manual_rum.get("/") - footer = response.html.html.body.script.string - data = json.loads(footer.split("NREUM.info=")[1]) + header = response.html.html.head.script.string + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) assert data["sslForHttp"] is True @@ -175,8 +173,8 @@ def test_ssl_for_http_is_false(): assert settings.browser_monitoring.ssl_for_http is False response = target_application_manual_rum.get("/") - footer = response.html.html.body.script.string - data = json.loads(footer.split("NREUM.info=")[1]) + header = response.html.html.head.script.string + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) assert data["sslForHttp"] is False @@ -211,7 +209,7 @@ def test_html_insertion_yield_single_no_head(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. response.mustcontain("NREUM HEADER", "NREUM.info") @@ -247,7 +245,7 @@ def test_html_insertion_yield_multi_no_head(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. response.mustcontain("NREUM HEADER", "NREUM.info") @@ -287,7 +285,7 @@ def test_html_insertion_unnamed_attachment_header(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. response.mustcontain(no=["NREUM HEADER", "NREUM.info"]) @@ -327,7 +325,7 @@ def test_html_insertion_named_attachment_header(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. response.mustcontain(no=["NREUM HEADER", "NREUM.info"]) @@ -367,7 +365,7 @@ def test_html_insertion_inline_attachment_header(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. response.mustcontain("NREUM HEADER", "NREUM.info") @@ -400,7 +398,7 @@ def test_html_insertion_empty_list(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. response.mustcontain(no=["NREUM HEADER", "NREUM.info"]) @@ -435,7 +433,7 @@ def test_html_insertion_single_empty_string(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. response.mustcontain(no=["NREUM HEADER", "NREUM.info"]) @@ -470,7 +468,7 @@ def test_html_insertion_multiple_empty_string(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. response.mustcontain(no=["NREUM HEADER", "NREUM.info"]) @@ -504,7 +502,7 @@ def test_html_insertion_single_large_prelude(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert "Content-Type" in response.headers assert "Content-Length" in response.headers @@ -543,7 +541,7 @@ def test_html_insertion_multi_large_prelude(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert "Content-Type" in response.headers assert "Content-Length" in response.headers @@ -588,7 +586,7 @@ def test_html_insertion_yield_before_start(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. response.mustcontain("NREUM HEADER", "NREUM.info") @@ -626,7 +624,7 @@ def test_html_insertion_start_yield_start(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. assert "Content-Type" in response.headers assert "Content-Length" in response.headers @@ -979,7 +977,7 @@ def test_html_insertion_disable_autorum_via_api(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. response.mustcontain(no=["NREUM HEADER", "NREUM.info"]) @@ -991,13 +989,9 @@ def target_wsgi_application_manual_rum_insertion(environ, start_response): output = b"

RESPONSE

" header = get_browser_timing_header() - footer = get_browser_timing_footer() - header = get_browser_timing_header() - footer = get_browser_timing_footer() assert header == "" - assert footer == "" response_headers = [("Content-Type", "text/html; charset=utf-8"), ("Content-Length", str(len(output)))] start_response(status, response_headers) @@ -1023,6 +1017,42 @@ def test_html_insertion_manual_rum_insertion(): # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. response.mustcontain(no=["NREUM HEADER", "NREUM.info"]) + + +_test_get_browser_timing_snippet_with_nonces = { + "browser_monitoring.enabled": True, + "browser_monitoring.auto_instrument": False, + "js_agent_loader": "", +} +_test_get_browser_timing_snippet_with_nonces_rum_info_re = re.compile(r"NREUM\.info={[^}]*}") + + +@override_application_settings(_test_get_browser_timing_snippet_with_nonces) +@web_transaction( + scheme="http", host="127.0.0.1", port=80, request_method="GET", request_path="/", query_string=None, headers={} +) +def test_get_browser_timing_snippet_with_nonces(): + header = get_browser_timing_header("NONCE") + + header = _test_get_browser_timing_snippet_with_nonces_rum_info_re.sub("NREUM.info={}", header) + assert ( + header + == '' + ) + + +@override_application_settings(_test_get_browser_timing_snippet_with_nonces) +@web_transaction( + scheme="http", host="127.0.0.1", port=80, request_method="GET", request_path="/", query_string=None, headers={} +) +def test_get_browser_timing_snippet_without_nonces(): + header = get_browser_timing_header() + + header = _test_get_browser_timing_snippet_with_nonces_rum_info_re.sub("NREUM.info={}", header) + assert ( + header + == '' + ) diff --git a/tests/agent_features/test_browser_middleware.py b/tests/agent_features/test_browser_middleware.py index 8bd11ea485..e08a87c29b 100644 --- a/tests/agent_features/test_browser_middleware.py +++ b/tests/agent_features/test_browser_middleware.py @@ -13,97 +13,116 @@ # limitations under the License. import pytest -import six import webtest +from testing_support.fixtures import ( + capture_transaction_metrics, + override_application_settings, +) from newrelic.api.wsgi_application import wsgi_application -from testing_support.fixtures import (override_application_settings, - capture_transaction_metrics) - -PAGE_CONTENTS = b'Hello World' +PAGE_CONTENTS = b"Hello World" _browser_enabled_settings = { - 'browser_monitoring.enabled': True, + "browser_monitoring.enabled": True, } _browser_disabled_settings = { - 'browser_monitoring.enabled': False, + "browser_monitoring.enabled": False, } + @wsgi_application() def _app_list(environ, start_response): - status = '200 OK' - response_headers = [('Content-type', 'text/plain')] + status = "200 OK" + response_headers = [("Content-type", "text/plain")] start_response(status, response_headers) return [PAGE_CONTENTS] + + target_application_list = webtest.TestApp(_app_list) + @wsgi_application() def _app_iter(environ, start_response): - status = '200 OK' - response_headers = [('Content-type', 'text/plain')] + status = "200 OK" + response_headers = [("Content-type", "text/plain")] start_response(status, response_headers) yield PAGE_CONTENTS + + target_application_iter = webtest.TestApp(_app_iter) + @wsgi_application() def _app_str(environ, start_response): - status = '200 OK' - response_headers = [('Content-type', 'text/plain')] + status = "200 OK" + response_headers = [("Content-type", "text/plain")] start_response(status, response_headers) return PAGE_CONTENTS + + target_application_str = webtest.TestApp(_app_str) + @wsgi_application() def _app_list_exc_1(environ, start_response): - status = '200 OK' - response_headers = [('Content-type', 'text/plain')] + status = "200 OK" + response_headers = [("Content-type", "text/plain")] start_response(status, response_headers) - 1/0 + 1 / 0 return [PAGE_CONTENTS] + + target_application_list_exc_1 = webtest.TestApp(_app_list_exc_1) + @wsgi_application() def _app_list_exc_2(environ, start_response): - status = '200 OK' - response_headers = [('Content-type', 'text/plain')] - 1/0 + status = "200 OK" + response_headers = [("Content-type", "text/plain")] + 1 / 0 start_response(status, response_headers) return [PAGE_CONTENTS] + + target_application_list_exc_2 = webtest.TestApp(_app_list_exc_2) + @wsgi_application() def _app_iter_exc_1(environ, start_response): - status = '200 OK' - response_headers = [('Content-type', 'text/plain')] + status = "200 OK" + response_headers = [("Content-type", "text/plain")] start_response(status, response_headers) - 1/0 + 1 / 0 yield PAGE_CONTENTS + + target_application_iter_exc_1 = webtest.TestApp(_app_iter_exc_1) + @wsgi_application() def _app_iter_exc_2(environ, start_response): - status = '200 OK' - response_headers = [('Content-type', 'text/plain')] - 1/0 + status = "200 OK" + response_headers = [("Content-type", "text/plain")] + 1 / 0 start_response(status, response_headers) yield PAGE_CONTENTS + + target_application_iter_exc_2 = webtest.TestApp(_app_iter_exc_2) _target_applications = [ target_application_list, target_application_iter, - pytest.param(target_application_str, marks=pytest.mark.skipif( - six.PY3, reason='PY3 webtest expects type(byte) ' - 'so this test doesnt apply')), target_application_list_exc_1, target_application_list_exc_2, target_application_iter_exc_1, target_application_iter_exc_2, ] -@pytest.mark.parametrize('target_application', _target_applications) + +@pytest.mark.parametrize("target_application", _target_applications) def test_metrics_same_with_and_without_browser_middleware(target_application): with_browser_metrics = [] without_browser_metrics = [] @@ -112,7 +131,7 @@ def test_metrics_same_with_and_without_browser_middleware(target_application): @override_application_settings(_browser_enabled_settings) def run_app_with_browser(): try: - resp = target_application.get('/') + resp = target_application.get("/") except ZeroDivisionError: pass else: @@ -122,7 +141,7 @@ def run_app_with_browser(): @override_application_settings(_browser_disabled_settings) def run_app_without_browser(): try: - resp = target_application.get('/') + resp = target_application.get("/") except ZeroDivisionError: pass else: diff --git a/tests/agent_features/test_cat.py b/tests/agent_features/test_cat.py index d812975996..ae926085a1 100644 --- a/tests/agent_features/test_cat.py +++ b/tests/agent_features/test_cat.py @@ -39,7 +39,7 @@ @wsgi_application() def target_wsgi_application(environ, start_response): status_code = int(environ["PATH_INFO"].strip("/")) - status = "%d STATUS" % status_code + status = f"{status_code} STATUS" if status_code == 304: output = b"" diff --git a/tests/agent_features/test_code_level_metrics.py b/tests/agent_features/test_code_level_metrics.py index a7aeaa39a5..e0cece6456 100644 --- a/tests/agent_features/test_code_level_metrics.py +++ b/tests/agent_features/test_code_level_metrics.py @@ -35,18 +35,16 @@ from testing_support.fixtures import dt_enabled, override_application_settings from testing_support.validators.validate_span_events import validate_span_events -import newrelic.packages.six as six from newrelic.api.background_task import background_task from newrelic.api.function_trace import FunctionTrace is_pypy = hasattr(sys, "pypy_version_info") NAMESPACE = "_test_code_level_metrics" -CLASS_NAMESPACE = ".".join((NAMESPACE, "ExerciseClass")) -CALLABLE_CLASS_NAMESPACE = ".".join((NAMESPACE, "ExerciseClassCallable")) -TYPE_CONSTRUCTOR_NAMESPACE = ".".join((NAMESPACE, "ExerciseTypeConstructor")) -TYPE_CONSTRUCTOR_CALLABLE_NAMESPACE = ".".join((NAMESPACE, "ExerciseTypeConstructorCallable")) -FUZZY_NAMESPACE = CLASS_NAMESPACE if six.PY3 else NAMESPACE +CLASS_NAMESPACE = f"{NAMESPACE}.ExerciseClass" +CALLABLE_CLASS_NAMESPACE = f"{NAMESPACE}.ExerciseClassCallable" +TYPE_CONSTRUCTOR_NAMESPACE = f"{NAMESPACE}.ExerciseTypeConstructor" +TYPE_CONSTRUCTOR_CALLABLE_NAMESPACE = f"{NAMESPACE}.ExerciseTypeConstructorCallable" if FILE_PATH.endswith(".pyc"): FILE_PATH = FILE_PATH[:-1] @@ -108,7 +106,7 @@ def _extract(obj): merge_dicts( { "code.function": "max", - "code.namespace": "builtins" if six.PY3 else "__builtin__", + "code.namespace": "builtins", }, BUILTIN_ATTRS, ), @@ -129,7 +127,7 @@ def _extract(obj): @pytest.mark.parametrize( "func,args,agents", - [pytest.param(*args, id=id_) for id_, args in six.iteritems(_TEST_BASIC_CALLABLES)], + [pytest.param(*args, id=id_) for id_, args in _TEST_BASIC_CALLABLES.items()], ) def test_code_level_metrics_basic_callables(func, args, agents, extract): @override_application_settings( @@ -167,7 +165,7 @@ def _test(): "code.filepath": FILE_PATH, "code.function": "exercise_static_method", "code.lineno": 25, - "code.namespace": FUZZY_NAMESPACE, + "code.namespace": CLASS_NAMESPACE, }, ), "class_method": ( @@ -206,7 +204,7 @@ def _test(): @pytest.mark.parametrize( "func,args,agents", - [pytest.param(*args, id=id_) for id_, args in six.iteritems(_TEST_METHODS)], + [pytest.param(*args, id=id_) for id_, args in _TEST_METHODS.items()], ) def test_code_level_metrics_methods(func, args, agents, extract): @override_application_settings( @@ -264,8 +262,7 @@ def _test(): "code.filepath": FILE_PATH, "code.function": "", "code.lineno": 61, - # Lambdas behave strangely in type constructors on Python 2 and use the class namespace. - "code.namespace": NAMESPACE if six.PY3 else TYPE_CONSTRUCTOR_NAMESPACE, + "code.namespace": NAMESPACE, }, ), "call_method": ( @@ -283,7 +280,7 @@ def _test(): @pytest.mark.parametrize( "func,args,agents", - [pytest.param(*args, id=id_) for id_, args in six.iteritems(_TEST_TYPE_CONSTRUCTOR_METHODS)], + [pytest.param(*args, id=id_) for id_, args in _TEST_TYPE_CONSTRUCTOR_METHODS.items()], ) def test_code_level_metrics_type_constructor_methods(func, args, agents, extract): @override_application_settings( @@ -352,7 +349,7 @@ def _test(): @pytest.mark.parametrize( "obj,agents", - [pytest.param(*args, id=id_) for id_, args in six.iteritems(_TEST_OBJECTS)], + [pytest.param(*args, id=id_) for id_, args in _TEST_OBJECTS.items()], ) def test_code_level_metrics_objects(obj, agents, extract): @override_application_settings( diff --git a/tests/agent_features/test_collector_payloads.py b/tests/agent_features/test_collector_payloads.py index 0c1b2367ce..42510e5c74 100644 --- a/tests/agent_features/test_collector_payloads.py +++ b/tests/agent_features/test_collector_payloads.py @@ -14,15 +14,15 @@ import pytest import webtest -from testing_support.fixtures import ( - override_application_settings, - validate_custom_event_collector_json, -) +from testing_support.fixtures import override_application_settings from testing_support.sample_applications import ( simple_app, simple_custom_event_app, simple_exceptional_app, ) +from testing_support.validators.validate_custom_event_collector_json import ( + validate_custom_event_collector_json, +) from testing_support.validators.validate_error_event_collector_json import ( validate_error_event_collector_json, ) diff --git a/tests/agent_features/test_configuration.py b/tests/agent_features/test_configuration.py index 5df69d71e9..83082398a6 100644 --- a/tests/agent_features/test_configuration.py +++ b/tests/agent_features/test_configuration.py @@ -13,16 +13,26 @@ # limitations under the License. import collections +import tempfile + +import urllib.parse as urlparse import pytest -try: - import urlparse -except ImportError: - import urllib.parse as urlparse +import logging + +from testing_support.fixtures import override_generic_settings +from newrelic.api.exceptions import ConfigurationError from newrelic.common.object_names import callable_name -from newrelic.config import delete_setting, translate_deprecated_settings +from newrelic.config import ( + _reset_config_parser, + _reset_configuration_done, + _reset_instrumentation_done, + delete_setting, + initialize, + translate_deprecated_settings, +) from newrelic.core.config import ( Settings, apply_config_setting, @@ -34,6 +44,10 @@ ) +def function_to_trace(): + pass + + def parameterize_local_config(settings_list): settings_object_list = [] @@ -262,7 +276,6 @@ def parameterize_local_config(settings_list): @parameterize_local_config(_test_dictionary_local_config) def test_dict_parse(settings): - assert "NR-SESSION" in settings.request_headers_map config = settings.event_harvest_config @@ -577,9 +590,389 @@ def test_translate_deprecated_ignored_params_with_new_setting(): ("agent_run_id", None), ("entity_guid", None), ("distributed_tracing.exclude_newrelic_header", False), + ("otlp_host", "otlp.nr-data.net"), + ("otlp_port", 0), ), ) +@override_generic_settings(global_settings(), {"host": "collector.newrelic.com"}) def test_default_values(name, expected_value): settings = global_settings() value = fetch_config_setting(settings, name) assert value == expected_value + + +def test_initialize(): + initialize() + + +newrelic_ini_contents = b""" +[newrelic] +app_name = Python Agent Test (agent_features) +""" + + +def test_initialize_raises_if_config_does_not_match_previous(): + error_message = "Configuration has already been done against " "differing configuration file or environment.*" + with pytest.raises(ConfigurationError, match=error_message): + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name) + + +def test_initialize_via_config_file(): + _reset_configuration_done() + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name) + + +def test_initialize_no_config_file(): + _reset_configuration_done() + initialize() + + +def test_initialize_config_file_does_not_exist(): + _reset_configuration_done() + error_message = "Unable to open configuration file does-not-exist." + with pytest.raises(ConfigurationError, match=error_message): + initialize(config_file="does-not-exist") + + +def test_initialize_environment(): + _reset_configuration_done() + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name, environment="developement") + + +def test_initialize_log_level(): + _reset_configuration_done() + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name, log_level="debug") + + +def test_initialize_log_file(): + _reset_configuration_done() + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name, log_file="stdout") + + +@pytest.mark.parametrize( + "feature_flag,expect_warning", + ( + (["django.instrumentation.inclusion-tags.r1"], False), + (["noexist"], True), + ), +) +def test_initialize_config_file_feature_flag(feature_flag, expect_warning, logger): + settings = global_settings() + apply_config_setting(settings, "feature_flag", feature_flag) + _reset_configuration_done() + + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name) + + message = ( + "Unknown agent feature flag 'noexist' provided. " + "Check agent documentation or release notes, or " + "contact New Relic support for clarification of " + "validity of the specific feature flag." + ) + if expect_warning: + assert message in logger.caplog.records + else: + assert message not in logger.caplog.records + + apply_config_setting(settings, "feature_flag", []) + + +@pytest.mark.parametrize( + "feature_flag,expect_warning", + ( + (["django.instrumentation.inclusion-tags.r1"], False), + (["noexist"], True), + ), +) +def test_initialize_no_config_file_feature_flag(feature_flag, expect_warning, logger): + settings = global_settings() + apply_config_setting(settings, "feature_flag", feature_flag) + _reset_configuration_done() + + initialize() + + message = ( + "Unknown agent feature flag 'noexist' provided. " + "Check agent documentation or release notes, or " + "contact New Relic support for clarification of " + "validity of the specific feature flag." + ) + + if expect_warning: + assert message in logger.caplog.records + else: + assert message not in logger.caplog.records + + apply_config_setting(settings, "feature_flag", []) + + +@pytest.mark.parametrize( + "setting_name,setting_value,expect_error", + ( + ("transaction_tracer.function_trace", [callable_name(function_to_trace)], False), + ("transaction_tracer.generator_trace", [callable_name(function_to_trace)], False), + ("transaction_tracer.function_trace", ["no_exist"], True), + ("transaction_tracer.generator_trace", ["no_exist"], True), + ), +) +def test_initialize_config_file_with_traces(setting_name, setting_value, expect_error, logger): + settings = global_settings() + apply_config_setting(settings, setting_name, setting_value) + _reset_configuration_done() + + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.seek(0) + + initialize(config_file=f.name) + + if expect_error: + assert "CONFIGURATION ERROR" in logger.caplog.records + else: + assert "CONFIGURATION ERROR" not in logger.caplog.records + + apply_config_setting(settings, setting_name, []) + + +func_newrelic_ini = b""" +[function-trace:] +enabled = True +function = test_configuration:function_to_trace +name = function_to_trace +group = group +label = label +terminal = False +rollup = foo/all +""" + +bad_func_newrelic_ini = b""" +[function-trace:] +enabled = True +function = function_to_trace +""" + +func_missing_enabled_newrelic_ini = b""" +[function-trace:] +function = function_to_trace +""" + +external_newrelic_ini = b""" +[external-trace:] +enabled = True +function = test_configuration:function_to_trace +library = "foo" +url = localhost:80/foo +method = GET +""" + +bad_external_newrelic_ini = b""" +[external-trace:] +enabled = True +function = function_to_trace +""" + +external_missing_enabled_newrelic_ini = b""" +[external-trace:] +function = function_to_trace +""" + +generator_newrelic_ini = b""" +[generator-trace:] +enabled = True +function = test_configuration:function_to_trace +name = function_to_trace +group = group +""" + +bad_generator_newrelic_ini = b""" +[generator-trace:] +enabled = True +function = function_to_trace +""" + +generator_missing_enabled_newrelic_ini = b""" +[generator-trace:] +function = function_to_trace +""" + +bg_task_newrelic_ini = b""" +[background-task:] +enabled = True +function = test_configuration:function_to_trace +lambda = test_configuration:function_to_trace +""" + +bad_bg_task_newrelic_ini = b""" +[background-task:] +enabled = True +function = function_to_trace +""" + +bg_task_missing_enabled_newrelic_ini = b""" +[background-task:] +function = function_to_trace +""" + +db_trace_newrelic_ini = b""" +[database-trace:] +enabled = True +function = test_configuration:function_to_trace +sql = test_configuration:function_to_trace +""" + +bad_db_trace_newrelic_ini = b""" +[database-trace:] +enabled = True +function = function_to_trace +""" + +db_trace_missing_enabled_newrelic_ini = b""" +[database-trace:] +function = function_to_trace +""" + +wsgi_newrelic_ini = b""" +[wsgi-application:] +enabled = True +function = test_configuration:function_to_trace +application = app +""" + +bad_wsgi_newrelic_ini = b""" +[wsgi-application:] +enabled = True +function = function_to_trace +application = app +""" + +wsgi_missing_enabled_newrelic_ini = b""" +[wsgi-application:] +function = function_to_trace +application = app +""" + +wsgi_unparseable_enabled_newrelic_ini = b""" +[wsgi-application:] +enabled = not-a-bool +function = function_to_trace +application = app +""" + + +@pytest.mark.parametrize( + "section,expect_error", + ( + (func_newrelic_ini, False), + (bad_func_newrelic_ini, True), + (func_missing_enabled_newrelic_ini, False), + (external_newrelic_ini, False), + (bad_external_newrelic_ini, True), + (external_missing_enabled_newrelic_ini, False), + (generator_newrelic_ini, False), + (bad_generator_newrelic_ini, True), + (generator_missing_enabled_newrelic_ini, False), + (bg_task_newrelic_ini, False), + (bad_bg_task_newrelic_ini, True), + (bg_task_missing_enabled_newrelic_ini, False), + (db_trace_newrelic_ini, False), + (bad_db_trace_newrelic_ini, True), + (db_trace_missing_enabled_newrelic_ini, False), + (wsgi_newrelic_ini, False), + (bad_wsgi_newrelic_ini, True), + (wsgi_missing_enabled_newrelic_ini, False), + (wsgi_unparseable_enabled_newrelic_ini, True), + ), + ids=( + "func_newrelic_ini", + "bad_func_newrelic_ini", + "func_missing_enabled_newrelic_ini", + "external_newrelic_ini", + "bad_external_newrelic_ini", + "external_missing_enabled_newrelic_ini", + "generator_newrelic_ini", + "bad_generator_newrelic_ini", + "generator_missing_enabled_newrelic_ini", + "bg_task_newrelic_ini", + "bad_bg_task_newrelic_ini", + "bg_task_missing_enabled_newrelic_ini", + "db_trace_newrelic_ini", + "bad_db_trace_newrelic_ini", + "db_trace_missing_enabled_newrelic_ini", + "wsgi_newrelic_ini", + "bad_wsgi_newrelic_ini", + "wsgi_missing_enabled_newrelic_ini", + "wsgi_unparseable_enabled_newrelic_ini", + ), +) +def test_initialize_developer_mode(section, expect_error, logger): + settings = global_settings() + apply_config_setting(settings, "monitor_mode", False) + apply_config_setting(settings, "developer_mode", True) + _reset_configuration_done() + _reset_instrumentation_done() + _reset_config_parser() + + with tempfile.NamedTemporaryFile() as f: + f.write(newrelic_ini_contents) + f.write(section) + f.seek(0) + + initialize(config_file=f.name) + + if expect_error: + assert "CONFIGURATION ERROR" in logger.caplog.records + else: + assert "CONFIGURATION ERROR" not in logger.caplog.records + + +@pytest.fixture +def caplog_handler(): + class CaplogHandler(logging.StreamHandler): + """ + To prevent possible issues with pytest's monkey patching + use a custom Caplog handler to capture all records + """ + + def __init__(self, *args, **kwargs): + self.records = [] + super(CaplogHandler, self).__init__(*args, **kwargs) + + def emit(self, record): + self.records.append(self.format(record)) + + return CaplogHandler() + + +@pytest.fixture +def logger(caplog_handler): + _logger = logging.getLogger("newrelic.config") + _logger.addHandler(caplog_handler) + _logger.caplog = caplog_handler + _logger.setLevel(logging.WARNING) + yield _logger + del caplog_handler.records[:] + _logger.removeHandler(caplog_handler) diff --git a/tests/agent_features/test_coroutine_trace.py b/tests/agent_features/test_coroutine_trace.py index 36e365bc46..2043f13268 100644 --- a/tests/agent_features/test_coroutine_trace.py +++ b/tests/agent_features/test_coroutine_trace.py @@ -31,6 +31,7 @@ from newrelic.api.datastore_trace import datastore_trace from newrelic.api.external_trace import external_trace from newrelic.api.function_trace import function_trace +from newrelic.api.graphql_trace import graphql_operation_trace, graphql_resolver_trace from newrelic.api.memcache_trace import memcache_trace from newrelic.api.message_trace import message_trace @@ -47,6 +48,8 @@ (functools.partial(datastore_trace, "lib", "foo", "bar"), "Datastore/statement/lib/foo/bar"), (functools.partial(message_trace, "lib", "op", "typ", "name"), "MessageBroker/lib/typ/op/Named/name"), (functools.partial(memcache_trace, "cmd"), "Memcache/cmd"), + (functools.partial(graphql_operation_trace), "GraphQL/operation/GraphQL///"), + (functools.partial(graphql_resolver_trace), "GraphQL/resolve/GraphQL/"), ], ) def test_coroutine_timing(trace, metric): @@ -337,6 +340,37 @@ def coro(): pass +@validate_transaction_metrics( + "test_multiple_throws_yield_a_value", + background_task=True, + scoped_metrics=[("Function/coro", 1)], + rollup_metrics=[("Function/coro", 1)], +) +@background_task(name="test_multiple_throws_yield_a_value") +def test_multiple_throws_yield_a_value(): + @function_trace(name="coro") + def coro(): + value = None + for _ in range(4): + try: + yield value + value = "bar" + except MyException: + value = "foo" + + c = coro() + + # kickstart the coroutine + assert next(c) is None + assert c.throw(MyException) == "foo" + assert c.throw(MyException) == "foo" + assert next(c) == "bar" + + # finish consumption of the coroutine if necessary + for _ in c: + pass + + @pytest.mark.parametrize( "trace", [ diff --git a/tests/agent_features/test_coroutine_transaction.py b/tests/agent_features/test_coroutine_transaction.py index 8b602ffc01..a9590b7531 100644 --- a/tests/agent_features/test_coroutine_transaction.py +++ b/tests/agent_features/test_coroutine_transaction.py @@ -209,7 +209,7 @@ def _test_async_coroutine_throw_error(): _test_async_coroutine_throw_error() assert metrics.count((metric, "")) == num_coroutines, metrics - assert metrics.count(("Errors/" + metric, "")) == num_coroutines, metrics + assert metrics.count((f"Errors/{metric}", "")) == num_coroutines, metrics assert metrics.count(("Errors/all", "")) == num_coroutines, metrics diff --git a/tests/agent_features/test_custom_events.py b/tests/agent_features/test_custom_events.py index d03feea291..103a305bcc 100644 --- a/tests/agent_features/test_custom_events.py +++ b/tests/agent_features/test_custom_events.py @@ -14,128 +14,204 @@ import time +import pytest +from testing_support.fixtures import ( + function_not_called, + override_application_settings, + reset_core_stats_engine, +) +from testing_support.validators.validate_custom_event import ( + validate_custom_event_count, + validate_custom_event_in_application_stats_engine, +) +from testing_support.validators.validate_custom_events import validate_custom_events + from newrelic.api.application import application_instance as application from newrelic.api.background_task import background_task -from newrelic.api.transaction import record_custom_event +from newrelic.api.transaction import current_transaction, record_custom_event from newrelic.core.custom_event import process_event_type -from testing_support.fixtures import (reset_core_stats_engine, - validate_custom_event_count, - validate_custom_event_in_application_stats_engine, - override_application_settings, function_not_called) - -# Test process_event_type() def test_process_event_type_name_is_string(): - name = 'string' + name = "string" assert process_event_type(name) == name + def test_process_event_type_name_is_not_string(): name = 42 assert process_event_type(name) is None + def test_process_event_type_name_ok_length(): - ok_name = 'CustomEventType' + ok_name = "CustomEventType" assert process_event_type(ok_name) == ok_name + def test_process_event_type_name_too_long(): - too_long = 'a' * 256 + too_long = "a" * 256 assert process_event_type(too_long) is None + def test_process_event_type_name_valid_chars(): - valid_name = 'az09: ' + valid_name = "az09: " assert process_event_type(valid_name) == valid_name + def test_process_event_type_name_invalid_chars(): - invalid_name = '&' + invalid_name = "&" assert process_event_type(invalid_name) is None + _now = time.time() _intrinsics = { - 'type': 'FooEvent', - 'timestamp': _now, + "type": "FooEvent", + "timestamp": _now, } -_user_params = {'foo': 'bar'} +_user_params = {"foo": "bar"} _event = [_intrinsics, _user_params] + +@reset_core_stats_engine() +@validate_custom_event_in_application_stats_engine(_event) +@background_task() +def test_add_custom_event_to_transaction_stats_engine_drops_none_attr(): + attrs = {"drop-me": None} + attrs.update(_user_params) + record_custom_event("FooEvent", attrs) + + +@reset_core_stats_engine() +@validate_custom_event_in_application_stats_engine(_event) +def test_add_custom_event_to_application_stats_engine_drops_none_attr(): + attrs = {"drop-me": None} + attrs.update(_user_params) + + app = application() + record_custom_event("FooEvent", attrs, application=app) + + @reset_core_stats_engine() @validate_custom_event_in_application_stats_engine(_event) @background_task() def test_add_custom_event_to_transaction_stats_engine(): - record_custom_event('FooEvent', _user_params) + record_custom_event("FooEvent", _user_params) + @reset_core_stats_engine() @validate_custom_event_in_application_stats_engine(_event) def test_add_custom_event_to_application_stats_engine(): app = application() - record_custom_event('FooEvent', _user_params, application=app) + record_custom_event("FooEvent", _user_params, application=app) + @reset_core_stats_engine() @validate_custom_event_count(count=0) @background_task() def test_custom_event_inside_transaction_bad_event_type(): - record_custom_event('!@#$%^&*()', {'foo': 'bar'}) + record_custom_event("!@#$%^&*()", {"foo": "bar"}) + @reset_core_stats_engine() @validate_custom_event_count(count=0) @background_task() def test_custom_event_outside_transaction_bad_event_type(): app = application() - record_custom_event('!@#$%^&*()', {'foo': 'bar'}, application=app) + record_custom_event("!@#$%^&*()", {"foo": "bar"}, application=app) + + +_mixed_params = {"foo": "bar", 123: "bad key"} -_mixed_params = {'foo': 'bar', 123: 'bad key'} @reset_core_stats_engine() @validate_custom_event_in_application_stats_engine(_event) @background_task() def test_custom_event_inside_transaction_mixed_params(): - record_custom_event('FooEvent', _mixed_params) + record_custom_event("FooEvent", _mixed_params) + + +@override_application_settings({"custom_insights_events.max_attribute_value": 4095}) +@reset_core_stats_engine() +@validate_custom_event_in_application_stats_engine([_intrinsics, {"foo": "bar", "bar": "a" * 4095}]) +@background_task() +def test_custom_event_inside_transaction_max_attribute_value(): + record_custom_event("FooEvent", {"foo": "bar", 123: "bad key", "bar": "a" * 5000}) + + +@reset_core_stats_engine() +@validate_custom_event_in_application_stats_engine([_intrinsics, {"foo": "bar", "bar": "a" * 255}]) +@background_task() +def test_custom_event_inside_transaction_default_attribute_value(): + record_custom_event("FooEvent", {"foo": "bar", 123: "bad key", "bar": "a" * 5000}) + + +@override_application_settings({"custom_insights_events.max_attribute_value": 4095}) +@reset_core_stats_engine() +@validate_custom_event_in_application_stats_engine([_intrinsics, {"foo": "bar", "bar": "a" * 4095}]) +def test_custom_event_outside_transaction_max_attribute_value(): + app = application() + record_custom_event("FooEvent", {"foo": "bar", 123: "bad key", "bar": "a" * 5000}, application=app) + + +@reset_core_stats_engine() +@validate_custom_event_in_application_stats_engine([_intrinsics, {"foo": "bar", "bar": "a" * 255}]) +def test_custom_event_outside_transaction_default_attribute_value(): + app = application() + record_custom_event("FooEvent", {"foo": "bar", 123: "bad key", "bar": "a" * 5000}, application=app) + @reset_core_stats_engine() @validate_custom_event_in_application_stats_engine(_event) @background_task() def test_custom_event_outside_transaction_mixed_params(): app = application() - record_custom_event('FooEvent', _mixed_params, application=app) + record_custom_event("FooEvent", _mixed_params, application=app) + + +_bad_params = {"*" * 256: "too long", 123: "bad key"} +_event_with_no_params = [{"type": "FooEvent", "timestamp": _now}, {}] -_bad_params = {'*' * 256: 'too long', 123: 'bad key'} -_event_with_no_params = [{'type': 'FooEvent', 'timestamp': _now}, {}] @reset_core_stats_engine() @validate_custom_event_in_application_stats_engine(_event_with_no_params) @background_task() def test_custom_event_inside_transaction_bad_params(): - record_custom_event('FooEvent', _bad_params) + record_custom_event("FooEvent", _bad_params) + @reset_core_stats_engine() @validate_custom_event_in_application_stats_engine(_event_with_no_params) @background_task() def test_custom_event_outside_transaction_bad_params(): app = application() - record_custom_event('FooEvent', _bad_params, application=app) + record_custom_event("FooEvent", _bad_params, application=app) + @reset_core_stats_engine() @validate_custom_event_count(count=0) @background_task() def test_custom_event_params_not_a_dict(): - record_custom_event('ParamsListEvent', ['not', 'a', 'dict']) + record_custom_event("ParamsListEvent", ["not", "a", "dict"]) + # Tests for Custom Events configuration settings -@override_application_settings({'collect_custom_events': False}) + +@override_application_settings({"collect_custom_events": False}) @reset_core_stats_engine() @validate_custom_event_count(count=0) @background_task() def test_custom_event_settings_check_collector_flag(): - record_custom_event('FooEvent', _user_params) + record_custom_event("FooEvent", _user_params) + -@override_application_settings({'custom_insights_events.enabled': False}) +@override_application_settings({"custom_insights_events.enabled": False}) @reset_core_stats_engine() @validate_custom_event_count(count=0) @background_task() def test_custom_event_settings_check_custom_insights_enabled(): - record_custom_event('FooEvent', _user_params) + record_custom_event("FooEvent", _user_params) + # Test that record_custom_event() methods will short-circuit. # @@ -143,15 +219,77 @@ def test_custom_event_settings_check_custom_insights_enabled(): # `create_custom_event()` function is not called, in order to avoid the # event_type and attribute processing. -@override_application_settings({'custom_insights_events.enabled': False}) -@function_not_called('newrelic.api.transaction', 'create_custom_event') + +@override_application_settings({"custom_insights_events.enabled": False}) +@function_not_called("newrelic.api.transaction", "create_custom_event") @background_task() def test_transaction_create_custom_event_not_called(): - record_custom_event('FooEvent', _user_params) + record_custom_event("FooEvent", _user_params) + -@override_application_settings({'custom_insights_events.enabled': False}) -@function_not_called('newrelic.core.application', 'create_custom_event') +@override_application_settings({"custom_insights_events.enabled": False}) +@function_not_called("newrelic.core.application", "create_custom_event") @background_task() def test_application_create_custom_event_not_called(): app = application() - record_custom_event('FooEvent', _user_params, application=app) + record_custom_event("FooEvent", _user_params, application=app) + + +# Test completness of LLM content/input despite attribute limits being set + + +@pytest.mark.parametrize( + "event_type,event_data,expected_event_data", + ( + [ + "LlmChatCompletionMessage", + { + "content": "A" * 9001, + "input": "B" * 9001, + "foo": f"b{'a' * 9000}r", + }, + { + "content": "A" * 9001, + "input": "B" * 300, + "foo": f"b{'a' * 299}", + }, + ], + [ + "LlmEmbedding", + { + "content": "A" * 9001, + "input": "B" * 9001, + "foo": f"b{'a' * 9000}r", + }, + { + "content": "A" * 300, + "input": "B" * 9001, + "foo": f"b{'a' * 299}", + }, + ], + [ + "MyCustomEvent", + { + "content": "A" * 9001, + "input": "B" * 9001, + "foo": f"b{'a' * 9000}r", + }, + { + "content": "A" * 300, + "input": "B" * 300, + "foo": f"b{'a' * 299}", + }, + ], + ), +) +def test_create_custom_event_no_limit(event_type, event_data, expected_event_data): + @reset_core_stats_engine() + @override_application_settings({"custom_insights_events.max_attribute_value": 300}) + @validate_custom_event_count(1) + @validate_custom_events([({"type": event_type}, expected_event_data)]) + @background_task() + def _test(): + transaction = current_transaction() + transaction.record_custom_event(event_type, event_data) + + _test() diff --git a/tests/agent_features/test_custom_metrics.py b/tests/agent_features/test_custom_metrics.py new file mode 100644 index 0000000000..21a67149a2 --- /dev/null +++ b/tests/agent_features/test_custom_metrics.py @@ -0,0 +1,62 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import reset_core_stats_engine +from testing_support.validators.validate_custom_metrics_outside_transaction import ( + validate_custom_metrics_outside_transaction, +) + +from newrelic.api.application import application_instance as application +from newrelic.api.background_task import background_task +from newrelic.api.transaction import ( + current_transaction, + record_custom_metric, + record_custom_metrics, +) + + +# Testing record_custom_metric +@reset_core_stats_engine() +@background_task() +def test_custom_metric_inside_transaction(): + transaction = current_transaction() + record_custom_metric("CustomMetric/InsideTransaction/Count", 1) + for metric in transaction._custom_metrics.metrics(): + assert metric == ("CustomMetric/InsideTransaction/Count", [1, 1, 1, 1, 1, 1]) + + +@reset_core_stats_engine() +@validate_custom_metrics_outside_transaction([("CustomMetric/OutsideTransaction/Count", 1)]) +@background_task() +def test_custom_metric_outside_transaction_with_app(): + app = application() + record_custom_metric("CustomMetric/OutsideTransaction/Count", 1, application=app) + + +# Testing record_custom_metricS +@reset_core_stats_engine() +@background_task() +def test_custom_metrics_inside_transaction(): + transaction = current_transaction() + record_custom_metrics([("CustomMetrics/InsideTransaction/Count", 1)]) + for metric in transaction._custom_metrics.metrics(): + assert metric == ("CustomMetrics/InsideTransaction/Count", [1, 1, 1, 1, 1, 1]) + + +@reset_core_stats_engine() +@validate_custom_metrics_outside_transaction([("CustomMetrics/OutsideTransaction/Count", 1)]) +@background_task() +def test_custom_metrics_outside_transaction_with_app(): + app = application() + record_custom_metrics([("CustomMetrics/OutsideTransaction/Count", 1)], application=app) diff --git a/tests/agent_features/test_datastore_trace.py b/tests/agent_features/test_datastore_trace.py new file mode 100644 index 0000000000..08067e0402 --- /dev/null +++ b/tests/agent_features/test_datastore_trace.py @@ -0,0 +1,89 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.validators.validate_datastore_trace_inputs import ( + validate_datastore_trace_inputs, +) + +from newrelic.api.background_task import background_task +from newrelic.api.datastore_trace import DatastoreTrace, DatastoreTraceWrapper + + +@validate_datastore_trace_inputs( + operation="test_operation", + target="test_target", + host="test_host", + port_path_or_id="test_port", + database_name="test_db_name", +) +@background_task() +def test_dt_trace_all_args(): + with DatastoreTrace( + product="Agent Features", + target="test_target", + operation="test_operation", + host="test_host", + port_path_or_id="test_port", + database_name="test_db_name", + ): + pass + + +@validate_datastore_trace_inputs(operation=None, target=None, host=None, port_path_or_id=None, database_name=None) +@background_task() +def test_dt_trace_empty(): + with DatastoreTrace(product=None, target=None, operation=None): + pass + + +@background_task() +def test_dt_trace_callable_args(): + def product_callable(): + return "Agent Features" + + def target_callable(): + return "test_target" + + def operation_callable(): + return "test_operation" + + def host_callable(): + return "test_host" + + def port_path_id_callable(): + return "test_port" + + def db_name_callable(): + return "test_db_name" + + @validate_datastore_trace_inputs( + operation="test_operation", + target="test_target", + host="test_host", + port_path_or_id="test_port", + database_name="test_db_name", + ) + def _test(): + pass + + wrapped_fn = DatastoreTraceWrapper( + _test, + product=product_callable, + target=target_callable, + operation=operation_callable, + host=host_callable, + port_path_or_id=port_path_id_callable, + database_name=db_name_callable, + ) + wrapped_fn() diff --git a/tests/agent_features/test_dead_transactions.py b/tests/agent_features/test_dead_transactions.py index 60d03003ec..6eaf2d2be9 100644 --- a/tests/agent_features/test_dead_transactions.py +++ b/tests/agent_features/test_dead_transactions.py @@ -14,7 +14,6 @@ import gc import pytest -import newrelic.packages.six as six from newrelic.api.background_task import BackgroundTask from newrelic.api.application import application_instance @@ -43,11 +42,6 @@ def capture_errors(wrapped, instance, args, kwargs): @pytest.mark.parametrize('circular', (True, False)) @capture_errors def test_dead_transaction_ends(circular): - if circular and six.PY2: - pytest.skip("Circular references in py2 result in a memory leak. " - "There is no way to remove transactions from the weakref " - "cache in this case.") - transaction = BackgroundTask( application_instance(), "test_dead_transaction_ends") if circular: diff --git a/tests/agent_features/test_dimensional_metrics.py b/tests/agent_features/test_dimensional_metrics.py new file mode 100644 index 0000000000..a4b5054797 --- /dev/null +++ b/tests/agent_features/test_dimensional_metrics.py @@ -0,0 +1,216 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from importlib import reload + +import pytest +from testing_support.fixtures import reset_core_stats_engine +from testing_support.validators.validate_dimensional_metric_payload import ( + validate_dimensional_metric_payload, +) +from testing_support.validators.validate_dimensional_metrics_outside_transaction import ( + validate_dimensional_metrics_outside_transaction, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +import newrelic.core.otlp_utils +from newrelic.api.application import application_instance +from newrelic.api.background_task import background_task +from newrelic.api.transaction import ( + record_dimensional_metric, + record_dimensional_metrics, +) +from newrelic.common.metric_utils import create_metric_identity +from newrelic.core.config import global_settings + + +@pytest.fixture(scope="module", autouse=True, params=["protobuf", "json"]) +def otlp_content_encoding(request): + _settings = global_settings() + prev = _settings.debug.otlp_content_encoding + _settings.debug.otlp_content_encoding = request.param + reload(newrelic.core.otlp_utils) + assert newrelic.core.otlp_utils.otlp_content_setting == request.param, "Content encoding mismatch." + + yield + + _settings.debug.otlp_content_encoding = prev + + +_test_tags_examples = [ + (None, None), + ({}, None), + ({"drop-me": None}, None), + ([], None), + ({"str": "a"}, frozenset({("str", "a")})), + ({"int": 1}, frozenset({("int", 1)})), + ({"float": 1.0}, frozenset({("float", 1.0)})), + ({"bool": True}, frozenset({("bool", True)})), + ({"list": [1]}, frozenset({("list", "[1]")})), + ({"dict": {"subtag": 1}}, frozenset({("dict", "{'subtag': 1}")})), + ([("tags-as-list", 1)], frozenset({("tags-as-list", 1)})), +] + + +@pytest.mark.parametrize("tags,expected", _test_tags_examples) +def test_create_metric_identity(tags, expected): + name = "Metric" + output_name, output_tags = create_metric_identity(name, tags=tags) + assert output_name == name, "Name does not match." + assert output_tags == expected, "Output tags do not match." + + +@pytest.mark.parametrize("tags,expected", _test_tags_examples) +@reset_core_stats_engine() +def test_record_dimensional_metric_inside_transaction(tags, expected): + @validate_transaction_metrics( + "test_record_dimensional_metric_inside_transaction", + background_task=True, + dimensional_metrics=[ + ("Metric", expected, 1), + ], + ) + @background_task(name="test_record_dimensional_metric_inside_transaction") + def _test(): + record_dimensional_metric("Metric", 1, tags=tags) + + _test() + + +@pytest.mark.parametrize("tags,expected", _test_tags_examples) +@reset_core_stats_engine() +def test_record_dimensional_metric_outside_transaction(tags, expected): + @validate_dimensional_metrics_outside_transaction([("Metric", expected, 1)]) + def _test(): + app = application_instance() + record_dimensional_metric("Metric", 1, tags=tags, application=app) + + _test() + + +@pytest.mark.parametrize("tags,expected", _test_tags_examples) +@reset_core_stats_engine() +def test_record_dimensional_metrics_inside_transaction(tags, expected): + @validate_transaction_metrics( + "test_record_dimensional_metrics_inside_transaction", + background_task=True, + dimensional_metrics=[("Metric.1", expected, 1), ("Metric.2", expected, 1)], + ) + @background_task(name="test_record_dimensional_metrics_inside_transaction") + def _test(): + record_dimensional_metrics([("Metric.1", 1, tags), ("Metric.2", 1, tags)]) + + _test() + + +@pytest.mark.parametrize("tags,expected", _test_tags_examples) +@reset_core_stats_engine() +def test_record_dimensional_metrics_outside_transaction(tags, expected): + @validate_dimensional_metrics_outside_transaction([("Metric.1", expected, 1), ("Metric.2", expected, 1)]) + def _test(): + app = application_instance() + record_dimensional_metrics([("Metric.1", 1, tags), ("Metric.2", 1, tags)], application=app) + + _test() + + +@reset_core_stats_engine() +def test_dimensional_metrics_different_tags(): + @validate_transaction_metrics( + "test_dimensional_metrics_different_tags", + background_task=True, + dimensional_metrics=[ + ("Metric", frozenset({("tag", 1)}), 1), + ("Metric", frozenset({("tag", 2)}), 2), + ], + ) + @background_task(name="test_dimensional_metrics_different_tags") + def _test(): + record_dimensional_metrics( + [ + ("Metric", 1, {"tag": 1}), + ("Metric", 1, {"tag": 2}), + ] + ) + record_dimensional_metric("Metric", 1, {"tag": 2}) + + _test() + + +@reset_core_stats_engine() +@validate_dimensional_metric_payload( + summary_metrics=[ + ("Metric.Summary", {"tag": 1}, 1), + ("Metric.Summary", {"tag": 2}, 1), + ("Metric.Summary", None, 1), + ("Metric.Mixed", {"tag": 1}, 1), + ("Metric.NotPresent", None, None), + ], + count_metrics=[ + ("Metric.Count", {"tag": 1}, 1), + ("Metric.Count", {"tag": 2}, 2), + ("Metric.Count", None, 3), + ("Metric.Mixed", {"tag": 2}, 2), + ("Metric.NotPresent", None, None), + ], +) +def test_dimensional_metrics_payload(): + @background_task(name="test_dimensional_metric_payload") + def _test(): + record_dimensional_metrics( + [ + ("Metric.Summary", 1, {"tag": 1}), + ("Metric.Summary", 2, {"tag": 2}), + ("Metric.Summary", 3), # No tags + ("Metric.Count", {"count": 1}, {"tag": 1}), + ("Metric.Count", {"count": 2}, {"tag": 2}), + ("Metric.Count", {"count": 3}), # No tags + ("Metric.Mixed", 1, {"tag": 1}), + ("Metric.Mixed", {"count": 2}, {"tag": 2}), + ] + ) + + _test() + app = application_instance() + core_app = app._agent.application(app.name) + core_app.harvest() + + +@reset_core_stats_engine() +@validate_dimensional_metric_payload( + summary_metrics=[ + ("Metric.Summary", None, 1), + ("Metric.Count", None, None), # Should NOT be present + ], + count_metrics=[ + ("Metric.Count", None, 1), + ("Metric.Summary", None, None), # Should NOT be present + ], +) +def test_dimensional_metrics_no_duplicate_encodings(): + @background_task(name="test_dimensional_metric_payload") + def _test(): + record_dimensional_metrics( + [ + ("Metric.Summary", 1), + ("Metric.Count", {"count": 1}), + ] + ) + + _test() + app = application_instance() + core_app = app._agent.application(app.name) + core_app.harvest() diff --git a/tests/agent_features/test_distributed_tracing.py b/tests/agent_features/test_distributed_tracing.py index 7f795573a6..370dc3774b 100644 --- a/tests/agent_features/test_distributed_tracing.py +++ b/tests/agent_features/test_distributed_tracing.py @@ -12,71 +12,86 @@ # See the License for the specific language governing permissions and # limitations under the License. +import copy import json + import pytest import webtest -import copy +from testing_support.fixtures import override_application_settings, validate_attributes +from testing_support.validators.validate_error_event_attributes import ( + validate_error_event_attributes, +) +from testing_support.validators.validate_transaction_event_attributes import ( + validate_transaction_event_attributes, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.application import application_instance -from newrelic.api.background_task import background_task, BackgroundTask -from newrelic.api.transaction import (current_transaction, current_trace_id, - current_span_id) +from newrelic.api.background_task import BackgroundTask, background_task +from newrelic.api.external_trace import ExternalTrace from newrelic.api.time_trace import current_trace +from newrelic.api.transaction import ( + accept_distributed_trace_headers, + accept_distributed_trace_payload, + create_distributed_trace_payload, + current_span_id, + current_trace_id, + current_transaction, +) from newrelic.api.web_transaction import WSGIWebTransaction from newrelic.api.wsgi_application import wsgi_application -from testing_support.fixtures import (override_application_settings, - validate_attributes, - validate_error_event_attributes) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes - -distributed_trace_intrinsics = ['guid', 'traceId', 'priority', 'sampled'] -inbound_payload_intrinsics = ['parent.type', 'parent.app', 'parent.account', - 'parent.transportType', 'parent.transportDuration'] +distributed_trace_intrinsics = ["guid", "traceId", "priority", "sampled"] +inbound_payload_intrinsics = [ + "parent.type", + "parent.app", + "parent.account", + "parent.transportType", + "parent.transportDuration", +] payload = { - 'v': [0, 1], - 'd': { - 'ac': '1', - 'ap': '2827902', - 'id': '7d3efb1b173fecfa', - 'pa': '5e5733a911cfbc73', - 'pr': 10.001, - 'sa': True, - 'ti': 1518469636035, - 'tr': 'd6b4ba0c3a712ca', - 'ty': 'App', - } + "v": [0, 1], + "d": { + "ac": "1", + "ap": "2827902", + "id": "7d3efb1b173fecfa", + "pa": "5e5733a911cfbc73", + "pr": 10.001, + "sa": True, + "ti": 1518469636035, + "tr": "d6b4ba0c3a712ca", + "ty": "App", + }, } -parent_order = ['parent_type', 'parent_account', - 'parent_app', 'parent_transport_type'] +parent_order = ["parent_type", "parent_account", "parent_app", "parent_transport_type"] parent_info = { - 'parent_type': payload['d']['ty'], - 'parent_account': payload['d']['ac'], - 'parent_app': payload['d']['ap'], - 'parent_transport_type': 'HTTP' + "parent_type": payload["d"]["ty"], + "parent_account": payload["d"]["ac"], + "parent_app": payload["d"]["ap"], + "parent_transport_type": "HTTP", } @wsgi_application() def target_wsgi_application(environ, start_response): - status = '200 OK' - output = b'hello world' - response_headers = [('Content-type', 'text/html; charset=utf-8'), - ('Content-Length', str(len(output)))] + status = "200 OK" + output = b"hello world" + response_headers = [("Content-type", "text/html; charset=utf-8"), ("Content-Length", str(len(output)))] txn = current_transaction() # Make assertions on the WSGIWebTransaction object assert txn._distributed_trace_state - assert txn.parent_type == 'App' - assert txn.parent_app == '2827902' - assert txn.parent_account == '1' - assert txn.parent_span == '7d3efb1b173fecfa' - assert txn.parent_transport_type == 'HTTP' + assert txn.parent_type == "App" + assert txn.parent_app == "2827902" + assert txn.parent_account == "1" + assert txn.parent_span == "7d3efb1b173fecfa" + assert txn.parent_transport_type == "HTTP" assert isinstance(txn.parent_transport_duration, float) - assert txn._trace_id == 'd6b4ba0c3a712ca' + assert txn._trace_id == "d6b4ba0c3a712ca" assert txn.priority == 10.001 assert txn.sampled @@ -87,90 +102,75 @@ def target_wsgi_application(environ, start_response): test_application = webtest.TestApp(target_wsgi_application) _override_settings = { - 'trusted_account_key': '1', - 'distributed_tracing.enabled': True, + "trusted_account_key": "1", + "distributed_tracing.enabled": True, } _metrics = [ - ('Supportability/DistributedTrace/AcceptPayload/Success', 1), - ('Supportability/TraceContext/Accept/Success', None) + ("Supportability/DistributedTrace/AcceptPayload/Success", 1), + ("Supportability/TraceContext/Accept/Success", None), ] @override_application_settings(_override_settings) -@validate_transaction_metrics( - '', - group='Uri', - rollup_metrics=_metrics) +@validate_transaction_metrics("", group="Uri", rollup_metrics=_metrics) def test_distributed_tracing_web_transaction(): - headers = {'newrelic': json.dumps(payload)} - response = test_application.get('/', headers=headers) - assert 'X-NewRelic-App-Data' not in response.headers + headers = {"newrelic": json.dumps(payload)} + response = test_application.get("/", headers=headers) + assert "X-NewRelic-App-Data" not in response.headers -@pytest.mark.parametrize('span_events', (True, False)) -@pytest.mark.parametrize('accept_payload', (True, False)) +@pytest.mark.parametrize("span_events", (True, False)) +@pytest.mark.parametrize("accept_payload", (True, False)) def test_distributed_trace_attributes(span_events, accept_payload): if accept_payload: - _required_intrinsics = ( - distributed_trace_intrinsics + inbound_payload_intrinsics) + _required_intrinsics = distributed_trace_intrinsics + inbound_payload_intrinsics _forgone_txn_intrinsics = [] _forgone_error_intrinsics = [] _exact_intrinsics = { - 'parent.type': 'Mobile', - 'parent.app': '2827902', - 'parent.account': '1', - 'parent.transportType': 'HTTP', - 'traceId': 'd6b4ba0c3a712ca', + "parent.type": "Mobile", + "parent.app": "2827902", + "parent.account": "1", + "parent.transportType": "HTTP", + "traceId": "d6b4ba0c3a712ca", } - _exact_txn_attributes = {'agent': {}, 'user': {}, - 'intrinsic': _exact_intrinsics.copy()} - _exact_error_attributes = {'agent': {}, 'user': {}, - 'intrinsic': _exact_intrinsics.copy()} - _exact_txn_attributes['intrinsic']['parentId'] = '7d3efb1b173fecfa' - _exact_txn_attributes['intrinsic']['parentSpanId'] = 'c86df80de2e6f51c' - - _forgone_error_intrinsics.append('parentId') - _forgone_error_intrinsics.append('parentSpanId') - _forgone_txn_intrinsics.append('grandparentId') - _forgone_error_intrinsics.append('grandparentId') - - _required_attributes = { - 'intrinsic': _required_intrinsics, 'agent': [], 'user': []} - _forgone_txn_attributes = {'intrinsic': _forgone_txn_intrinsics, - 'agent': [], 'user': []} - _forgone_error_attributes = {'intrinsic': _forgone_error_intrinsics, - 'agent': [], 'user': []} + _exact_txn_attributes = {"agent": {}, "user": {}, "intrinsic": _exact_intrinsics.copy()} + _exact_error_attributes = {"agent": {}, "user": {}, "intrinsic": _exact_intrinsics.copy()} + _exact_txn_attributes["intrinsic"]["parentId"] = "7d3efb1b173fecfa" + _exact_txn_attributes["intrinsic"]["parentSpanId"] = "c86df80de2e6f51c" + + _forgone_error_intrinsics.append("parentId") + _forgone_error_intrinsics.append("parentSpanId") + _forgone_txn_intrinsics.append("grandparentId") + _forgone_error_intrinsics.append("grandparentId") + + _required_attributes = {"intrinsic": _required_intrinsics, "agent": [], "user": []} + _forgone_txn_attributes = {"intrinsic": _forgone_txn_intrinsics, "agent": [], "user": []} + _forgone_error_attributes = {"intrinsic": _forgone_error_intrinsics, "agent": [], "user": []} else: _required_intrinsics = distributed_trace_intrinsics - _forgone_txn_intrinsics = _forgone_error_intrinsics = \ - inbound_payload_intrinsics + ['grandparentId', 'parentId', - 'parentSpanId'] - - _required_attributes = { - 'intrinsic': _required_intrinsics, 'agent': [], 'user': []} - _forgone_txn_attributes = {'intrinsic': _forgone_txn_intrinsics, - 'agent': [], 'user': []} - _forgone_error_attributes = {'intrinsic': _forgone_error_intrinsics, - 'agent': [], 'user': []} + _forgone_txn_intrinsics = _forgone_error_intrinsics = inbound_payload_intrinsics + [ + "grandparentId", + "parentId", + "parentSpanId", + ] + + _required_attributes = {"intrinsic": _required_intrinsics, "agent": [], "user": []} + _forgone_txn_attributes = {"intrinsic": _forgone_txn_intrinsics, "agent": [], "user": []} + _forgone_error_attributes = {"intrinsic": _forgone_error_intrinsics, "agent": [], "user": []} _exact_txn_attributes = _exact_error_attributes = None _forgone_trace_intrinsics = _forgone_error_intrinsics test_settings = _override_settings.copy() - test_settings['span_events.enabled'] = span_events + test_settings["span_events.enabled"] = span_events @override_application_settings(test_settings) - @validate_transaction_event_attributes( - _required_attributes, _forgone_txn_attributes, - _exact_txn_attributes) - @validate_error_event_attributes( - _required_attributes, _forgone_error_attributes, - _exact_error_attributes) - @validate_attributes('intrinsic', - _required_intrinsics, _forgone_trace_intrinsics) - @background_task(name='test_distributed_trace_attributes') + @validate_transaction_event_attributes(_required_attributes, _forgone_txn_attributes, _exact_txn_attributes) + @validate_error_event_attributes(_required_attributes, _forgone_error_attributes, _exact_error_attributes) + @validate_attributes("intrinsic", _required_intrinsics, _forgone_trace_intrinsics) + @background_task(name="test_distributed_trace_attributes") def _test(): txn = current_transaction() @@ -183,19 +183,19 @@ def _test(): "id": "c86df80de2e6f51c", "tr": "d6b4ba0c3a712ca", "ti": 1518469636035, - "tx": "7d3efb1b173fecfa" - } + "tx": "7d3efb1b173fecfa", + }, } - payload['d']['pa'] = "5e5733a911cfbc73" + payload["d"]["pa"] = "5e5733a911cfbc73" if accept_payload: - result = txn.accept_distributed_trace_payload(payload) + result = accept_distributed_trace_payload(payload) assert result else: - txn._create_distributed_trace_payload() + create_distributed_trace_payload() try: - raise ValueError('cookies') + raise ValueError("cookies") except ValueError: txn.notice_error() @@ -203,33 +203,30 @@ def _test(): _forgone_attributes = { - 'agent': [], - 'user': [], - 'intrinsic': (inbound_payload_intrinsics + ['grandparentId']), + "agent": [], + "user": [], + "intrinsic": (inbound_payload_intrinsics + ["grandparentId"]), } @override_application_settings(_override_settings) -@validate_transaction_event_attributes( - {}, _forgone_attributes) -@validate_error_event_attributes( - {}, _forgone_attributes) -@validate_attributes('intrinsic', - {}, _forgone_attributes['intrinsic']) -@background_task(name='test_distributed_trace_attrs_omitted') +@validate_transaction_event_attributes({}, _forgone_attributes) +@validate_error_event_attributes({}, _forgone_attributes) +@validate_attributes("intrinsic", {}, _forgone_attributes["intrinsic"]) +@background_task(name="test_distributed_trace_attrs_omitted") def test_distributed_trace_attrs_omitted(): txn = current_transaction() try: - raise ValueError('cookies') + raise ValueError("cookies") except ValueError: txn.notice_error() # test our distributed_trace metrics by creating a transaction and then forcing # it to process a distributed trace payload -@pytest.mark.parametrize('web_transaction', (True, False)) -@pytest.mark.parametrize('gen_error', (True, False)) -@pytest.mark.parametrize('has_parent', (True, False)) +@pytest.mark.parametrize("web_transaction", (True, False)) +@pytest.mark.parametrize("gen_error", (True, False)) +@pytest.mark.parametrize("has_parent", (True, False)) def test_distributed_tracing_metrics(web_transaction, gen_error, has_parent): def _make_dt_tag(pi): return "%s/%s/%s/%s/all" % tuple(pi[x] for x in parent_order) @@ -237,11 +234,11 @@ def _make_dt_tag(pi): # figure out which metrics we'll see based on the test params # note: we'll always see DurationByCaller if the distributed # tracing flag is turned on - metrics = ['DurationByCaller'] + metrics = ["DurationByCaller"] if gen_error: - metrics.append('ErrorsByCaller') + metrics.append("ErrorsByCaller") if has_parent: - metrics.append('TransportDuration') + metrics.append("TransportDuration") tag = None dt_payload = copy.deepcopy(payload) @@ -251,15 +248,14 @@ def _make_dt_tag(pi): if has_parent: tag = _make_dt_tag(parent_info) else: - tag = _make_dt_tag(dict((x, 'Unknown') for x in parent_info.keys())) - del dt_payload['d']['tr'] + # tag = _make_dt_tag(dict((x, "Unknown") for x in parent_order)) + tag = _make_dt_tag(dict((x, "Unknown") for x in parent_info.keys())) + del dt_payload["d"]["tr"] # now run the test - transaction_name = "test_dt_metrics_%s" % '_'.join(metrics) + transaction_name = f"test_dt_metrics_{'_'.join(metrics)}" _rollup_metrics = [ - ("%s/%s%s" % (x, tag, bt), 1) - for x in metrics - for bt in ['', 'Web' if web_transaction else 'Other'] + (f"{x}/{tag}{bt}", 1) for x in metrics for bt in ["", "Web" if web_transaction else "Other"] ] def _make_test_transaction(): @@ -268,16 +264,15 @@ def _make_test_transaction(): if not web_transaction: return BackgroundTask(application, transaction_name) - environ = {'REQUEST_URI': '/trace_ends_after_txn'} + environ = {"REQUEST_URI": "/trace_ends_after_txn"} tn = WSGIWebTransaction(application, environ) tn.set_transaction_name(transaction_name) return tn @override_application_settings(_override_settings) @validate_transaction_metrics( - transaction_name, - background_task=not(web_transaction), - rollup_metrics=_rollup_metrics) + transaction_name, background_task=not (web_transaction), rollup_metrics=_rollup_metrics + ) def _test(): with _make_test_transaction() as transaction: transaction.accept_distributed_trace_payload(dt_payload) @@ -291,62 +286,62 @@ def _test(): _test() -NEW_RELIC_ACCEPTED = \ - [('Supportability/DistributedTrace/AcceptPayload/Success', 1), - ('Supportability/TraceContext/Accept/Success', None), - ('Supportability/TraceContext/TraceParent/Accept/Success', None), - ('Supportability/TraceContext/Accept/Success', None)] -TRACE_CONTEXT_ACCEPTED = \ - [('Supportability/TraceContext/Accept/Success', 1), - ('Supportability/TraceContext/TraceParent/Accept/Success', 1), - ('Supportability/TraceContext/Accept/Success', 1), - ('Supportability/DistributedTrace/AcceptPayload/Success', None)] -NO_HEADERS_ACCEPTED = \ - [('Supportability/DistributedTrace/AcceptPayload/Success', None), - ('Supportability/TraceContext/Accept/Success', None), - ('Supportability/TraceContext/TraceParent/Accept/Success', None), - ('Supportability/TraceContext/Accept/Success', None)] -TRACEPARENT = '00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01' -TRACESTATE = 'rojo=f06a0ba902b7,congo=t61rcWkgMzE' - - -@pytest.mark.parametrize('traceparent,tracestate,newrelic,metrics', - [(False, False, False, NO_HEADERS_ACCEPTED), - (False, False, True, NEW_RELIC_ACCEPTED), - (False, True, True, NEW_RELIC_ACCEPTED), - (False, True, False, NO_HEADERS_ACCEPTED), - (True, True, True, TRACE_CONTEXT_ACCEPTED), - (True, False, False, TRACE_CONTEXT_ACCEPTED), - (True, False, True, TRACE_CONTEXT_ACCEPTED), - (True, True, False, TRACE_CONTEXT_ACCEPTED)] - ) +NEW_RELIC_ACCEPTED = [ + ("Supportability/DistributedTrace/AcceptPayload/Success", 1), + ("Supportability/TraceContext/Accept/Success", None), + ("Supportability/TraceContext/TraceParent/Accept/Success", None), + ("Supportability/TraceContext/Accept/Success", None), +] +TRACE_CONTEXT_ACCEPTED = [ + ("Supportability/TraceContext/Accept/Success", 1), + ("Supportability/TraceContext/TraceParent/Accept/Success", 1), + ("Supportability/TraceContext/Accept/Success", 1), + ("Supportability/DistributedTrace/AcceptPayload/Success", None), +] +NO_HEADERS_ACCEPTED = [ + ("Supportability/DistributedTrace/AcceptPayload/Success", None), + ("Supportability/TraceContext/Accept/Success", None), + ("Supportability/TraceContext/TraceParent/Accept/Success", None), + ("Supportability/TraceContext/Accept/Success", None), +] +TRACEPARENT = "00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-01" +TRACESTATE = "rojo=f06a0ba902b7,congo=t61rcWkgMzE" + + +@pytest.mark.parametrize( + "traceparent,tracestate,newrelic,metrics", + [ + (False, False, False, NO_HEADERS_ACCEPTED), + (False, False, True, NEW_RELIC_ACCEPTED), + (False, True, True, NEW_RELIC_ACCEPTED), + (False, True, False, NO_HEADERS_ACCEPTED), + (True, True, True, TRACE_CONTEXT_ACCEPTED), + (True, False, False, TRACE_CONTEXT_ACCEPTED), + (True, False, True, TRACE_CONTEXT_ACCEPTED), + (True, True, False, TRACE_CONTEXT_ACCEPTED), + ], +) @override_application_settings(_override_settings) -def test_distributed_tracing_backwards_compatibility(traceparent, - tracestate, - newrelic, - metrics): - +def test_distributed_tracing_backwards_compatibility(traceparent, tracestate, newrelic, metrics): headers = [] if traceparent: - headers.append(('traceparent', TRACEPARENT)) + headers.append(("traceparent", TRACEPARENT)) if tracestate: - headers.append(('tracestate', TRACESTATE)) + headers.append(("tracestate", TRACESTATE)) if newrelic: - headers.append(('newrelic', json.dumps(payload))) + headers.append(("newrelic", json.dumps(payload))) @validate_transaction_metrics( - "test_distributed_tracing_backwards_compatibility", - background_task=True, - rollup_metrics=metrics) - @background_task(name='test_distributed_tracing_backwards_compatibility') + "test_distributed_tracing_backwards_compatibility", background_task=True, rollup_metrics=metrics + ) + @background_task(name="test_distributed_tracing_backwards_compatibility") def _test(): - transaction = current_transaction() - transaction.accept_distributed_trace_headers(headers) + accept_distributed_trace_headers(headers) _test() -@background_task(name='test_current_trace_id_api_inside_transaction') +@background_task(name="test_current_trace_id_api_inside_transaction") def test_current_trace_id_api_inside_transaction(): trace_id = current_trace_id() assert len(trace_id) == 32 @@ -358,7 +353,7 @@ def test_current_trace_id_api_outside_transaction(): assert trace_id is None -@background_task(name='test_current_span_id_api_inside_transaction') +@background_task(name="test_current_span_id_api_inside_transaction") def test_current_span_id_inside_transaction(): span_id = current_span_id() assert span_id == current_trace().guid @@ -367,3 +362,65 @@ def test_current_span_id_inside_transaction(): def test_current_span_id_outside_transaction(): span_id = current_span_id() assert span_id is None + + +@pytest.mark.parametrize("trusted_account_key", ("1", None), ids=("tk_set", "tk_unset")) +def test_outbound_dt_payload_generation(trusted_account_key): + @override_application_settings( + { + "distributed_tracing.enabled": True, + "account_id": "1", + "trusted_account_key": trusted_account_key, + "primary_application_id": "1", + } + ) + @background_task(name="test_outbound_dt_payload_generation") + def _test_outbound_dt_payload_generation(): + transaction = current_transaction() + payload = ExternalTrace.generate_request_headers(transaction) + if trusted_account_key: + assert payload + # Ensure trusted account key present as vendor + assert dict(payload)["tracestate"].startswith("1@nr=") + else: + assert not payload + + _test_outbound_dt_payload_generation() + + +@pytest.mark.parametrize("trusted_account_key", ("1", None), ids=("tk_set", "tk_unset")) +def test_inbound_dt_payload_acceptance(trusted_account_key): + @override_application_settings( + { + "distributed_tracing.enabled": True, + "account_id": "1", + "trusted_account_key": trusted_account_key, + "primary_application_id": "1", + } + ) + @background_task(name="_test_inbound_dt_payload_acceptance") + def _test_inbound_dt_payload_acceptance(): + transaction = current_transaction() + + payload = { + "v": [0, 1], + "d": { + "ty": "Mobile", + "ac": "1", + "tk": "1", + "ap": "2827902", + "pa": "5e5733a911cfbc73", + "id": "7d3efb1b173fecfa", + "tr": "d6b4ba0c3a712ca", + "ti": 1518469636035, + "tx": "8703ff3d88eefe9d", + }, + } + + result = transaction.accept_distributed_trace_payload(payload) + if trusted_account_key: + assert result + else: + assert not result + + _test_inbound_dt_payload_acceptance() diff --git a/tests/agent_features/test_error_events.py b/tests/agent_features/test_error_events.py index 72bdb14f7c..ad10d3a148 100644 --- a/tests/agent_features/test_error_events.py +++ b/tests/agent_features/test_error_events.py @@ -16,23 +16,21 @@ import time import webtest - from testing_support.fixtures import ( cat_enabled, make_cross_agent_headers, - make_synthetics_header, + make_synthetics_headers, override_application_settings, reset_core_stats_engine, validate_error_event_sample_data, - validate_transaction_error_event_count, ) from testing_support.sample_applications import fully_featured_app -from testing_support.validators.validate_error_trace_attributes import ( - validate_error_trace_attributes, -) from testing_support.validators.validate_non_transaction_error_event import ( validate_non_transaction_error_event, ) +from testing_support.validators.validate_transaction_error_event_count import ( + validate_transaction_error_event_count, +) from newrelic.api.application import application_instance as application from newrelic.api.application import application_settings @@ -43,6 +41,9 @@ SYNTHETICS_RESOURCE_ID = "09845779-16ef-4fa7-b7f2-44da8e62931c" SYNTHETICS_JOB_ID = "8c7dd3ba-4933-4cbb-b1ed-b62f511782f4" SYNTHETICS_MONITOR_ID = "dc452ae9-1a93-4ab5-8a33-600521e9cd00" +SYNTHETICS_TYPE = "scheduled" +SYNTHETICS_INITIATOR = "graphql" +SYNTHETICS_ATTRIBUTES = {"exampleAttribute": "1"} ERR_MESSAGE = "Transaction had bad value" ERROR = ValueError(ERR_MESSAGE) @@ -83,7 +84,7 @@ def test_transaction_error_event_lotsa_attributes(): "err_message": ERR_MESSAGE, "external": "2", "db": "2", - "mod_wsgi.queue_start": ("t=%r" % time.time()), + "mod_wsgi.queue_start": (f"t={time.time()!r}"), "SERVER_PORT": "8888", } response = fully_featured_application.get("/", extra_environ=test_environ) @@ -135,6 +136,9 @@ def test_transaction_error_cross_agent(): "nr.syntheticsResourceId": SYNTHETICS_RESOURCE_ID, "nr.syntheticsJobId": SYNTHETICS_JOB_ID, "nr.syntheticsMonitorId": SYNTHETICS_MONITOR_ID, + "nr.syntheticsType": SYNTHETICS_TYPE, + "nr.syntheticsInitiator": SYNTHETICS_INITIATOR, + "nr.syntheticsExampleAttribute": "1", } @@ -144,12 +148,15 @@ def test_transaction_error_with_synthetics(): "err_message": ERR_MESSAGE, } settings = application_settings() - headers = make_synthetics_header( + headers = make_synthetics_headers( + settings.encoding_key, settings.trusted_account_ids[0], SYNTHETICS_RESOURCE_ID, SYNTHETICS_JOB_ID, SYNTHETICS_MONITOR_ID, - settings.encoding_key, + SYNTHETICS_TYPE, + SYNTHETICS_INITIATOR, + SYNTHETICS_ATTRIBUTES, ) response = fully_featured_application.get("/", headers=headers, extra_environ=test_environ) @@ -240,11 +247,14 @@ def test_error_event_outside_transaction(): @reset_core_stats_engine() @validate_non_transaction_error_event(_intrinsic_attributes, required_user=_err_params) def test_error_event_with_params_outside_transaction(): + attrs = {"drop-me": None} + attrs.update(_err_params) + try: raise outside_error except ErrorEventOutsideTransactionError: app = application() - notice_error(sys.exc_info(), attributes=_err_params, application=app) + notice_error(sys.exc_info(), attributes=attrs, application=app) @reset_core_stats_engine() diff --git a/tests/agent_features/test_error_group_callback.py b/tests/agent_features/test_error_group_callback.py index 742391162c..c739e26826 100644 --- a/tests/agent_features/test_error_group_callback.py +++ b/tests/agent_features/test_error_group_callback.py @@ -12,35 +12,40 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys import threading import traceback -import sys import pytest - from testing_support.fixtures import ( override_application_settings, reset_core_stats_engine, +) +from testing_support.validators.validate_error_event_attributes import ( validate_error_event_attributes, +) +from testing_support.validators.validate_error_event_attributes_outside_transaction import ( validate_error_event_attributes_outside_transaction, - validate_error_trace_attributes_outside_transaction, ) from testing_support.validators.validate_error_trace_attributes import ( validate_error_trace_attributes, ) +from testing_support.validators.validate_error_trace_attributes_outside_transaction import ( + validate_error_trace_attributes_outside_transaction, +) from newrelic.api.application import application_instance as application from newrelic.api.background_task import background_task +from newrelic.api.settings import set_error_group_callback from newrelic.api.time_trace import notice_error from newrelic.api.transaction import current_transaction -from newrelic.api.settings import set_error_group_callback from newrelic.api.web_transaction import web_transaction from newrelic.common.object_names import callable_name - _callback_called = threading.Event() _truncated_value = "A" * 300 + def error_group_callback(exc, data): _callback_called.set() @@ -64,12 +69,9 @@ def test_clear_error_group_callback(): assert settings.error_collector.error_group_callback is None, "Failed to clear callback." -@pytest.mark.parametrize("callback,accepted", [ - (error_group_callback, True), - (lambda x, y: None, True), - (None, False), - ("string", False) -]) +@pytest.mark.parametrize( + "callback,accepted", [(error_group_callback, True), (lambda x, y: None, True), (None, False), ("string", False)] +) def test_set_error_group_callback(callback, accepted): try: set_error_group_callback(callback) @@ -82,15 +84,19 @@ def test_set_error_group_callback(callback, accepted): set_error_group_callback(None) -@pytest.mark.parametrize("exc_class,group_name,high_security", [ - (ValueError, "value", False), - (ValueError, "value", True), - (TypeError, None, False), - (RuntimeError, None, False), - (IndexError, None, False), - (LookupError, None, False), - (ZeroDivisionError, _truncated_value[:255], False), -], ids=("standard", "high-security", "empty-string", "None-value", "list-type", "int-type", "truncated-value")) +@pytest.mark.parametrize( + "exc_class,group_name,high_security", + [ + (ValueError, "value", False), + (ValueError, "value", True), + (TypeError, None, False), + (RuntimeError, None, False), + (IndexError, None, False), + (LookupError, None, False), + (ZeroDivisionError, _truncated_value[:255], False), + ], + ids=("standard", "high-security", "empty-string", "None-value", "list-type", "int-type", "truncated-value"), +) @reset_core_stats_engine() def test_error_group_name_callback(exc_class, group_name, high_security): _callback_called.clear() @@ -102,9 +108,7 @@ def test_error_group_name_callback(exc_class, group_name, high_security): exact = None forgone = {"user": [], "intrinsic": [], "agent": ["error.group.name"]} - @validate_error_trace_attributes( - callable_name(exc_class), forgone_params=forgone, exact_attrs=exact - ) + @validate_error_trace_attributes(callable_name(exc_class), forgone_params=forgone, exact_attrs=exact) @validate_error_event_attributes(forgone_params=forgone, exact_attrs=exact) @override_application_settings({"high_security": high_security}) @background_task() @@ -124,15 +128,19 @@ def _test(): set_error_group_callback(None) -@pytest.mark.parametrize("exc_class,group_name,high_security", [ - (ValueError, "value", False), - (ValueError, "value", True), - (TypeError, None, False), - (RuntimeError, None, False), - (IndexError, None, False), - (LookupError, None, False), - (ZeroDivisionError, _truncated_value[:255], False), -], ids=("standard", "high-security", "empty-string", "None-value", "list-type", "int-type", "truncated-value")) +@pytest.mark.parametrize( + "exc_class,group_name,high_security", + [ + (ValueError, "value", False), + (ValueError, "value", True), + (TypeError, None, False), + (RuntimeError, None, False), + (IndexError, None, False), + (LookupError, None, False), + (ZeroDivisionError, _truncated_value[:255], False), + ], + ids=("standard", "high-security", "empty-string", "None-value", "list-type", "int-type", "truncated-value"), +) @reset_core_stats_engine() def test_error_group_name_callback_outside_transaction(exc_class, group_name, high_security): _callback_called.clear() @@ -155,7 +163,7 @@ def _test(): except Exception: app = application() notice_error(application=app) - + assert _callback_called.is_set() try: @@ -165,11 +173,22 @@ def _test(): set_error_group_callback(None) -@pytest.mark.parametrize("transaction_decorator", [ - background_task(name="TestBackgroundTask"), - web_transaction(name="TestWebTransaction", host="localhost", port=1234, request_method="GET", request_path="/", headers=[],), - None, -], ids=("background_task", "web_transation", "outside_transaction")) +@pytest.mark.parametrize( + "transaction_decorator", + [ + background_task(name="TestBackgroundTask"), + web_transaction( + name="TestWebTransaction", + host="localhost", + port=1234, + request_method="GET", + request_path="/", + headers=[], + ), + None, + ], + ids=("background_task", "web_transation", "outside_transaction"), +) @reset_core_stats_engine() def test_error_group_name_callback_attributes(transaction_decorator): callback_errors = [] @@ -178,6 +197,7 @@ def test_error_group_name_callback_attributes(transaction_decorator): def callback(error, data): def _callback(): import types + _data.append(data) txn = current_transaction() @@ -191,23 +211,23 @@ def _callback(): # All attributes should always be included, but set to None when not relevant. if txn is None: # Outside transaction assert data["transactionName"] is None - assert data["custom_params"] == {'notice_error_attribute': 1} + assert data["custom_params"] == {"notice_error_attribute": 1} assert data["response.status"] is None assert data["request.method"] is None assert data["request.uri"] is None elif txn.background_task: # Background task assert data["transactionName"] == "TestBackgroundTask" - assert data["custom_params"] == {'notice_error_attribute': 1, 'txn_attribute': 2} + assert data["custom_params"] == {"notice_error_attribute": 1, "txn_attribute": 2} assert data["response.status"] is None assert data["request.method"] is None assert data["request.uri"] is None else: # Web transaction assert data["transactionName"] == "TestWebTransaction" - assert data["custom_params"] == {'notice_error_attribute': 1, 'txn_attribute': 2} + assert data["custom_params"] == {"notice_error_attribute": 1, "txn_attribute": 2} assert data["response.status"] == 200 assert data["request.method"] == "GET" assert data["request.uri"] == "/" - + try: _callback() except Exception: @@ -225,8 +245,8 @@ def _test(): except Exception: app = application() if transaction_decorator is None else None # Only set outside transaction notice_error(application=app, attributes={"notice_error_attribute": 1}) - - assert not callback_errors, "Callback inputs failed to validate.\nerror: %s\ndata: %s" % (traceback.format_exception(*callback_errors[0]), str(_data[0])) + + assert not callback_errors, f"Callback inputs failed to validate.\nerror: {traceback.format_exception(*callback_errors[0])}\ndata: {str(_data[0])}" if transaction_decorator is not None: _test = transaction_decorator(_test) # Manually decorate test function diff --git a/tests/agent_features/test_exception_messages.py b/tests/agent_features/test_exception_messages.py index e9944f9205..c9effea4ca 100644 --- a/tests/agent_features/test_exception_messages.py +++ b/tests/agent_features/test_exception_messages.py @@ -13,130 +13,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -import six -import pytest +from testing_support.fixtures import ( + reset_core_stats_engine, + validate_application_exception_message, + validate_transaction_exception_message, +) from newrelic.api.application import application_instance as application from newrelic.api.background_task import background_task from newrelic.api.time_trace import notice_error -from testing_support.fixtures import (validate_transaction_exception_message, - set_default_encoding, validate_application_exception_message, - reset_core_stats_engine) -UNICODE_MESSAGE = u'I💜🐍' -UNICODE_ENGLISH = u'I love python' +UNICODE_MESSAGE = 'I💜🐍' +UNICODE_ENGLISH = 'I love python' BYTES_ENGLISH = b'I love python' BYTES_UTF8_ENCODED = b'I\xf0\x9f\x92\x9c\xf0\x9f\x90\x8d' -INCORRECTLY_DECODED_BYTES_PY2 = u'I\u00f0\u009f\u0092\u009c\u00f0\u009f\u0090\u008d' -INCORRECTLY_DECODED_BYTES_PY3 = u"b'I\\xf0\\x9f\\x92\\x9c\\xf0\\x9f\\x90\\x8d'" -# =================== Exception messages during transaction ==================== - -# ---------------- Python 2 - -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@set_default_encoding('ascii') -@validate_transaction_exception_message(UNICODE_MESSAGE) -@background_task() -def test_py2_transaction_exception_message_unicode(): - """Assert unicode message when using non-ascii characters is preserved, - with sys default encoding""" - try: - raise ValueError(UNICODE_MESSAGE) - except ValueError: - notice_error() +INCORRECTLY_DECODED_BYTES_PY2 = 'I\u00f0\u009f\u0092\u009c\u00f0\u009f\u0090\u008d' +INCORRECTLY_DECODED_BYTES_PY3 = "b'I\\xf0\\x9f\\x92\\x9c\\xf0\\x9f\\x90\\x8d'" -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@set_default_encoding('ascii') -@validate_transaction_exception_message(UNICODE_ENGLISH) -@background_task() -def test_py2_transaction_exception_message_unicode_english(): - """Assert unicode message when using ascii compatible characters preserved, - with sys default encoding""" - try: - raise ValueError(UNICODE_ENGLISH) - except ValueError: - notice_error() - -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@set_default_encoding('ascii') -@validate_transaction_exception_message(UNICODE_ENGLISH) -@background_task() -def test_py2_transaction_exception_message_bytes_english(): - """Assert byte string of ascii characters decodes sensibly""" - try: - raise ValueError(BYTES_ENGLISH) - except ValueError: - notice_error() - -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@set_default_encoding('ascii') -@validate_transaction_exception_message(INCORRECTLY_DECODED_BYTES_PY2) -@background_task() -def test_py2_transaction_exception_message_bytes_non_english(): - """Assert known situation where (explicitly) utf-8 encoded byte string gets - mangled when default sys encoding is ascii. THIS TEST ASSERTS THAT THE - MESSAGE IS WRONG. We do not expect it to work now, or in the future. - """ - try: - raise ValueError(BYTES_UTF8_ENCODED) - except ValueError: - notice_error() - -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@set_default_encoding('ascii') -@validate_transaction_exception_message(INCORRECTLY_DECODED_BYTES_PY2) -@background_task() -def test_py2_transaction_exception_message_bytes_implicit_encoding_non_english(): - """Assert known situation where (implicitly) utf-8 encoded byte string gets - mangled when default sys encoding is ascii. THIS TEST ASSERTS THAT THE - MESSAGE IS WRONG. We do not expect it to work now, or in the future. - """ - try: - - # Bytes literal with non-ascii compatible characters only allowed in - # python 2 - - raise ValueError('I💜🐍') - except ValueError: - notice_error() - -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@set_default_encoding('utf-8') -@validate_transaction_exception_message(UNICODE_MESSAGE) -@background_task() -def test_py2_transaction_exception_message_unicode_utf8_encoding(): - """Assert unicode error message is preserved with sys non-default utf-8 - encoding - """ - try: - raise ValueError(UNICODE_MESSAGE) - except ValueError: - notice_error() - -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@set_default_encoding('utf-8') -@validate_transaction_exception_message(UNICODE_MESSAGE) -@background_task() -def test_py2_transaction_exception_message_bytes_utf8_encoding_non_english(): - """Assert utf-8 encoded byte produces correct exception message when sys - encoding is also utf-8. - """ - try: - - # Bytes literal with non-ascii compatible characters only allowed in - # python 2 - - raise ValueError('I💜🐍') - except ValueError: - notice_error() - -# ---------------- Python 3 +# =================== Exception messages during transaction ==================== -@pytest.mark.skipif(six.PY2, reason="Testing Python 3 string behavior") @validate_transaction_exception_message(UNICODE_MESSAGE) @background_task() -def test_py3_transaction_exception_message_bytes_non_english_unicode(): +def test_transaction_exception_message_bytes_non_english_unicode(): """Assert (native) unicode exception message is preserved when when non-ascii compatible characters present""" try: @@ -144,10 +43,10 @@ def test_py3_transaction_exception_message_bytes_non_english_unicode(): except ValueError: notice_error() -@pytest.mark.skipif(six.PY2, reason="Testing Python 3 string behavior") + @validate_transaction_exception_message(UNICODE_ENGLISH) @background_task() -def test_py3_transaction_exception_message_unicode_english(): +def test_transaction_exception_message_unicode_english(): """Assert (native) unicode exception message is preserved, when characters are ascii-compatible""" try: @@ -155,10 +54,10 @@ def test_py3_transaction_exception_message_unicode_english(): except ValueError: notice_error() -@pytest.mark.skipif(six.PY2, reason="Testing Python 3 string behavior") + @validate_transaction_exception_message(INCORRECTLY_DECODED_BYTES_PY3) @background_task() -def test_py3_transaction_exception_message_bytes_non_english(): +def test_transaction_exception_message_bytes_non_english(): """An issue can occur if you cast from bytes to a string in python 3 (that is using str(), not using encode/decode methods). This is because all characters in bytes are literals, no implicit @@ -173,118 +72,9 @@ def test_py3_transaction_exception_message_bytes_non_english(): # =================== Exception messages outside transaction ==================== -# ---------------- Python 2 - -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@reset_core_stats_engine() -@set_default_encoding('ascii') -@validate_application_exception_message(UNICODE_MESSAGE) -def test_py2_application_exception_message_unicode(): - """Assert unicode message when using non-ascii characters is preserved, - with sys default encoding""" - try: - raise ValueError(UNICODE_MESSAGE) - except ValueError: - app = application() - notice_error(application=app) - -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@reset_core_stats_engine() -@set_default_encoding('ascii') -@validate_application_exception_message(UNICODE_ENGLISH) -def test_py2_application_exception_message_unicode_english(): - """Assert unicode message when using ascii compatible characters preserved, - with sys default encoding""" - try: - raise ValueError(UNICODE_ENGLISH) - except ValueError: - app = application() - notice_error(application=app) - -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@reset_core_stats_engine() -@set_default_encoding('ascii') -@validate_application_exception_message(UNICODE_ENGLISH) -def test_py2_application_exception_message_bytes_english(): - """Assert byte string of ascii characters decodes sensibly""" - try: - raise ValueError(BYTES_ENGLISH) - except ValueError: - app = application() - notice_error(application=app) - -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@reset_core_stats_engine() -@set_default_encoding('ascii') -@validate_application_exception_message(INCORRECTLY_DECODED_BYTES_PY2) -def test_py2_application_exception_message_bytes_non_english(): - """Assert known situation where (explicitly) utf-8 encoded byte string gets - mangled when default sys encoding is ascii. THIS TEST ASSERTS THAT THE - MESSAGE IS WRONG. We do not expect it to work now, or in the future. - """ - try: - raise ValueError(BYTES_UTF8_ENCODED) - except ValueError: - app = application() - notice_error(application=app) - -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@reset_core_stats_engine() -@set_default_encoding('ascii') -@validate_application_exception_message(INCORRECTLY_DECODED_BYTES_PY2) -def test_py2_application_exception_message_bytes_implicit_encoding_non_english(): - """Assert known situation where (implicitly) utf-8 encoded byte string gets - mangled when default sys encoding is ascii. THIS TEST ASSERTS THAT THE - MESSAGE IS WRONG. We do not expect it to work now, or in the future. - """ - try: - - # Bytes literal with non-ascii compatible characters only allowed in - # python 2 - - raise ValueError('I💜🐍') - except ValueError: - app = application() - notice_error(application=app) - -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@reset_core_stats_engine() -@set_default_encoding('utf-8') -@validate_application_exception_message(UNICODE_MESSAGE) -def test_py2_application_exception_message_unicode_utf8_encoding(): - """Assert unicode error message is preserved with sys non-default utf-8 - encoding - """ - try: - raise ValueError(UNICODE_MESSAGE) - except ValueError: - app = application() - notice_error(application=app) - -@pytest.mark.skipif(six.PY3, reason="Testing Python 2 string behavior") -@reset_core_stats_engine() -@set_default_encoding('utf-8') -@validate_application_exception_message(UNICODE_MESSAGE) -def test_py2_application_exception_message_bytes_utf8_encoding_non_english(): - """Assert utf-8 encoded byte produces correct exception message when sys - encoding is also utf-8. - """ - try: - - # Bytes literal with non-ascii compatible characters only allowed in - # python 2 - - raise ValueError('I💜🐍') - except ValueError: - app = application() - notice_error(application=app) - -# ---------------- Python 3 - -@pytest.mark.skipif(six.PY2, reason="Testing Python 3 string behavior") @reset_core_stats_engine() @validate_application_exception_message(UNICODE_MESSAGE) -def test_py3_application_exception_message_bytes_non_english_unicode(): +def test_application_exception_message_bytes_non_english_unicode(): """Assert (native) unicode exception message is preserved when when non-ascii compatible characters present""" try: @@ -293,10 +83,10 @@ def test_py3_application_exception_message_bytes_non_english_unicode(): app = application() notice_error(application=app) -@pytest.mark.skipif(six.PY2, reason="Testing Python 3 string behavior") + @reset_core_stats_engine() @validate_application_exception_message(UNICODE_ENGLISH) -def test_py3_application_exception_message_unicode_english(): +def test_application_exception_message_unicode_english(): """Assert (native) unicode exception message is preserved, when characters are ascii-compatible""" try: @@ -305,10 +95,10 @@ def test_py3_application_exception_message_unicode_english(): app = application() notice_error(application=app) -@pytest.mark.skipif(six.PY2, reason="Testing Python 3 string behavior") + @reset_core_stats_engine() @validate_application_exception_message(INCORRECTLY_DECODED_BYTES_PY3) -def test_py3_application_exception_message_bytes_non_english(): +def test_application_exception_message_bytes_non_english(): """It really makes a mess of things when you cast from bytes to a string in python 3 (that is using str(), not using encode/decode methods). This is because all characters in bytes are literals, no implicit @@ -321,3 +111,15 @@ def test_py3_application_exception_message_bytes_non_english(): except ValueError: app = application() notice_error(application=app) + + +@reset_core_stats_engine() +@validate_application_exception_message("My custom message") +def test_nr_message_exception_attr_override(): + """Override the message using the _nr_message attribute.""" + try: + raise ValueError("Original error message") + except ValueError as e: + e._nr_message = "My custom message" + app = application() + notice_error(application=app) diff --git a/tests/agent_features/test_high_security_mode.py b/tests/agent_features/test_high_security_mode.py index dad7edc295..17b4d97f0a 100644 --- a/tests/agent_features/test_high_security_mode.py +++ b/tests/agent_features/test_high_security_mode.py @@ -22,10 +22,11 @@ override_generic_settings, reset_core_stats_engine, validate_attributes_complete, + validate_request_params_omitted, +) +from testing_support.validators.validate_custom_event import ( validate_custom_event_count, validate_custom_event_in_application_stats_engine, - validate_request_params_omitted, - validate_tt_segment_params, ) from testing_support.validators.validate_custom_parameters import ( validate_custom_parameters, @@ -36,6 +37,9 @@ from testing_support.validators.validate_transaction_errors import ( validate_transaction_errors, ) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.application import application_instance as application from newrelic.api.background_task import background_task @@ -77,8 +81,11 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_events_value.enabled": True, + "ai_monitoring.enabled": True, }, { "high_security": False, @@ -86,8 +93,11 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_events_value.enabled": True, + "ai_monitoring.enabled": True, }, { "high_security": False, @@ -95,8 +105,11 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_events_value.enabled": False, + "ai_monitoring.enabled": False, }, { "high_security": False, @@ -104,8 +117,11 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "off", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_events_value.enabled": False, + "ai_monitoring.enabled": False, }, ] @@ -116,8 +132,11 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": False, + "machine_learning.inference_events_value.enabled": False, + "ai_monitoring.enabled": False, }, { "high_security": True, @@ -125,8 +144,11 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": False, + "machine_learning.inference_events_value.enabled": False, + "ai_monitoring.enabled": False, }, { "high_security": True, @@ -134,8 +156,11 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": False, + "machine_learning.inference_events_value.enabled": False, + "ai_monitoring.enabled": False, }, { "high_security": True, @@ -143,8 +168,11 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_events_value.enabled": True, + "ai_monitoring.enabled": True, }, { "high_security": True, @@ -152,8 +180,11 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_events_value.enabled": True, + "ai_monitoring.enabled": True, }, { "high_security": True, @@ -161,8 +192,11 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "off", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": True, + "machine_learning.inference_events_value.enabled": True, + "ai_monitoring.enabled": True, }, { "high_security": True, @@ -170,8 +204,11 @@ def test_hsm_configuration_default(): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "message_tracer.segment_parameters_enabled": False, "application_logging.forwarding.enabled": True, + "machine_learning.inference_events_value.enabled": True, + "ai_monitoring.enabled": True, }, ] @@ -194,17 +231,25 @@ def test_local_config_file_override_hsm_disabled(settings): original_record_sql = settings.transaction_tracer.record_sql original_strip_messages = settings.strip_exception_messages.enabled original_custom_events = settings.custom_insights_events.enabled + original_ml_events = settings.ml_insights_events.enabled original_message_segment_params_enabled = settings.message_tracer.segment_parameters_enabled original_application_logging_forwarding_enabled = settings.application_logging.forwarding.enabled - + original_machine_learning_inference_event_value_enabled = settings.machine_learning.inference_events_value.enabled + original_ai_monitoring_enabled = settings.ai_monitoring.enabled apply_local_high_security_mode_setting(settings) assert settings.capture_params == original_capture_params assert settings.transaction_tracer.record_sql == original_record_sql assert settings.strip_exception_messages.enabled == original_strip_messages assert settings.custom_insights_events.enabled == original_custom_events + assert settings.ml_insights_events.enabled == original_ml_events assert settings.message_tracer.segment_parameters_enabled == original_message_segment_params_enabled assert settings.application_logging.forwarding.enabled == original_application_logging_forwarding_enabled + assert ( + settings.machine_learning.inference_events_value.enabled + == original_machine_learning_inference_event_value_enabled + ) + assert settings.ai_monitoring.enabled == original_ai_monitoring_enabled @parameterize_hsm_local_config(_hsm_local_config_file_settings_enabled) @@ -215,8 +260,11 @@ def test_local_config_file_override_hsm_enabled(settings): assert settings.transaction_tracer.record_sql in ("off", "obfuscated") assert settings.strip_exception_messages.enabled assert settings.custom_insights_events.enabled is False + assert settings.ml_insights_events.enabled is False assert settings.message_tracer.segment_parameters_enabled is False assert settings.application_logging.forwarding.enabled is False + assert settings.machine_learning.inference_events_value.enabled is False + assert settings.ai_monitoring.enabled is False _server_side_config_settings_hsm_disabled = [ @@ -227,7 +275,10 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_events_value.enabled": False, + "ai_monitoring.enabled": False, }, { "agent_config": { @@ -235,7 +286,10 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_events_value.enabled": True, + "ai_monitoring.enabled": True, }, }, ), @@ -246,7 +300,10 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_events_value.enabled": True, + "ai_monitoring.enabled": True, }, { "agent_config": { @@ -254,7 +311,10 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "off", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_events_value.enabled": False, + "ai_monitoring.enabled": False, }, }, ), @@ -268,7 +328,10 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_events_value.enabled": False, + "ai_monitoring.enabled": False, }, { "high_security": True, @@ -276,13 +339,19 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_events_value.enabled": False, + "ai_monitoring.enabled": False, "agent_config": { "capture_params": False, "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_events_value.enabled": False, + "ai_monitoring.enabled": False, }, }, ), @@ -293,7 +362,10 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_events_value.enabled": False, + "ai_monitoring.enabled": False, }, { "high_security": True, @@ -301,13 +373,19 @@ def test_local_config_file_override_hsm_enabled(settings): "transaction_tracer.record_sql": "obfuscated", "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, "application_logging.forwarding.enabled": False, + "machine_learning.inference_events_value.enabled": False, + "ai_monitoring.enabled": False, "agent_config": { "capture_params": True, "transaction_tracer.record_sql": "raw", "strip_exception_messages.enabled": False, "custom_insights_events.enabled": True, + "ml_insights_events.enabled": True, "application_logging.forwarding.enabled": True, + "machine_learning.inference_events_value.enabled": True, + "ai_monitoring.enabled": True, }, }, ), @@ -327,7 +405,10 @@ def test_remote_config_fixups_hsm_disabled(local_settings, server_settings): original_record_sql = agent_config["transaction_tracer.record_sql"] original_strip_messages = agent_config["strip_exception_messages.enabled"] original_custom_events = agent_config["custom_insights_events.enabled"] + original_ml_events = agent_config["ml_insights_events.enabled"] original_log_forwarding = agent_config["application_logging.forwarding.enabled"] + original_machine_learning_events = agent_config["machine_learning.inference_events_value.enabled"] + original_ai_monitoring = agent_config["ai_monitoring.enabled"] _settings = global_settings() settings = override_generic_settings(_settings, local_settings)(AgentProtocol._apply_high_security_mode_fixups)( @@ -342,7 +423,10 @@ def test_remote_config_fixups_hsm_disabled(local_settings, server_settings): assert agent_config["transaction_tracer.record_sql"] == original_record_sql assert agent_config["strip_exception_messages.enabled"] == original_strip_messages assert agent_config["custom_insights_events.enabled"] == original_custom_events + assert agent_config["ml_insights_events.enabled"] == original_ml_events assert agent_config["application_logging.forwarding.enabled"] == original_log_forwarding + assert agent_config["machine_learning.inference_events_value.enabled"] == original_machine_learning_events + assert agent_config["ai_monitoring.enabled"] == original_ai_monitoring @pytest.mark.parametrize("local_settings,server_settings", _server_side_config_settings_hsm_enabled) @@ -364,13 +448,19 @@ def test_remote_config_fixups_hsm_enabled(local_settings, server_settings): assert "transaction_tracer.record_sql" not in settings assert "strip_exception_messages.enabled" not in settings assert "custom_insights_events.enabled" not in settings + assert "ml_insights_events.enabled" not in settings assert "application_logging.forwarding.enabled" not in settings + assert "machine_learning.inference_events_value.enabled" not in settings + assert "ai_monitoring.enabled" not in settings assert "capture_params" not in agent_config assert "transaction_tracer.record_sql" not in agent_config assert "strip_exception_messages.enabled" not in agent_config assert "custom_insights_events.enabled" not in agent_config + assert "ml_insights_events.enabled" not in agent_config assert "application_logging.forwarding.enabled" not in agent_config + assert "machine_learning.inference_events_value.enabled" not in agent_config + assert "ai_monitoring.enabled" not in agent_config def test_remote_config_hsm_fixups_server_side_disabled(): @@ -395,6 +485,7 @@ def test_remote_config_hsm_fixups_server_side_disabled(): "high_security": True, "strip_exception_messages.enabled": True, "custom_insights_events.enabled": False, + "ml_insights_events.enabled": False, } @@ -432,7 +523,7 @@ class TestException(Exception): pass -_test_exception_name = "%s:%s" % (__name__, TestException.__name__) +_test_exception_name = f"{__name__}:{TestException.__name__}" @override_application_settings(_test_transaction_settings_hsm_disabled) diff --git a/tests/agent_features/test_ignore_expected_errors.py b/tests/agent_features/test_ignore_expected_errors.py index 93595aa35b..ee26245c5d 100644 --- a/tests/agent_features/test_ignore_expected_errors.py +++ b/tests/agent_features/test_ignore_expected_errors.py @@ -16,8 +16,12 @@ from testing_support.fixtures import ( override_application_settings, reset_core_stats_engine, - validate_error_event_attributes_outside_transaction, validate_error_event_sample_data, +) +from testing_support.validators.validate_error_event_attributes_outside_transaction import ( + validate_error_event_attributes_outside_transaction, +) +from testing_support.validators.validate_error_trace_attributes_outside_transaction import ( validate_error_trace_attributes_outside_transaction, ) from testing_support.validators.validate_time_metrics_outside_transaction import ( diff --git a/tests/agent_features/test_lambda_handler.py b/tests/agent_features/test_lambda_handler.py index 40b6944072..8658a0bca3 100644 --- a/tests/agent_features/test_lambda_handler.py +++ b/tests/agent_features/test_lambda_handler.py @@ -92,7 +92,7 @@ def handler(event, context): } -class Context(object): +class Context(): aws_request_id = "cookies" invoked_function_arn = "arn" function_name = "cats" @@ -100,6 +100,8 @@ class Context(object): memory_limit_in_mb = 128 +# The lambda_hander has been deprecated for 3+ years +@pytest.mark.skip(reason="The lambda_handler has been deprecated") @pytest.mark.parametrize("is_cold", (False, True)) def test_lambda_transaction_attributes(is_cold, monkeypatch): # setup copies of the attribute lists for this test only @@ -139,6 +141,8 @@ def _test(): _test() +# The lambda_hander has been deprecated for 3+ years +@pytest.mark.skip(reason="The lambda_handler has been deprecated") @validate_transaction_trace_attributes(_expected_attributes) @validate_transaction_event_attributes(_expected_attributes) @override_application_settings(_override_settings) @@ -193,6 +197,8 @@ def test_lambda_malformed_request_headers(): } +# The lambda_hander has been deprecated for 3+ years +@pytest.mark.skip(reason="The lambda_handler has been deprecated") @validate_transaction_trace_attributes(_malformed_response_attributes) @validate_transaction_event_attributes(_malformed_response_attributes) @override_application_settings(_override_settings) @@ -229,6 +235,8 @@ def handler(event, context): } +# The lambda_hander has been deprecated for 3+ years +@pytest.mark.skip(reason="The lambda_handler has been deprecated") @validate_transaction_trace_attributes(_no_status_code_response) @validate_transaction_event_attributes(_no_status_code_response) @override_application_settings(_override_settings) @@ -253,6 +261,8 @@ def handler(event, context): ) +# The lambda_hander has been deprecated for 3+ years +@pytest.mark.skip(reason="The lambda_handler has been deprecated") @pytest.mark.parametrize("event,arn", ((empty_event, None), (firehose_event, "arn:aws:kinesis:EXAMPLE"))) def test_lambda_event_source_arn_attribute(event, arn): if arn is None: @@ -285,6 +295,8 @@ def _test(): _test() +# The lambda_hander has been deprecated for 3+ years +@pytest.mark.skip(reason="The lambda_handler has been deprecated") @pytest.mark.parametrize( "api", ( diff --git a/tests/agent_features/test_llm_custom_attributes.py b/tests/agent_features/test_llm_custom_attributes.py new file mode 100644 index 0000000000..1f02c231c0 --- /dev/null +++ b/tests/agent_features/test_llm_custom_attributes.py @@ -0,0 +1,50 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.api.transaction import current_transaction + + +@background_task() +def test_llm_custom_attributes(): + transaction = current_transaction() + with WithLlmCustomAttributes({"test": "attr", "test1": "attr1"}): + assert transaction._llm_context_attrs == {"llm.test": "attr", "llm.test1": "attr1"} + + assert not hasattr(transaction, "_llm_context_attrs") + + +@pytest.mark.parametrize("context_attrs", (None, "not-a-dict")) +@background_task() +def test_llm_custom_attributes_no_attrs(context_attrs): + transaction = current_transaction() + + with pytest.raises(TypeError): + with WithLlmCustomAttributes(context_attrs): + pass + + assert not hasattr(transaction, "_llm_context_attrs") + + +@background_task() +def test_llm_custom_attributes_prefixed_attrs(): + transaction = current_transaction() + with WithLlmCustomAttributes({"llm.test": "attr", "test1": "attr1"}): + # Validate API does not prefix attributes that already begin with "llm." + assert transaction._llm_context_attrs == {"llm.test": "attr", "llm.test1": "attr1"} + + assert not hasattr(transaction, "_llm_context_attrs") diff --git a/tests/agent_features/test_llm_token_count_callback.py b/tests/agent_features/test_llm_token_count_callback.py new file mode 100644 index 0000000000..3d923fe84b --- /dev/null +++ b/tests/agent_features/test_llm_token_count_callback.py @@ -0,0 +1,69 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from newrelic.api.application import application_instance as application +from newrelic.api.ml_model import set_llm_token_count_callback + + +def test_unset_llm_token_count_callback(): + settings = application().settings + + set_llm_token_count_callback(lambda model, content: 45) + + assert callable(settings.ai_monitoring.llm_token_count_callback) + + set_llm_token_count_callback(None) + + assert settings.ai_monitoring.llm_token_count_callback is None + + +@pytest.mark.parametrize( + "set_args,call_args,expected_value", + [ + ((lambda model, content: 45,), ("model", "content"), 45), + ((lambda model, content: 45, application().settings), ("model", "content"), 45), + ((lambda model, content: 1.1,), ("model", "content"), None), + ((lambda model, content: -1,), ("model", "content"), None), + ((lambda model, content: 45,), (None, "content"), None), + ((lambda model, content: 45,), ("model", None), None), + ], +) +def test_set_llm_token_count_callback(set_args, call_args, expected_value): + settings = application().settings + + set_llm_token_count_callback(*set_args) + + assert settings.ai_monitoring.llm_token_count_callback(*call_args) == expected_value + + +def test_exception_in_user_callback(): + settings = application().settings + + def user_exc(): + raise TypeError() + + set_llm_token_count_callback(user_exc) + + with pytest.raises(TypeError): + settings.ai_monitoring.llm_token_count_callback("model", "content") + + +def test_with_application_not_active(): + settings = application(activate=False).settings + + set_llm_token_count_callback(lambda model, content: 45) + + assert settings.ai_monitoring.llm_token_count_callback("model", "content") == 45 diff --git a/tests/agent_features/test_log_events.py b/tests/agent_features/test_log_events.py index bb173d6c4e..9a619d8de4 100644 --- a/tests/agent_features/test_log_events.py +++ b/tests/agent_features/test_log_events.py @@ -12,14 +12,48 @@ # See the License for the specific language governing permissions and # limitations under the License. -from newrelic.api.background_task import background_task -from newrelic.api.time_trace import current_trace -from newrelic.api.transaction import current_transaction, record_log_event, ignore_transaction -from testing_support.fixtures import override_application_settings, reset_core_stats_engine +import pytest +from testing_support.fixtures import ( + override_application_settings, + reset_core_stats_engine, +) from testing_support.validators.validate_log_event_count import validate_log_event_count -from testing_support.validators.validate_log_event_count_outside_transaction import validate_log_event_count_outside_transaction +from testing_support.validators.validate_log_event_count_outside_transaction import ( + validate_log_event_count_outside_transaction, +) from testing_support.validators.validate_log_events import validate_log_events -from testing_support.validators.validate_log_events_outside_transaction import validate_log_events_outside_transaction +from testing_support.validators.validate_log_events_outside_transaction import ( + validate_log_events_outside_transaction, +) + +from newrelic.api.background_task import background_task +from newrelic.api.time_trace import current_trace +from newrelic.api.transaction import ( + current_transaction, + ignore_transaction, + record_log_event, +) +from newrelic.core.config import _parse_attributes + + +class NonPrintableObject(): + def __str__(self): + raise RuntimeError("Unable to print object.") + + __repr__ = __str__ + + +class NonSerializableObject(): + def __str__(self): + return f"<{self.__class__.__name__} object>" + + __repr__ = __str__ + + +def combine_dicts(defaults, overrides): + combined = defaults.copy() + combined.update(overrides) + return combined def set_trace_ids(): @@ -31,155 +65,334 @@ def set_trace_ids(): trace.guid = "abcdefgh" -def exercise_record_log_event(message="A"): +def exercise_record_log_event(): set_trace_ids() - record_log_event(message, "ERROR") - -enable_log_forwarding = override_application_settings({"application_logging.forwarding.enabled": True}) + record_log_event("no_other_arguments") + # Attributes with value None should be dropped. + record_log_event("keyword_arguments", timestamp=1234, level="ERROR", attributes={"key": "value", "drop-me": None}) + record_log_event("positional_arguments", "WARNING", 2345, {"key": "value"}) + record_log_event("serialized_attributes", attributes=_serialized_attributes) + record_log_event(None, attributes={"attributes_only": "value"}) + record_log_event({"attributes_only": "value"}) + record_log_event({"message": "dict_message"}) + record_log_event({"message": 123}) + + # Unsent due to message content missing + record_log_event("") + record_log_event(" ") + record_log_event(NonPrintableObject()) + record_log_event({"message": ""}) + record_log_event({"message": NonPrintableObject()}) + record_log_event({"filtered_attribute": "should_be_removed"}) + record_log_event(None) + + +enable_log_forwarding = override_application_settings( + { + "application_logging.forwarding.enabled": True, + "application_logging.forwarding.context_data.enabled": True, + "application_logging.forwarding.context_data.exclude": ["filtered_attribute"], + } +) disable_log_forwarding = override_application_settings({"application_logging.forwarding.enabled": False}) -_common_attributes_service_linking = {"timestamp": None, "hostname": None, "entity.name": "Python Agent Test (agent_features)", "entity.guid": None} +disable_log_attributes = override_application_settings( + {"application_logging.forwarding.enabled": True, "application_logging.forwarding.context_data.enabled": False} +) + +_common_attributes_service_linking = { + "timestamp": None, + "hostname": None, + "entity.name": "Python Agent Test (agent_features)", + "entity.guid": None, +} _common_attributes_trace_linking = {"span.id": "abcdefgh", "trace.id": "abcdefgh12345678"} _common_attributes_trace_linking.update(_common_attributes_service_linking) -_test_record_log_event_inside_transaction_events = [{"message": "A", "level": "ERROR"}] -_test_record_log_event_inside_transaction_events[0].update(_common_attributes_trace_linking) + +_serialized_attributes = { + "str_attr": "Value", + "bytes_attr": b"value", + "int_attr": 1, + "dict_attr": {"key": "value"}, + "non_serializable_attr": NonSerializableObject(), + "non_printable_attr": NonPrintableObject(), + "attr_value_too_long": "*" * 256, + f"attr_name_too_long_{'*' * 237}": "value", + f"attr_name_with_prefix_too_long_{'*' * 220}": "value", +} + +_exercise_record_log_event_events = [ + {"message": "no_other_arguments", "level": "UNKNOWN"}, + {"message": "keyword_arguments", "level": "ERROR", "timestamp": 1234, "context.key": "value"}, + {"message": "positional_arguments", "level": "WARNING", "timestamp": 2345, "context.key": "value"}, + { + "message": "serialized_attributes", + "context.str_attr": "Value", + "context.bytes_attr": b"value", + "context.int_attr": 1, + "context.dict_attr": "{'key': 'value'}", + "context.non_serializable_attr": "", + "context.attr_value_too_long": "*" * 255, + }, + {"context.attributes_only": "value"}, + {"message.attributes_only": "value"}, + {"message": "dict_message"}, + {"message": "123"}, +] +_exercise_record_log_event_inside_transaction_events = [ + combine_dicts(_common_attributes_trace_linking, log) for log in _exercise_record_log_event_events +] +_exercise_record_log_event_outside_transaction_events = [ + combine_dicts(_common_attributes_service_linking, log) for log in _exercise_record_log_event_events +] +_exercise_record_log_event_forgone_attrs = [ + "context.non_printable_attr", + "attr_name_too_long_", + "attr_name_with_prefix_too_long_", +] + + +# Test Log Forwarding + @enable_log_forwarding def test_record_log_event_inside_transaction(): - @validate_log_events(_test_record_log_event_inside_transaction_events) - @validate_log_event_count(1) + @validate_log_events( + _exercise_record_log_event_inside_transaction_events, forgone_attrs=_exercise_record_log_event_forgone_attrs + ) + @validate_log_event_count(len(_exercise_record_log_event_inside_transaction_events)) @background_task() def test(): exercise_record_log_event() - - test() + test() -_test_record_log_event_outside_transaction_events = [{"message": "A", "level": "ERROR"}] -_test_record_log_event_outside_transaction_events[0].update(_common_attributes_service_linking) @enable_log_forwarding @reset_core_stats_engine() def test_record_log_event_outside_transaction(): - @validate_log_events_outside_transaction(_test_record_log_event_outside_transaction_events) - @validate_log_event_count_outside_transaction(1) + @validate_log_events_outside_transaction( + _exercise_record_log_event_outside_transaction_events, forgone_attrs=_exercise_record_log_event_forgone_attrs + ) + @validate_log_event_count_outside_transaction(len(_exercise_record_log_event_outside_transaction_events)) def test(): exercise_record_log_event() test() -_test_record_log_event_unknown_level_inside_transaction_events = [{"message": "A", "level": "UNKNOWN"}] -_test_record_log_event_unknown_level_inside_transaction_events[0].update(_common_attributes_trace_linking) +@enable_log_forwarding +def test_ignored_transaction_logs_not_forwarded(): + @validate_log_event_count(0) + @background_task() + def test(): + ignore_transaction() + exercise_record_log_event() + + test() + + +# Test Message Truncation + +_test_log_event_truncation_events = [{"message": "A" * 32768}] + @enable_log_forwarding -def test_record_log_event_unknown_level_inside_transaction(): - @validate_log_events(_test_record_log_event_unknown_level_inside_transaction_events) +def test_log_event_truncation_inside_transaction(): + @validate_log_events(_test_log_event_truncation_events) @validate_log_event_count(1) @background_task() def test(): - set_trace_ids() - record_log_event("A") - - test() + record_log_event("A" * 33000) + test() -_test_record_log_event_unknown_level_outside_transaction_events = [{"message": "A", "level": "UNKNOWN"}] -_test_record_log_event_unknown_level_outside_transaction_events[0].update(_common_attributes_service_linking) @enable_log_forwarding @reset_core_stats_engine() -def test_record_log_event_unknown_level_outside_transaction(): - @validate_log_events_outside_transaction(_test_record_log_event_unknown_level_outside_transaction_events) +def test_log_event_truncation_outside_transaction(): + @validate_log_events_outside_transaction(_test_log_event_truncation_events) @validate_log_event_count_outside_transaction(1) def test(): - set_trace_ids() - record_log_event("A") + record_log_event("A" * 33000) test() -@enable_log_forwarding -def test_record_log_event_empty_message_inside_transaction(): +# Test Log Forwarding Settings + + +@disable_log_forwarding +def test_disabled_record_log_event_inside_transaction(): @validate_log_event_count(0) @background_task() def test(): - exercise_record_log_event("") - + exercise_record_log_event() + test() -@enable_log_forwarding +@disable_log_forwarding @reset_core_stats_engine() -def test_record_log_event_empty_message_outside_transaction(): +def test_disabled_record_log_event_outside_transaction(): @validate_log_event_count_outside_transaction(0) def test(): - exercise_record_log_event("") + exercise_record_log_event() test() -@enable_log_forwarding -def test_record_log_event_whitespace_inside_transaction(): - @validate_log_event_count(0) +# Test Log Attribute Settings + + +@disable_log_attributes +def test_attributes_disabled_inside_transaction(): + @validate_log_events([{"message": "A"}], forgone_attrs=["context.key"]) + @validate_log_event_count(1) @background_task() def test(): - exercise_record_log_event(" ") + record_log_event("A", attributes={"key": "value"}) test() -@enable_log_forwarding +@disable_log_attributes @reset_core_stats_engine() -def test_record_log_event_whitespace_outside_transaction(): - @validate_log_event_count_outside_transaction(0) +def test_attributes_disabled_outside_transaction(): + @validate_log_events_outside_transaction([{"message": "A"}], forgone_attrs=["context.key"]) + @validate_log_event_count_outside_transaction(1) def test(): - exercise_record_log_event(" ") + record_log_event("A", attributes={"key": "value"}) test() -@enable_log_forwarding -def test_ignored_transaction_logs_not_forwarded(): - @validate_log_event_count(0) +_test_record_log_event_context_attribute_filtering_params = [ + ("", "", "A", True), + ("", "A", "A", False), + ("", "A", "B", True), + ("A B", "*", "A", True), + ("A B", "*", "B", True), + ("A B", "*", "C", False), + ("A B", "C", "A", True), + ("A B", "C", "C", False), + ("A B", "B", "A", True), + ("A B", "B", "B", False), + ("A", "A *", "A", False), + ("A", "A *", "B", False), + ("A*", "", "A", True), + ("A*", "", "AB", True), + ("", "A*", "A", False), + ("", "A*", "B", True), + ("A*", "AB", "AC", True), + ("A*", "AB", "AB", False), + ("AB", "A*", "AB", True), + ("A*", "AB*", "ACB", True), + ("A*", "AB*", "ABC", False), +] + + +@pytest.mark.parametrize("prefix", ("context", "message")) +@pytest.mark.parametrize("include,exclude,attr,expected", _test_record_log_event_context_attribute_filtering_params) +def test_record_log_event_context_attribute_filtering_inside_transaction(include, exclude, attr, expected, prefix): + if expected: + expected_event = {"required_attrs": [f"{prefix}.{attr}"]} + else: + expected_event = {"forgone_attrs": [f"{prefix}.{attr}"]} + + @override_application_settings( + { + "application_logging.forwarding.enabled": True, + "application_logging.forwarding.context_data.enabled": True, + "application_logging.forwarding.context_data.include": _parse_attributes(include), + "application_logging.forwarding.context_data.exclude": _parse_attributes(exclude), + } + ) + @validate_log_events(**expected_event) + @validate_log_event_count(1) @background_task() def test(): - ignore_transaction() - exercise_record_log_event() + if prefix == "context": + record_log_event("A", attributes={attr: 1}) + else: + record_log_event({"message": "A", attr: 1}) test() -_test_log_event_truncation_events = [{"message": "A" * 32768, "level": "ERROR"}] -_test_log_event_truncation_events[0].update(_common_attributes_trace_linking) - -@enable_log_forwarding -def test_log_event_truncation(): - @validate_log_events(_test_log_event_truncation_events) - @validate_log_event_count(1) - @background_task() +@pytest.mark.parametrize("prefix", ("context", "message")) +@pytest.mark.parametrize("include,exclude,attr,expected", _test_record_log_event_context_attribute_filtering_params) +@reset_core_stats_engine() +def test_record_log_event_context_attribute_filtering_outside_transaction(include, exclude, attr, expected, prefix): + if expected: + expected_event = {"required_attrs": [f"{prefix}.{attr}"]} + else: + expected_event = {"forgone_attrs": [f"{prefix}.{attr}"]} + + @override_application_settings( + { + "application_logging.forwarding.enabled": True, + "application_logging.forwarding.context_data.enabled": True, + "application_logging.forwarding.context_data.include": _parse_attributes(include), + "application_logging.forwarding.context_data.exclude": _parse_attributes(exclude), + } + ) + @validate_log_events_outside_transaction(**expected_event) + @validate_log_event_count_outside_transaction(1) def test(): - exercise_record_log_event("A" * 33000) + if prefix == "context": + record_log_event("A", attributes={attr: 1}) + else: + record_log_event({"message": "A", attr: 1}) test() -@disable_log_forwarding -def test_record_log_event_inside_transaction(): - @validate_log_event_count(0) +_test_record_log_event_linking_attribute_no_filtering_params = [ + ("", ""), + ("", "entity.name"), + ("", "*"), +] + + +@pytest.mark.parametrize("include,exclude", _test_record_log_event_linking_attribute_no_filtering_params) +def test_record_log_event_linking_attribute_no_filtering_inside_transaction(include, exclude): + attr = "entity.name" + + @override_application_settings( + { + "application_logging.forwarding.enabled": True, + "application_logging.forwarding.context_data.enabled": True, + "application_logging.forwarding.context_data.include": _parse_attributes(include), + "application_logging.forwarding.context_data.exclude": _parse_attributes(exclude), + } + ) + @validate_log_events(required_attrs=[attr]) + @validate_log_event_count(1) @background_task() def test(): - exercise_record_log_event() - + record_log_event("A") + test() -@disable_log_forwarding +@pytest.mark.parametrize("include,exclude", _test_record_log_event_linking_attribute_no_filtering_params) @reset_core_stats_engine() -def test_record_log_event_outside_transaction(): - @validate_log_event_count_outside_transaction(0) +def test_record_log_event_linking_attribute_filtering_outside_transaction(include, exclude): + attr = "entity.name" + + @override_application_settings( + { + "application_logging.forwarding.enabled": True, + "application_logging.forwarding.context_data.enabled": True, + "application_logging.forwarding.context_data.include": _parse_attributes(include), + "application_logging.forwarding.context_data.exclude": _parse_attributes(exclude), + } + ) + @validate_log_events_outside_transaction(required_attrs=[attr]) + @validate_log_event_count_outside_transaction(1) def test(): - exercise_record_log_event() + record_log_event("A") test() diff --git a/tests/agent_features/test_logs_in_context.py b/tests/agent_features/test_logs_in_context.py index 90b6c92672..a0174da01f 100644 --- a/tests/agent_features/test_logs_in_context.py +++ b/tests/agent_features/test_logs_in_context.py @@ -14,6 +14,10 @@ import json import logging +import sys + +from io import StringIO as Buffer +from traceback import format_tb import pytest @@ -21,12 +25,7 @@ from newrelic.api.background_task import background_task from newrelic.api.function_trace import FunctionTrace from newrelic.api.log import NewRelicContextFormatter -from newrelic.packages import six -if six.PY2: - from io import BytesIO as Buffer -else: - from io import StringIO as Buffer _logger = logging.getLogger(__name__) @@ -47,12 +46,76 @@ def log_buffer(caplog): _logger.removeHandler(_handler) -class NonPrintableObject(object): +@pytest.fixture +def log_buffer_with_stack_trace(caplog): + buf = Buffer() + + _formatter = NewRelicContextFormatter("", datefmt="ISO8601", stack_trace_limit=None) + _handler = logging.StreamHandler(buf) + _handler.setFormatter(_formatter) + + _logger.addHandler(_handler) + caplog.set_level(logging.INFO, logger=__name__) + + yield buf + + _logger.removeHandler(_handler) + + +class NonPrintableObject(): def __str__(self): raise RuntimeError("Unable to print object.") - def __repr__(self): - raise RuntimeError("Unable to print object.") + __repr__ = __str__ + + +class NonSerializableObject(): + def __str__(self): + return f"<{self.__class__.__name__} object>" + + __repr__ = __str__ + + +def test_newrelic_logger_min_extra_keys_no_error(log_buffer): + extra = { + "string": "foo", + } + _logger.info("Hello %s", "World", extra=extra) + + log_buffer.seek(0) + message = json.load(log_buffer) + + timestamp = message.pop("timestamp") + thread_id = message.pop("thread.id") + process_id = message.pop("process.id") + filename = message.pop("file.name") + line_number = message.pop("line.number") + + assert isinstance(timestamp, int) + assert isinstance(thread_id, int) + assert isinstance(process_id, int) + assert filename.endswith("/test_logs_in_context.py") + assert isinstance(line_number, int) + + expected = { + "entity.name": "Python Agent Test (agent_features)", + "entity.type": "SERVICE", + "message": "Hello World", + "log.level": "INFO", + "logger.name": "test_logs_in_context", + "thread.name": "MainThread", + "process.name": "MainProcess", + "extra.string": "foo", + } + expected_extra_txn_keys = ( + "entity.guid", + "hostname", + ) + + for k, v in expected.items(): + assert message.pop(k) == v + + assert set(message.keys()) == set(expected_extra_txn_keys) def test_newrelic_logger_no_error(log_buffer): @@ -63,14 +126,15 @@ def test_newrelic_logger_no_error(log_buffer): "null": None, "array": [1, 2, 3], "bool": True, - "non_serializable": {"set"}, + "set": {"set"}, + "non_serializable": NonSerializableObject(), "non_printable": NonPrintableObject(), "object": { "first": "bar", "second": "baz", }, } - _logger.info(u"Hello %s", u"World", extra=extra) + _logger.info("Hello %s", "World", extra=extra) log_buffer.seek(0) message = json.load(log_buffer) @@ -88,24 +152,25 @@ def test_newrelic_logger_no_error(log_buffer): assert isinstance(line_number, int) expected = { - u"entity.name": u"Python Agent Test (agent_features)", - u"entity.type": u"SERVICE", - u"message": u"Hello World", - u"log.level": u"INFO", - u"logger.name": u"test_logs_in_context", - u"thread.name": u"MainThread", - u"process.name": u"MainProcess", - u"extra.string": u"foo", - u"extra.integer": 1, - u"extra.float": 1.23, - u"extra.null": None, - u"extra.array": [1, 2, 3], - u"extra.bool": True, - u"extra.non_serializable": u"set(['set'])" if six.PY2 else u"{'set'}", - u"extra.non_printable": u"", - u"extra.object": { - u"first": u"bar", - u"second": u"baz", + "entity.name": "Python Agent Test (agent_features)", + "entity.type": "SERVICE", + "message": "Hello World", + "log.level": "INFO", + "logger.name": "test_logs_in_context", + "thread.name": "MainThread", + "process.name": "MainProcess", + "extra.string": "foo", + "extra.integer": 1, + "extra.float": 1.23, + "extra.null": None, + "extra.array": [1, 2, 3], + "extra.bool": True, + "extra.set": '["set"]', + "extra.non_serializable": "", + "extra.non_printable": "", + "extra.object": { + "first": "bar", + "second": "baz", }, } expected_extra_txn_keys = ( @@ -119,17 +184,16 @@ def test_newrelic_logger_no_error(log_buffer): assert set(message.keys()) == set(expected_extra_txn_keys) - class ExceptionForTest(ValueError): pass @background_task() -def test_newrelic_logger_error_inside_transaction(log_buffer): +def test_newrelic_logger_error_inside_transaction_no_stack_trace(log_buffer): try: raise ExceptionForTest except ExceptionForTest: - _logger.exception(u"oops") + _logger.exception("oops") log_buffer.seek(0) message = json.load(log_buffer) @@ -147,16 +211,16 @@ def test_newrelic_logger_error_inside_transaction(log_buffer): assert isinstance(line_number, int) expected = { - u"entity.name": u"Python Agent Test (agent_features)", - u"entity.type": u"SERVICE", - u"message": u"oops", - u"log.level": u"ERROR", - u"logger.name": u"test_logs_in_context", - u"thread.name": u"MainThread", - u"process.name": u"MainProcess", - u"error.class": u"test_logs_in_context:ExceptionForTest", - u"error.message": u"", - u"error.expected": False, + "entity.name": "Python Agent Test (agent_features)", + "entity.type": "SERVICE", + "message": "oops", + "log.level": "ERROR", + "logger.name": "test_logs_in_context", + "thread.name": "MainThread", + "process.name": "MainProcess", + "error.class": "test_logs_in_context:ExceptionForTest", + "error.message": "", + "error.expected": False, } expected_extra_txn_keys = ( "trace.id", @@ -171,11 +235,63 @@ def test_newrelic_logger_error_inside_transaction(log_buffer): assert set(message.keys()) == set(expected_extra_txn_keys) -def test_newrelic_logger_error_outside_transaction(log_buffer): +@background_task() +def test_newrelic_logger_error_inside_transaction_with_stack_trace(log_buffer_with_stack_trace): + expected_stack_trace = "" + try: + raise ExceptionForTest + except ExceptionForTest: + _logger.exception("oops") + expected_stack_trace = "".join(format_tb(sys.exc_info()[2])) + + log_buffer_with_stack_trace.seek(0) + message = json.load(log_buffer_with_stack_trace) + + timestamp = message.pop("timestamp") + thread_id = message.pop("thread.id") + process_id = message.pop("process.id") + filename = message.pop("file.name") + line_number = message.pop("line.number") + stack_trace = message.pop("error.stack_trace") + + assert isinstance(timestamp, int) + assert isinstance(thread_id, int) + assert isinstance(process_id, int) + assert filename.endswith("/test_logs_in_context.py") + assert isinstance(line_number, int) + assert isinstance(stack_trace, str) + assert stack_trace and stack_trace == expected_stack_trace + + expected = { + "entity.name": "Python Agent Test (agent_features)", + "entity.type": "SERVICE", + "message": "oops", + "log.level": "ERROR", + "logger.name": "test_logs_in_context", + "thread.name": "MainThread", + "process.name": "MainProcess", + "error.class": "test_logs_in_context:ExceptionForTest", + "error.message": "", + "error.expected": False + } + expected_extra_txn_keys = ( + "trace.id", + "span.id", + "entity.guid", + "hostname" + ) + + for k, v in expected.items(): + assert message.pop(k) == v + + assert set(message.keys()) == set(expected_extra_txn_keys) + + +def test_newrelic_logger_error_outside_transaction_no_stack_trace(log_buffer): try: raise ExceptionForTest except ExceptionForTest: - _logger.exception(u"oops") + _logger.exception("oops") log_buffer.seek(0) message = json.load(log_buffer) @@ -193,15 +309,15 @@ def test_newrelic_logger_error_outside_transaction(log_buffer): assert isinstance(line_number, int) expected = { - u"entity.name": u"Python Agent Test (agent_features)", - u"entity.type": u"SERVICE", - u"message": u"oops", - u"log.level": u"ERROR", - u"logger.name": u"test_logs_in_context", - u"thread.name": u"MainThread", - u"process.name": u"MainProcess", - u"error.class": u"test_logs_in_context:ExceptionForTest", - u"error.message": u"", + "entity.name": "Python Agent Test (agent_features)", + "entity.type": "SERVICE", + "message": "oops", + "log.level": "ERROR", + "logger.name": "test_logs_in_context", + "thread.name": "MainThread", + "process.name": "MainProcess", + "error.class": "test_logs_in_context:ExceptionForTest", + "error.message": "", } expected_extra_txn_keys = ( "entity.guid", @@ -214,6 +330,53 @@ def test_newrelic_logger_error_outside_transaction(log_buffer): assert set(message.keys()) == set(expected_extra_txn_keys) +def test_newrelic_logger_error_outside_transaction_with_stack_trace(log_buffer_with_stack_trace): + expected_stack_trace = "" + try: + raise ExceptionForTest + except ExceptionForTest: + _logger.exception("oops") + expected_stack_trace = "".join(format_tb(sys.exc_info()[2])) + + log_buffer_with_stack_trace.seek(0) + message = json.load(log_buffer_with_stack_trace) + + timestamp = message.pop("timestamp") + thread_id = message.pop("thread.id") + process_id = message.pop("process.id") + filename = message.pop("file.name") + line_number = message.pop("line.number") + stack_trace = message.pop("error.stack_trace") + + assert isinstance(timestamp, int) + assert isinstance(thread_id, int) + assert isinstance(process_id, int) + assert filename.endswith("/test_logs_in_context.py") + assert isinstance(line_number, int) + assert isinstance(stack_trace, str) + assert stack_trace and stack_trace == expected_stack_trace + + expected = { + "entity.name": "Python Agent Test (agent_features)", + "entity.type": "SERVICE", + "message": "oops", + "log.level": "ERROR", + "logger.name": "test_logs_in_context", + "thread.name": "MainThread", + "process.name": "MainProcess", + "error.class": "test_logs_in_context:ExceptionForTest", + "error.message": "", + } + expected_extra_txn_keys = ( + "entity.guid", + "hostname", + ) + + for k, v in expected.items(): + assert message.pop(k) == v + + assert set(message.keys()) == set(expected_extra_txn_keys) + EXPECTED_KEYS_TXN = ( "trace.id", @@ -221,7 +384,7 @@ def test_newrelic_logger_error_outside_transaction(log_buffer): "entity.name", "entity.type", "entity.guid", - "hostname", + "hostname", ) EXPECTED_KEYS_NO_TXN = EXPECTED_KEYS_TRACE_ENDED = ( diff --git a/tests/agent_features/test_metric_normalization.py b/tests/agent_features/test_metric_normalization.py new file mode 100644 index 0000000000..65f2903aee --- /dev/null +++ b/tests/agent_features/test_metric_normalization.py @@ -0,0 +1,78 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.fixtures import reset_core_stats_engine +from testing_support.validators.validate_dimensional_metric_payload import ( + validate_dimensional_metric_payload, +) +from testing_support.validators.validate_metric_payload import validate_metric_payload + +from newrelic.api.application import application_instance +from newrelic.api.background_task import background_task +from newrelic.api.transaction import record_custom_metric, record_dimensional_metric +from newrelic.core.rules_engine import NormalizationRule, RulesEngine + +RULES = [{"match_expression": "(replace)", "replacement": "expected", "ignore": False, "eval_order": 0}] +EXPECTED_TAGS = frozenset({"tag": 1}.items()) + + +def _prepare_rules(test_rules): + # ensure all keys are present, if not present set to an empty string + for rule in test_rules: + for key in NormalizationRule._fields: + rule[key] = rule.get(key, "") + return test_rules + + +@pytest.fixture(scope="session") +def core_app(collector_agent_registration): + app = collector_agent_registration + return app._agent.application(app.name) + + +@pytest.fixture(scope="function") +def rules_engine_fixture(core_app): + rules_engine = core_app._rules_engine + previous_rules = rules_engine["metric"] + + rules_engine["metric"] = RulesEngine(_prepare_rules(RULES)) + yield + rules_engine["metric"] = previous_rules # Restore after test run + + +@validate_dimensional_metric_payload(summary_metrics=[("Metric/expected", EXPECTED_TAGS, 1)]) +@validate_metric_payload([("Metric/expected", 1)]) +@reset_core_stats_engine() +def test_metric_normalization_inside_transaction(core_app, rules_engine_fixture): + @background_task(name="test_record_dimensional_metric_inside_transaction") + def _test(): + record_dimensional_metric("Metric/replace", 1, tags={"tag": 1}) + record_custom_metric("Metric/replace", 1) + + _test() + core_app.harvest() + + +@validate_dimensional_metric_payload(summary_metrics=[("Metric/expected", EXPECTED_TAGS, 1)]) +@validate_metric_payload([("Metric/expected", 1)]) +@reset_core_stats_engine() +def test_metric_normalization_outside_transaction(core_app, rules_engine_fixture): + def _test(): + app = application_instance() + record_dimensional_metric("Metric/replace", 1, tags={"tag": 1}, application=app) + record_custom_metric("Metric/replace", 1, application=app) + + _test() + core_app.harvest() diff --git a/tests/agent_features/test_ml_events.py b/tests/agent_features/test_ml_events.py new file mode 100644 index 0000000000..a68078b1c6 --- /dev/null +++ b/tests/agent_features/test_ml_events.py @@ -0,0 +1,372 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import time + +from importlib import reload + +import pytest +from testing_support.fixtures import ( + function_not_called, + override_application_settings, + reset_core_stats_engine, +) +from testing_support.validators.validate_ml_event_count import validate_ml_event_count +from testing_support.validators.validate_ml_event_payload import ( + validate_ml_event_payload, +) +from testing_support.validators.validate_ml_events import validate_ml_events +from testing_support.validators.validate_ml_events_outside_transaction import ( + validate_ml_events_outside_transaction, +) + +import newrelic.core.otlp_utils +from newrelic.api.application import application_instance as application +from newrelic.api.background_task import background_task +from newrelic.api.transaction import record_ml_event +from newrelic.core.config import global_settings + + +_now = time.time() + +_intrinsics = { + "type": "LabelEvent", + "timestamp": _now, +} + + +@pytest.fixture(scope="session") +def core_app(collector_agent_registration): + app = collector_agent_registration + return app._agent.application(app.name) + + +@validate_ml_event_payload( + { + "apm": [ + { + "foo": "bar", + "real_agent_id": "1234567", + "event.domain": "newrelic.ml_events", + "event.name": "MyCustomEvent", + } + ] + } +) +@reset_core_stats_engine() +def test_ml_event_payload_noninference_event_inside_transaction(core_app): + @background_task(name="test_ml_event_payload_inside_transaction") + def _test(): + record_ml_event("MyCustomEvent", {"foo": "bar"}) + + _test() + core_app.harvest() + + +@validate_ml_event_payload( + { + "inference": [ + { + "foo": "bar", + "real_agent_id": "1234567", + "event.domain": "newrelic.ml_events", + "event.name": "InferenceEvent", + } + ] + } +) +@reset_core_stats_engine() +def test_ml_event_payload_inference_event_inside_transaction(core_app): + @background_task(name="test_ml_event_payload_inside_transaction") + def _test(): + record_ml_event("InferenceEvent", {"foo": "bar"}) + + _test() + core_app.harvest() + + +@validate_ml_event_payload( + { + "apm": [ + { + "foo": "bar", + "real_agent_id": "1234567", + "event.domain": "newrelic.ml_events", + "event.name": "MyCustomEvent", + } + ], + "inference": [ + { + "foo": "bar", + "real_agent_id": "1234567", + "event.domain": "newrelic.ml_events", + "event.name": "InferenceEvent", + } + ], + } +) +@reset_core_stats_engine() +def test_ml_event_payload_both_events_inside_transaction(core_app): + @background_task(name="test_ml_event_payload_inside_transaction") + def _test(): + record_ml_event("InferenceEvent", {"foo": "bar"}) + record_ml_event("MyCustomEvent", {"foo": "bar"}) + + _test() + core_app.harvest() + + +@validate_ml_event_payload( + { + "inference": [ + { + "foo": "bar", + "real_agent_id": "1234567", + "event.domain": "newrelic.ml_events", + "event.name": "InferenceEvent", + } + ] + } +) +@reset_core_stats_engine() +def test_ml_event_payload_inference_event_outside_transaction(core_app): + def _test(): + app = application() + record_ml_event("InferenceEvent", {"foo": "bar"}, application=app) + + _test() + core_app.harvest() + + +@validate_ml_event_payload( + { + "apm": [ + { + "foo": "bar", + "real_agent_id": "1234567", + "event.domain": "newrelic.ml_events", + "event.name": "MyCustomEvent", + } + ], + "inference": [ + { + "foo": "bar", + "real_agent_id": "1234567", + "event.domain": "newrelic.ml_events", + "event.name": "InferenceEvent", + } + ], + } +) +@reset_core_stats_engine() +def test_ml_event_payload_both_events_outside_transaction(core_app): + def _test(): + app = application() + record_ml_event("InferenceEvent", {"foo": "bar"}, application=app) + record_ml_event("MyCustomEvent", {"foo": "bar"}, application=app) + + _test() + core_app.harvest() + + +@validate_ml_event_payload( + { + "apm": [ + { + "foo": "bar", + "real_agent_id": "1234567", + "event.domain": "newrelic.ml_events", + "event.name": "MyCustomEvent", + } + ] + } +) +@reset_core_stats_engine() +def test_ml_event_payload_noninference_event_outside_transaction(core_app): + def _test(): + app = application() + record_ml_event("MyCustomEvent", {"foo": "bar"}, application=app) + + _test() + core_app.harvest() + + +@pytest.mark.parametrize( + "params,expected", + [ + ({"foo": "bar"}, [(_intrinsics, {"foo": "bar"})]), + ({"foo": "bar", 123: "bad key"}, [(_intrinsics, {"foo": "bar"})]), + ({"foo": "bar", "*" * 256: "too long"}, [(_intrinsics, {"foo": "bar"})]), + ], + ids=["Valid key/value", "Bad key", "Value too long"], +) +@reset_core_stats_engine() +def test_record_ml_event_inside_transaction(params, expected): + @validate_ml_events(expected) + @background_task() + def _test(): + record_ml_event("LabelEvent", params) + + _test() + + +@reset_core_stats_engine() +def test_record_ml_event_truncation_inside_transaction(): + @validate_ml_events([(_intrinsics, {"a": "a" * 4095})]) + @background_task() + def _test(): + record_ml_event("LabelEvent", {"a": "a" * 4100}) + + _test() + + +@reset_core_stats_engine() +def test_record_ml_event_truncation_outside_transaction(): + @validate_ml_events_outside_transaction([(_intrinsics, {"a": "a" * 4095})]) + def _test(): + app = application() + record_ml_event("LabelEvent", {"a": "a" * 4100}, application=app) + + _test() + + +@reset_core_stats_engine() +def test_record_ml_event_max_num_attrs(): + too_many_attrs_event = {} + for i in range(65): + too_many_attrs_event[str(i)] = str(i) + + max_attrs_event = {} + for i in range(64): + max_attrs_event[str(i)] = str(i) + + @validate_ml_events([(_intrinsics, max_attrs_event)]) + @background_task() + def _test(): + record_ml_event("LabelEvent", too_many_attrs_event) + + _test() + + +@reset_core_stats_engine() +def test_record_ml_event_max_num_attrs_outside_transaction(): + too_many_attrs_event = {} + for i in range(65): + too_many_attrs_event[str(i)] = str(i) + + max_attrs_event = {} + for i in range(64): + max_attrs_event[str(i)] = str(i) + + @validate_ml_events_outside_transaction([(_intrinsics, max_attrs_event)]) + def _test(): + app = application() + record_ml_event("LabelEvent", too_many_attrs_event, application=app) + + _test() + + +@pytest.mark.parametrize( + "params,expected", + [ + ({"foo": "bar"}, [(_intrinsics, {"foo": "bar"})]), + ({"foo": "bar", 123: "bad key"}, [(_intrinsics, {"foo": "bar"})]), + ({"foo": "bar", "*" * 256: "too long"}, [(_intrinsics, {"foo": "bar"})]), + ], + ids=["Valid key/value", "Bad key", "Value too long"], +) +@reset_core_stats_engine() +def test_record_ml_event_outside_transaction(params, expected): + @validate_ml_events_outside_transaction(expected) + def _test(): + app = application() + record_ml_event("LabelEvent", params, application=app) + + _test() + + +@reset_core_stats_engine() +@validate_ml_event_count(count=0) +@background_task() +def test_record_ml_event_inside_transaction_bad_event_type(): + record_ml_event("!@#$%^&*()", {"foo": "bar"}) + + +@reset_core_stats_engine() +@validate_ml_event_count(count=0) +def test_record_ml_event_outside_transaction_bad_event_type(): + app = application() + record_ml_event("!@#$%^&*()", {"foo": "bar"}, application=app) + + +@reset_core_stats_engine() +@validate_ml_event_count(count=0) +@background_task() +def test_record_ml_event_inside_transaction_params_not_a_dict(): + record_ml_event("ParamsListEvent", ["not", "a", "dict"]) + + +@reset_core_stats_engine() +@validate_ml_event_count(count=0) +def test_record_ml_event_outside_transaction_params_not_a_dict(): + app = application() + record_ml_event("ParamsListEvent", ["not", "a", "dict"], application=app) + + +# Tests for ML Events configuration settings + + +@override_application_settings({"ml_insights_events.enabled": False}) +@reset_core_stats_engine() +@validate_ml_event_count(count=0) +@background_task() +def test_ml_event_settings_check_ml_insights_disabled(): + record_ml_event("FooEvent", {"foo": "bar"}) + + +# Test that record_ml_event() methods will short-circuit. +# +# If the ml_insights_events setting is False, verify that the +# `create_ml_event()` function is not called, in order to avoid the +# event_type and attribute processing. + + +@override_application_settings({"ml_insights_events.enabled": False}) +@reset_core_stats_engine() +@function_not_called("newrelic.api.transaction", "create_custom_event") +@background_task() +def test_transaction_create_ml_event_not_called(): + record_ml_event("FooEvent", {"foo": "bar"}) + + +@override_application_settings({"ml_insights_events.enabled": False}) +@reset_core_stats_engine() +@function_not_called("newrelic.core.application", "create_custom_event") +@background_task() +def test_application_create_ml_event_not_called(): + app = application() + record_ml_event("FooEvent", {"foo": "bar"}, application=app) + + +@pytest.fixture(scope="module", autouse=True, params=["protobuf", "json"]) +def otlp_content_encoding(request): + _settings = global_settings() + prev = _settings.debug.otlp_content_encoding + _settings.debug.otlp_content_encoding = request.param + reload(newrelic.core.otlp_utils) + assert newrelic.core.otlp_utils.otlp_content_setting == request.param, "Content encoding mismatch." + + yield + + _settings.debug.otlp_content_encoding = prev diff --git a/tests/agent_features/test_notice_error.py b/tests/agent_features/test_notice_error.py index 913ee92899..e67d822dff 100644 --- a/tests/agent_features/test_notice_error.py +++ b/tests/agent_features/test_notice_error.py @@ -19,7 +19,6 @@ error_is_saved, override_application_settings, reset_core_stats_engine, - validate_transaction_error_event_count, validate_transaction_error_trace_count, ) from testing_support.validators.validate_application_error_event_count import ( @@ -31,6 +30,9 @@ from testing_support.validators.validate_application_errors import ( validate_application_errors, ) +from testing_support.validators.validate_transaction_error_event_count import ( + validate_transaction_error_event_count, +) from testing_support.validators.validate_transaction_errors import ( validate_transaction_errors, ) @@ -376,7 +378,7 @@ def test_notice_error_strip_message_not_in_allowlist_outside_transaction(): def _raise_errors(num_errors, application=None): for i in range(num_errors): try: - raise RuntimeError("error" + str(i)) + raise RuntimeError(f"error{str(i)}") except RuntimeError: notice_error(application=application) diff --git a/tests/agent_features/test_priority_sampling.py b/tests/agent_features/test_priority_sampling.py index f73824f710..6ef697c12c 100644 --- a/tests/agent_features/test_priority_sampling.py +++ b/tests/agent_features/test_priority_sampling.py @@ -18,6 +18,7 @@ override_application_settings, reset_core_stats_engine, ) +from testing_support.util import retry from newrelic.api.application import application_instance as application from newrelic.api.background_task import BackgroundTask @@ -29,6 +30,7 @@ def test_priority_used_in_transaction_events(first_transaction_saved): first_priority = 1 if first_transaction_saved else 0 second_priority = 0 if first_transaction_saved else 1 + @retry(attempts=5, wait=2) # This test is flakey so add a retry. @reset_core_stats_engine() def _test(): # Stats engine @@ -61,6 +63,7 @@ def test_priority_used_in_transaction_error_events(first_transaction_saved): first_priority = 1 if first_transaction_saved else 0 second_priority = 0 if first_transaction_saved else 1 + @retry(attempts=5, wait=2) # This test is flakey so add a retry. @reset_core_stats_engine() def _test(): with BackgroundTask(application(), name="T1") as txn: diff --git a/tests/agent_features/test_profile_trace.py b/tests/agent_features/test_profile_trace.py new file mode 100644 index 0000000000..f696b74809 --- /dev/null +++ b/tests/agent_features/test_profile_trace.py @@ -0,0 +1,88 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.profile_trace import ProfileTraceWrapper, profile_trace + + +def test_profile_trace_wrapper(): + def _test(): + def nested_fn(): + pass + + nested_fn() + + wrapped_test = ProfileTraceWrapper(_test) + wrapped_test() + + +@validate_transaction_metrics("test_profile_trace:test_profile_trace_empty_args", background_task=True) +@background_task() +def test_profile_trace_empty_args(): + @profile_trace() + def _test(): + pass + + _test() + + +_test_profile_trace_defined_args_scoped_metrics = [("Custom/TestTrace", 1)] + + +@validate_transaction_metrics( + "test_profile_trace:test_profile_trace_defined_args", + scoped_metrics=_test_profile_trace_defined_args_scoped_metrics, + background_task=True, +) +@background_task() +def test_profile_trace_defined_args(): + @profile_trace(name="TestTrace", group="Custom", label="Label", params={"key": "value"}, depth=7) + def _test(): + pass + + _test() + + +_test_profile_trace_callable_args_scoped_metrics = [("Function/TestProfileTrace", 1)] + + +@validate_transaction_metrics( + "test_profile_trace:test_profile_trace_callable_args", + scoped_metrics=_test_profile_trace_callable_args_scoped_metrics, + background_task=True, +) +@background_task() +def test_profile_trace_callable_args(): + def name_callable(): + return "TestProfileTrace" + + def group_callable(): + return "Function" + + def label_callable(): + return "HSM" + + def params_callable(): + return {"account_id": "12345"} + + @profile_trace(name=name_callable, group=group_callable, label=label_callable, params=params_callable, depth=0) + def _test(): + pass + + _test() diff --git a/tests/agent_features/test_record_llm_feedback_event.py b/tests/agent_features/test_record_llm_feedback_event.py new file mode 100644 index 0000000000..aee570fadb --- /dev/null +++ b/tests/agent_features/test_record_llm_feedback_event.py @@ -0,0 +1,86 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import reset_core_stats_engine +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events + +from newrelic.api.background_task import background_task +from newrelic.api.ml_model import record_llm_feedback_event + + +@reset_core_stats_engine() +def test_record_llm_feedback_event_all_args_supplied(): + llm_feedback_all_args_recorded_events = [ + ( + {"type": "LlmFeedbackMessage"}, + { + "id": None, + "trace_id": "123456789abcdefgh", + "category": "informative", + "rating": 1, + "ingest_source": "Python", + "message": "message", + "foo": "bar", + }, + ), + ] + + @validate_custom_events(llm_feedback_all_args_recorded_events) + @background_task() + def _test(): + record_llm_feedback_event( + rating=1, + trace_id="123456789abcdefgh", + category="informative", + message="message", + # Add metadata key with same name as built-in event key to verify no override occurs in the event + metadata={"foo": "bar", "message": "custom-message"}, + ) + + _test() + + +@reset_core_stats_engine() +def test_record_llm_feedback_event_required_args_supplied(): + llm_feedback_required_args_recorded_events = [ + ( + {"type": "LlmFeedbackMessage"}, + { + "id": None, + "rating": "Good", + "trace_id": "123456789abcdefgh", + "ingest_source": "Python", + }, + ), + ] + + @validate_custom_events(llm_feedback_required_args_recorded_events) + @background_task() + def _test(): + record_llm_feedback_event(trace_id="123456789abcdefgh", rating="Good") + + _test() + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_record_llm_feedback_event_outside_txn(): + record_llm_feedback_event( + rating="Good", + trace_id="123456789abcdefgh", + category="informative", + message="message", + metadata={"foo": "bar"}, + ) diff --git a/tests/agent_features/test_serverless_mode.py b/tests/agent_features/test_serverless_mode.py index 75b5f0075d..195c8ac602 100644 --- a/tests/agent_features/test_serverless_mode.py +++ b/tests/agent_features/test_serverless_mode.py @@ -13,7 +13,16 @@ # limitations under the License. import json + import pytest +from testing_support.fixtures import override_generic_settings +from testing_support.validators.validate_serverless_data import validate_serverless_data +from testing_support.validators.validate_serverless_metadata import ( + validate_serverless_metadata, +) +from testing_support.validators.validate_serverless_payload import ( + validate_serverless_payload, +) from newrelic.api.application import application_instance from newrelic.api.background_task import background_task @@ -22,23 +31,14 @@ from newrelic.api.transaction import current_transaction from newrelic.core.config import global_settings -from testing_support.fixtures import override_generic_settings -from testing_support.validators.validate_serverless_data import ( - validate_serverless_data) -from testing_support.validators.validate_serverless_payload import ( - validate_serverless_payload) -from testing_support.validators.validate_serverless_metadata import ( - validate_serverless_metadata) - -@pytest.fixture(scope='function') +@pytest.fixture(scope="function") def serverless_application(request): settings = global_settings() orig = settings.serverless_mode.enabled settings.serverless_mode.enabled = True - application_name = 'Python Agent Test (test_serverless_mode:%s)' % ( - request.node.name) + application_name = f"Python Agent Test (test_serverless_mode:{request.node.name})" application = application_instance(application_name) application.activate() @@ -48,17 +48,18 @@ def serverless_application(request): def test_serverless_payload(capsys, serverless_application): - - @override_generic_settings(serverless_application.settings, { - 'distributed_tracing.enabled': True, - }) + @override_generic_settings( + serverless_application.settings, + { + "distributed_tracing.enabled": True, + }, + ) @validate_serverless_data( - expected_methods=('metric_data', 'analytic_event_data'), - forgone_methods=('preconnect', 'connect', 'get_agent_commands')) + expected_methods=("metric_data", "analytic_event_data"), + forgone_methods=("preconnect", "connect", "get_agent_commands"), + ) @validate_serverless_payload() - @background_task( - application=serverless_application, - name='test_serverless_payload') + @background_task(application=serverless_application, name="test_serverless_payload") def _test(): transaction = current_transaction() assert transaction.settings.serverless_mode.enabled @@ -75,17 +76,15 @@ def _test(): def test_no_cat_headers(serverless_application): - @background_task( - application=serverless_application, - name='test_cat_headers') + @background_task(application=serverless_application, name="test_cat_headers") def _test_cat_headers(): transaction = current_transaction() payload = ExternalTrace.generate_request_headers(transaction) assert not payload - trace = ExternalTrace('testlib', 'http://example.com') - response_headers = [('X-NewRelic-App-Data', 'Cookies')] + trace = ExternalTrace("testlib", "http://example.com") + response_headers = [("X-NewRelic-App-Data", "Cookies")] with trace: trace.process_response_headers(response_headers) @@ -94,61 +93,68 @@ def _test_cat_headers(): _test_cat_headers() -def test_dt_outbound(serverless_application): - @override_generic_settings(serverless_application.settings, { - 'distributed_tracing.enabled': True, - 'account_id': '1', - 'trusted_account_key': '1', - 'primary_application_id': '1', - }) - @background_task( - application=serverless_application, - name='test_dt_outbound') - def _test_dt_outbound(): +@pytest.mark.parametrize("trusted_account_key", ("1", None), ids=("tk_set", "tk_unset")) +def test_outbound_dt_payload_generation(serverless_application, trusted_account_key): + @override_generic_settings( + serverless_application.settings, + { + "distributed_tracing.enabled": True, + "account_id": "1", + "trusted_account_key": trusted_account_key, + "primary_application_id": "1", + }, + ) + @background_task(application=serverless_application, name="test_outbound_dt_payload_generation") + def _test_outbound_dt_payload_generation(): transaction = current_transaction() payload = ExternalTrace.generate_request_headers(transaction) assert payload - - _test_dt_outbound() - - -def test_dt_inbound(serverless_application): - @override_generic_settings(serverless_application.settings, { - 'distributed_tracing.enabled': True, - 'account_id': '1', - 'trusted_account_key': '1', - 'primary_application_id': '1', - }) - @background_task( - application=serverless_application, - name='test_dt_inbound') - def _test_dt_inbound(): + # Ensure trusted account key or account ID present as vendor + assert dict(payload)["tracestate"].startswith("1@nr=") + + _test_outbound_dt_payload_generation() + + +@pytest.mark.parametrize("trusted_account_key", ("1", None), ids=("tk_set", "tk_unset")) +def test_inbound_dt_payload_acceptance(serverless_application, trusted_account_key): + @override_generic_settings( + serverless_application.settings, + { + "distributed_tracing.enabled": True, + "account_id": "1", + "trusted_account_key": trusted_account_key, + "primary_application_id": "1", + }, + ) + @background_task(application=serverless_application, name="test_inbound_dt_payload_acceptance") + def _test_inbound_dt_payload_acceptance(): transaction = current_transaction() payload = { - 'v': [0, 1], - 'd': { - 'ty': 'Mobile', - 'ac': '1', - 'tk': '1', - 'ap': '2827902', - 'pa': '5e5733a911cfbc73', - 'id': '7d3efb1b173fecfa', - 'tr': 'd6b4ba0c3a712ca', - 'ti': 1518469636035, - 'tx': '8703ff3d88eefe9d', - } + "v": [0, 1], + "d": { + "ty": "Mobile", + "ac": "1", + "tk": "1", + "ap": "2827902", + "pa": "5e5733a911cfbc73", + "id": "7d3efb1b173fecfa", + "tr": "d6b4ba0c3a712ca", + "ti": 1518469636035, + "tx": "8703ff3d88eefe9d", + }, } result = transaction.accept_distributed_trace_payload(payload) assert result - _test_dt_inbound() + _test_inbound_dt_payload_acceptance() -@pytest.mark.parametrize('arn_set', (True, False)) +# The lambda_hander has been deprecated for 3+ years +@pytest.mark.skip(reason="The lambda_handler has been deprecated") +@pytest.mark.parametrize("arn_set", (True, False)) def test_payload_metadata_arn(serverless_application, arn_set): - # If the session object gathers the arn from the settings object before the # lambda handler records it there, then this test will fail. @@ -157,17 +163,17 @@ def test_payload_metadata_arn(serverless_application, arn_set): arn = None if arn_set: - arn = 'arrrrrrrrrrRrrrrrrrn' + arn = "arrrrrrrrrrRrrrrrrrn" - settings.aws_lambda_metadata.update({'arn': arn, 'function_version': '$LATEST'}) + settings.aws_lambda_metadata.update({"arn": arn, "function_version": "$LATEST"}) - class Context(object): + class Context(): invoked_function_arn = arn - @validate_serverless_metadata(exact_metadata={'arn': arn}) + @validate_serverless_metadata(exact_metadata={"arn": arn}) @lambda_handler(application=serverless_application) def handler(event, context): - assert settings.aws_lambda_metadata['arn'] == arn + assert settings.aws_lambda_metadata["arn"] == arn return {} try: diff --git a/tests/agent_features/test_span_events.py b/tests/agent_features/test_span_events.py index 655efee8ce..682415f21b 100644 --- a/tests/agent_features/test_span_events.py +++ b/tests/agent_features/test_span_events.py @@ -19,7 +19,6 @@ dt_enabled, function_not_called, override_application_settings, - validate_tt_segment_params, ) from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_event_attributes import ( @@ -28,6 +27,9 @@ from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.background_task import background_task from newrelic.api.database_trace import DatabaseTrace @@ -139,7 +141,6 @@ def test_each_span_type(trace_type, args): ) @background_task(name="test_each_span_type") def _test(): - transaction = current_transaction() transaction._sampled = True @@ -155,9 +156,9 @@ def _test(): pytest.param("a" * 2001, "raw", "".join(["a"] * 1997 + ["..."]), id="truncate"), pytest.param("a" * 2000, "raw", "".join(["a"] * 2000), id="no_truncate"), pytest.param( - "select * from %s" % "".join(["?"] * 2000), + f"select * from {''.join(['?'] * 2000)}", "obfuscated", - "select * from %s..." % ("".join(["?"] * (2000 - len("select * from ") - 3))), + f"select * from {''.join(['?'] * (2000 - len('select * from ') - 3))}...", id="truncate_obfuscated", ), pytest.param("select 1", "off", ""), @@ -305,7 +306,6 @@ def _test(): } ) def test_external_span_limits(kwarg_override, attr_override): - exact_intrinsics = { "type": "Span", "sampled": True, @@ -351,7 +351,7 @@ def _test(): "kwarg_override,attribute_override", ( ({"host": "a" * 256}, {"peer.hostname": "a" * 255, "peer.address": "a" * 255}), - ({"port_path_or_id": "a" * 256, "host": "a"}, {"peer.hostname": "a", "peer.address": "a:" + "a" * 253}), + ({"port_path_or_id": "a" * 256, "host": "a"}, {"peer.hostname": "a", "peer.address": f"a:{'a' * 253}"}), ({"database_name": "a" * 256}, {"db.instance": "a" * 255}), ), ) @@ -362,7 +362,6 @@ def _test(): } ) def test_datastore_span_limits(kwarg_override, attribute_override): - exact_intrinsics = { "type": "Span", "sampled": True, @@ -414,10 +413,6 @@ def _test(): @pytest.mark.parametrize("span_events_enabled", (False, True)) def test_collect_span_events_override(collect_span_events, span_events_enabled): spans_expected = collect_span_events and span_events_enabled - # if collect_span_events and span_events_enabled: - # spans_expected = True - # else: - # spans_expected = False span_count = 2 if spans_expected else 0 @@ -484,7 +479,7 @@ def _test(): _test() -class FakeTrace(object): +class FakeTrace(): def __enter__(self): pass @@ -507,7 +502,6 @@ def __exit__(self, *args): ) @pytest.mark.parametrize("exclude_attributes", (True, False)) def test_span_event_user_attributes(trace_type, args, exclude_attributes): - _settings = { "distributed_tracing.enabled": True, "span_events.enabled": True, @@ -583,8 +577,8 @@ def test_span_custom_attribute_limit(): for i in range(128): if i < 64: - span_custom_attrs.append("span_attr%i" % i) - txn_custom_attrs.append("txn_attr%i" % i) + span_custom_attrs.append(f"span_attr{i}") + txn_custom_attrs.append(f"txn_attr{i}") unexpected_txn_attrs.extend(span_custom_attrs) span_custom_attrs.extend(txn_custom_attrs[:64]) @@ -600,9 +594,9 @@ def _test(): transaction = current_transaction() for i in range(128): - transaction.add_custom_parameter("txn_attr%i" % i, "txnValue") + transaction.add_custom_parameter(f"txn_attr{i}", "txnValue") if i < 64: - add_custom_span_attribute("span_attr%i" % i, "spanValue") + add_custom_span_attribute(f"span_attr{i}", "spanValue") _test() @@ -624,7 +618,6 @@ def _test(): ), ) def test_span_event_error_attributes_notice_error(trace_type, args): - _settings = { "distributed_tracing.enabled": True, "span_events.enabled": True, @@ -672,7 +665,6 @@ def _test(): ), ) def test_span_event_error_attributes_observed(trace_type, args): - error = ValueError("whoops") exact_agents = { @@ -725,7 +717,7 @@ def test_span_event_notice_error_overrides_observed(trace_type, args): raise ERROR except Exception: notice_error() - raise ValueError # pylint: disable + raise ValueError # pylint: disable (Py2/Py3 compatibility) except ValueError: pass diff --git a/tests/agent_features/test_stack_trace.py b/tests/agent_features/test_stack_trace.py index e09c9d2147..adc3d24d6b 100644 --- a/tests/agent_features/test_stack_trace.py +++ b/tests/agent_features/test_stack_trace.py @@ -22,7 +22,7 @@ def _format_stack_trace_from_tuples(frames): result = ['Traceback (most recent call last):'] - result.extend(['File "{0}", line {1}, in {2}'.format(*v) for v in frames]) + result.extend([f'File "{v[0]}", line {v[1]}, in {v[2]}' for v in frames]) return result def function0(): diff --git a/tests/agent_features/test_synthetics.py b/tests/agent_features/test_synthetics.py index 2e08144cc7..350cab03f0 100644 --- a/tests/agent_features/test_synthetics.py +++ b/tests/agent_features/test_synthetics.py @@ -17,7 +17,7 @@ from testing_support.external_fixtures import validate_synthetics_external_trace_header from testing_support.fixtures import ( cat_enabled, - make_synthetics_header, + make_synthetics_headers, override_application_settings, ) from testing_support.validators.validate_synthetics_event import ( @@ -37,6 +37,9 @@ SYNTHETICS_RESOURCE_ID = "09845779-16ef-4fa7-b7f2-44da8e62931c" SYNTHETICS_JOB_ID = "8c7dd3ba-4933-4cbb-b1ed-b62f511782f4" SYNTHETICS_MONITOR_ID = "dc452ae9-1a93-4ab5-8a33-600521e9cd00" +SYNTHETICS_TYPE = "scheduled" +SYNTHETICS_INITIATOR = "graphql" +SYNTHETICS_ATTRIBUTES = {"exampleAttribute": "1"} _override_settings = { "encoding_key": ENCODING_KEY, @@ -45,15 +48,19 @@ } -def _make_synthetics_header( +def _make_synthetics_headers( version="1", account_id=ACCOUNT_ID, resource_id=SYNTHETICS_RESOURCE_ID, job_id=SYNTHETICS_JOB_ID, monitor_id=SYNTHETICS_MONITOR_ID, encoding_key=ENCODING_KEY, + info_version="1", + type_=SYNTHETICS_TYPE, + initiator=SYNTHETICS_INITIATOR, + attributes=SYNTHETICS_ATTRIBUTES, ): - return make_synthetics_header(account_id, resource_id, job_id, monitor_id, encoding_key, version) + return make_synthetics_headers(encoding_key, account_id, resource_id, job_id, monitor_id, type_, initiator, attributes, synthetics_version=version, synthetics_info_version=info_version) def decode_header(header, encoding_key=ENCODING_KEY): @@ -80,6 +87,9 @@ def target_wsgi_application(environ, start_response): ("nr.syntheticsResourceId", SYNTHETICS_RESOURCE_ID), ("nr.syntheticsJobId", SYNTHETICS_JOB_ID), ("nr.syntheticsMonitorId", SYNTHETICS_MONITOR_ID), + ("nr.syntheticsType", SYNTHETICS_TYPE), + ("nr.syntheticsInitiator", SYNTHETICS_INITIATOR), + ("nr.syntheticsExampleAttribute", "1"), ] _test_valid_synthetics_event_forgone = [] @@ -89,21 +99,51 @@ def target_wsgi_application(environ, start_response): ) @override_application_settings(_override_settings) def test_valid_synthetics_event(): - headers = _make_synthetics_header() + headers = _make_synthetics_headers() + response = target_application.get("/", headers=headers) + + +_test_valid_synthetics_event_without_info_required = [ + ("nr.syntheticsResourceId", SYNTHETICS_RESOURCE_ID), + ("nr.syntheticsJobId", SYNTHETICS_JOB_ID), + ("nr.syntheticsMonitorId", SYNTHETICS_MONITOR_ID), +] +_test_valid_synthetics_event_without_info_forgone = [ + "nr.syntheticsType", + "nr.syntheticsInitiator", + "nr.syntheticsExampleAttribute", +] + + +@validate_synthetics_event( + _test_valid_synthetics_event_without_info_required, _test_valid_synthetics_event_without_info_forgone, should_exist=True +) +@override_application_settings(_override_settings) +def test_valid_synthetics_event_without_info(): + headers = _make_synthetics_headers(type_=None, initiator=None, attributes=None) response = target_application.get("/", headers=headers) @validate_synthetics_event([], [], should_exist=False) @override_application_settings(_override_settings) def test_no_synthetics_event_unsupported_version(): - headers = _make_synthetics_header(version="0") + headers = _make_synthetics_headers(version="0") + response = target_application.get("/", headers=headers) + + +@validate_synthetics_event( + _test_valid_synthetics_event_without_info_required, _test_valid_synthetics_event_without_info_forgone, should_exist=True +) +@override_application_settings(_override_settings) +def test_synthetics_event_unsupported_info_version(): + headers = _make_synthetics_headers(info_version="0") response = target_application.get("/", headers=headers) @validate_synthetics_event([], [], should_exist=False) @override_application_settings(_override_settings) def test_no_synthetics_event_untrusted_account(): - headers = _make_synthetics_header(account_id="999") + headers = _make_synthetics_headers(account_id="999") response = target_application.get("/", headers=headers) @@ -111,7 +151,20 @@ def test_no_synthetics_event_untrusted_account(): @override_application_settings(_override_settings) def test_no_synthetics_event_mismatched_encoding_key(): encoding_key = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" - headers = _make_synthetics_header(encoding_key=encoding_key) + headers = _make_synthetics_headers(encoding_key=encoding_key) + response = target_application.get("/", headers=headers) + + +@validate_synthetics_event( + _test_valid_synthetics_event_without_info_required, _test_valid_synthetics_event_without_info_forgone, should_exist=True +) +@override_application_settings(_override_settings) +def test_synthetics_event_mismatched_info_encoding_key(): + encoding_key = "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz" + headers = { + "X-NewRelic-Synthetics": _make_synthetics_headers(type_=None)["X-NewRelic-Synthetics"], + "X-NewRelic-Synthetics-Info": _make_synthetics_headers(encoding_key=encoding_key)["X-NewRelic-Synthetics-Info"], + } response = target_application.get("/", headers=headers) @@ -119,6 +172,9 @@ def test_no_synthetics_event_mismatched_encoding_key(): "synthetics_resource_id": SYNTHETICS_RESOURCE_ID, "synthetics_job_id": SYNTHETICS_JOB_ID, "synthetics_monitor_id": SYNTHETICS_MONITOR_ID, + "synthetics_type": SYNTHETICS_TYPE, + "synthetics_initiator": SYNTHETICS_INITIATOR, + "synthetics_example_attribute": "1", } @@ -126,7 +182,7 @@ def test_no_synthetics_event_mismatched_encoding_key(): @validate_synthetics_transaction_trace(_test_valid_synthetics_tt_required) @override_application_settings(_override_settings) def test_valid_synthetics_in_transaction_trace(): - headers = _make_synthetics_header() + headers = _make_synthetics_headers() response = target_application.get("/", headers=headers) @@ -146,26 +202,36 @@ def test_no_synthetics_in_transaction_trace(): @validate_synthetics_event([], [], should_exist=False) @override_application_settings(_disabled_settings) def test_synthetics_disabled(): - headers = _make_synthetics_header() + headers = _make_synthetics_headers() response = target_application.get("/", headers=headers) -_external_synthetics_header = ("X-NewRelic-Synthetics", _make_synthetics_header()["X-NewRelic-Synthetics"]) +_external_synthetics_headers = _make_synthetics_headers() +_external_synthetics_header = _external_synthetics_headers["X-NewRelic-Synthetics"] +_external_synthetics_info_header = _external_synthetics_headers["X-NewRelic-Synthetics-Info"] @cat_enabled -@validate_synthetics_external_trace_header(required_header=_external_synthetics_header, should_exist=True) +@validate_synthetics_external_trace_header(_external_synthetics_header, _external_synthetics_info_header) @override_application_settings(_override_settings) def test_valid_synthetics_external_trace_header(): - headers = _make_synthetics_header() + headers = _make_synthetics_headers() + response = target_application.get("/", headers=headers) + + +@cat_enabled +@validate_synthetics_external_trace_header(_external_synthetics_header, None) +@override_application_settings(_override_settings) +def test_valid_synthetics_external_trace_header_without_info(): + headers = _make_synthetics_headers(type_=None) response = target_application.get("/", headers=headers) @cat_enabled -@validate_synthetics_external_trace_header(required_header=_external_synthetics_header, should_exist=True) +@validate_synthetics_external_trace_header(_external_synthetics_header, _external_synthetics_info_header) @override_application_settings(_override_settings) def test_valid_external_trace_header_with_byte_inbound_header(): - headers = _make_synthetics_header() + headers = _make_synthetics_headers() headers = {k.encode("utf-8"): v.encode("utf-8") for k, v in headers.items()} @web_transaction( @@ -178,7 +244,7 @@ def webapp(): webapp() -@validate_synthetics_external_trace_header(should_exist=False) +@validate_synthetics_external_trace_header(None, None) @override_application_settings(_override_settings) def test_no_synthetics_external_trace_header(): response = target_application.get("/") @@ -194,7 +260,7 @@ def _synthetics_limit_test(num_requests, num_events, num_transactions): # Send requests - headers = _make_synthetics_header() + headers = _make_synthetics_headers() for i in range(num_requests): response = target_application.get("/", headers=headers) diff --git a/tests/agent_features/test_time_trace.py b/tests/agent_features/test_time_trace.py index eccb4d7fed..d866343867 100644 --- a/tests/agent_features/test_time_trace.py +++ b/tests/agent_features/test_time_trace.py @@ -15,6 +15,8 @@ import logging import pytest +from testing_support.fixtures import dt_enabled +from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) @@ -71,3 +73,31 @@ def test_trace_finalizes_with_transaction_missing_settings(monkeypatch, trace_ty # Ensure transaction still has settings when it exits to prevent other crashes making errors hard to read monkeypatch.undo() assert txn.settings + + +@pytest.mark.parametrize( + "trace_type,args", + ( + (DatabaseTrace, ("select * from foo",)), + (DatastoreTrace, ("db_product", "db_target", "db_operation")), + (ExternalTrace, ("lib", "url")), + (FunctionTrace, ("name",)), + (GraphQLOperationTrace, ()), + (GraphQLResolverTrace, ()), + (MemcacheTrace, ("command",)), + (MessageTrace, ("lib", "operation", "dst_type", "dst_name")), + (SolrTrace, ("lib", "command")), + ), +) +@dt_enabled +@validate_span_events( + count=1, + expected_users=["foo"], + unexpected_users=["drop-me"], +) +@background_task() +def test_trace_filters_out_invalid_attributes(trace_type, args): + txn = current_transaction() + with trace_type(*args) as trace: + trace.add_custom_attribute("drop-me", None) + trace.add_custom_attribute("foo", "bar") diff --git a/tests/agent_features/test_transaction_event_data_and_some_browser_stuff_too.py b/tests/agent_features/test_transaction_event_data_and_some_browser_stuff_too.py index 73bdfcf535..08ed69dd71 100644 --- a/tests/agent_features/test_transaction_event_data_and_some_browser_stuff_too.py +++ b/tests/agent_features/test_transaction_event_data_and_some_browser_stuff_too.py @@ -29,6 +29,7 @@ from newrelic.api.application import application_settings from newrelic.api.background_task import background_task +from newrelic.api.transaction import current_transaction from newrelic.common.encoding_utils import deobfuscate from newrelic.common.object_wrapper import transient_function_wrapper @@ -59,7 +60,6 @@ def test_capture_attributes_enabled(): header = response.html.html.head.script.string content = response.html.html.body.p.string - footer = response.html.html.body.script.string # Validate actual body content. @@ -71,10 +71,10 @@ def test_capture_attributes_enabled(): assert header.find("NREUM") != -1 - # Now validate the various fields of the footer related to analytics. + # Now validate the various fields of the header related to analytics. # The fields are held by a JSON dictionary. - data = json.loads(footer.split("NREUM.info=")[1]) + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) obfuscation_key = settings.license_key[:13] @@ -96,7 +96,7 @@ def test_capture_attributes_enabled(): browser_attributes["multibyte-utf8"] = _user_attributes["multibyte-utf8"].decode("latin-1") for attr, value in browser_attributes.items(): - assert user_attrs[attr] == value, "attribute %r expected %r, found %r" % (attr, value, user_attrs[attr]) + assert user_attrs[attr] == value, f"attribute {attr!r} expected {value!r}, found {user_attrs[attr]!r}" _test_no_attributes_recorded_settings = {"browser_monitoring.attributes.enabled": True} @@ -116,7 +116,6 @@ def test_no_attributes_recorded(): header = response.html.html.head.script.string content = response.html.html.body.p.string - footer = response.html.html.body.script.string # Validate actual body content. @@ -128,13 +127,13 @@ def test_no_attributes_recorded(): assert header.find("NREUM") != -1 - # Now validate the various fields of the footer related to analytics. + # Now validate the various fields of the header related to analytics. # The fields are held by a JSON dictionary. - data = json.loads(footer.split("NREUM.info=")[1]) + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) # As we are not recording any user or agent attributes, we should not - # actually have an entry at all in the footer. + # actually have an entry at all in the header. assert "atts" not in data @@ -163,7 +162,6 @@ def test_analytic_events_capture_attributes_disabled(): header = response.html.html.head.script.string content = response.html.html.body.p.string - footer = response.html.html.body.script.string # Validate actual body content. @@ -178,7 +176,7 @@ def test_analytic_events_capture_attributes_disabled(): # Now validate that attributes are present, since browser monitoring should # be enabled. - data = json.loads(footer.split("NREUM.info=")[1]) + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) assert "atts" in data @@ -196,7 +194,6 @@ def test_capture_attributes_default(): header = response.html.html.head.script.string content = response.html.html.body.p.string - footer = response.html.html.body.script.string # Validate actual body content. @@ -211,7 +208,7 @@ def test_capture_attributes_default(): # Now validate that attributes are not present, since should # be disabled. - data = json.loads(footer.split("NREUM.info=")[1]) + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) assert "atts" not in data @@ -258,7 +255,6 @@ def test_capture_attributes_disabled(): header = response.html.html.head.script.string content = response.html.html.body.p.string - footer = response.html.html.body.script.string # Validate actual body content. @@ -273,7 +269,7 @@ def test_capture_attributes_disabled(): # Now validate that attributes are not present, since should # be disabled. - data = json.loads(footer.split("NREUM.info=")[1]) + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) assert "atts" not in data @@ -307,7 +303,6 @@ def test_collect_analytic_events_disabled(): header = response.html.html.head.script.string content = response.html.html.body.p.string - footer = response.html.html.body.script.string # Validate actual body content. @@ -322,7 +317,7 @@ def test_collect_analytic_events_disabled(): # Now validate that attributes are present, since should # be enabled. - data = json.loads(footer.split("NREUM.info=")[1]) + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) assert "atts" in data @@ -351,7 +346,6 @@ def test_analytic_events_disabled(): header = response.html.html.head.script.string content = response.html.html.body.p.string - footer = response.html.html.body.script.string # Validate actual body content. @@ -366,7 +360,7 @@ def test_analytic_events_disabled(): # Now validate that attributes are present, since should # be enabled. - data = json.loads(footer.split("NREUM.info=")[1]) + data = json.loads(header.split("NREUM.info=")[1].split(";\n")[0]) assert "atts" in data @@ -498,7 +492,7 @@ def test_database_and_external_attributes_in_analytics(): } _expected_absent_attributes = { - "user": ("foo"), + "user": ("foo", "drop-me"), "agent": ("response.status", "request.method"), "intrinsic": ("port"), } @@ -507,4 +501,6 @@ def test_database_and_external_attributes_in_analytics(): @validate_transaction_event_attributes(_expected_attributes, _expected_absent_attributes) @background_task() def test_background_task_intrinsics_has_no_port(): + transaction = current_transaction() + transaction.add_custom_attribute("drop-me", None) pass diff --git a/tests/agent_features/test_transaction_trace_segments.py b/tests/agent_features/test_transaction_trace_segments.py index b205afc3ce..8318c0fca7 100644 --- a/tests/agent_features/test_transaction_trace_segments.py +++ b/tests/agent_features/test_transaction_trace_segments.py @@ -13,8 +13,8 @@ # limitations under the License. import pytest -from testing_support.fixtures import ( - override_application_settings, +from testing_support.fixtures import override_application_settings +from testing_support.validators.validate_tt_segment_params import ( validate_tt_segment_params, ) diff --git a/tests/agent_features/test_w3c_trace_context.py b/tests/agent_features/test_w3c_trace_context.py index 726cf011aa..b18c188bab 100644 --- a/tests/agent_features/test_w3c_trace_context.py +++ b/tests/agent_features/test_w3c_trace_context.py @@ -60,7 +60,7 @@ def target_wsgi_application(environ, start_response): INBOUND_TRACESTATE = \ 'rojo=f06a0ba902b7,congo=t61rcWkgMzE' LONG_TRACESTATE = \ - ','.join(["{}@rojo=f06a0ba902b7".format(x) for x in range(32)]) + ','.join([f"{x}@rojo=f06a0ba902b7" for x in range(32)]) INBOUND_UNTRUSTED_NR_TRACESTATE = \ ('2@nr=0-0-1345936-55632452-27jjj2d8890283b4-b28ce285632jjhl9-' '1-1.1273-1569367663277') @@ -140,10 +140,10 @@ def _test(): @pytest.mark.parametrize('inbound_tracestate,expected', ( ('', None), - (INBOUND_NR_TRACESTATE + "," + INBOUND_TRACESTATE, INBOUND_TRACESTATE), + (f"{INBOUND_NR_TRACESTATE},{INBOUND_TRACESTATE}", INBOUND_TRACESTATE), (INBOUND_TRACESTATE, INBOUND_TRACESTATE), - (LONG_TRACESTATE + ',' + INBOUND_NR_TRACESTATE, - ','.join("{}@rojo=f06a0ba902b7".format(x) for x in range(31))), + (f"{LONG_TRACESTATE},{INBOUND_NR_TRACESTATE}", + ','.join(f"{x}@rojo=f06a0ba902b7" for x in range(31))), ), ids=( 'empty_inbound_payload', 'nr_payload', @@ -216,7 +216,7 @@ def _test(): "parentSpanId": "00f067aa0ba902b7", "parent.transportType": "HTTP"}, [("Supportability/TraceContext/TraceParent/Accept/Success", 1)]), - (INBOUND_TRACEPARENT + ' ', { + (f"{INBOUND_TRACEPARENT} ", { "traceId": "0af7651916cd43dd8448eb211c80319c", "parentSpanId": "00f067aa0ba902b7", "parent.transportType": "HTTP"}, @@ -267,16 +267,16 @@ def _test(): (INBOUND_NR_TRACESTATE, {'trustedParentId': '27ddd2d8890283b4'}), ('garbage', {'parentId': '00f067aa0ba902b7'}), - (INBOUND_TRACESTATE + ',' + INBOUND_NR_TRACESTATE, + (f"{INBOUND_TRACESTATE},{INBOUND_NR_TRACESTATE}", {'parentId': '00f067aa0ba902b7', 'trustedParentId': '27ddd2d8890283b4', 'tracingVendors': 'rojo,congo'}), - (INBOUND_TRACESTATE + ',' + INBOUND_UNTRUSTED_NR_TRACESTATE, + (f"{INBOUND_TRACESTATE},{INBOUND_UNTRUSTED_NR_TRACESTATE}", {'parentId': '00f067aa0ba902b7', 'tracingVendors': 'rojo,congo,2@nr'}), - ('rojo=12345,' + 'v' * 257 + '=x', + (f"rojo=12345,{'v' * 257}=x", {'tracingVendors': 'rojo'}), - ('rojo=12345,k=' + 'v' * 257, + (f"rojo=12345,k={'v' * 257}", {'tracingVendors': 'rojo'}), )) @override_application_settings(_override_settings) diff --git a/tests/agent_features/test_web_transaction.py b/tests/agent_features/test_web_transaction.py index 66cf258587..5c3609ae9d 100644 --- a/tests/agent_features/test_web_transaction.py +++ b/tests/agent_features/test_web_transaction.py @@ -27,7 +27,6 @@ from newrelic.api.application import application_instance from newrelic.api.web_transaction import WebTransaction from newrelic.api.wsgi_application import wsgi_application -from newrelic.packages import six application = webtest.TestApp(simple_app) @@ -118,10 +117,7 @@ def test_base_web_transaction(use_bytes): for name, value in request_headers.items(): name = name.encode("utf-8") - try: - value = value.encode("utf-8") - except UnicodeDecodeError: - assert six.PY2 + value = value.encode("utf-8") byte_headers[name] = value request_headers = byte_headers diff --git a/tests/agent_features/test_wsgi_attributes.py b/tests/agent_features/test_wsgi_attributes.py index e90410b6db..db9fc807a4 100644 --- a/tests/agent_features/test_wsgi_attributes.py +++ b/tests/agent_features/test_wsgi_attributes.py @@ -13,12 +13,11 @@ # limitations under the License. import webtest -from testing_support.fixtures import ( - dt_enabled, - override_application_settings, +from testing_support.fixtures import dt_enabled, override_application_settings +from testing_support.sample_applications import fully_featured_app +from testing_support.validators.validate_error_event_attributes import ( validate_error_event_attributes, ) -from testing_support.sample_applications import fully_featured_app from testing_support.validators.validate_transaction_error_trace_attributes import ( validate_transaction_error_trace_attributes, ) diff --git a/tests/agent_streaming/conftest.py b/tests/agent_streaming/conftest.py index 390aeda9cb..a88d47828c 100644 --- a/tests/agent_streaming/conftest.py +++ b/tests/agent_streaming/conftest.py @@ -15,8 +15,10 @@ import threading import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) from testing_support.mock_external_grpc_server import MockExternalgRPCServer from newrelic.common.streaming_utils import StreamBuffer @@ -25,6 +27,7 @@ _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, diff --git a/tests/agent_streaming/test_infinite_tracing.py b/tests/agent_streaming/test_infinite_tracing.py index f1119c38cd..2fc273b42c 100644 --- a/tests/agent_streaming/test_infinite_tracing.py +++ b/tests/agent_streaming/test_infinite_tracing.py @@ -17,7 +17,7 @@ import pytest from testing_support.fixtures import override_generic_settings -from testing_support.util import conditional_decorator +from testing_support.util import retry from testing_support.validators.validate_metric_payload import validate_metric_payload from newrelic.common.streaming_utils import StreamBuffer @@ -25,7 +25,6 @@ from newrelic.core.application import Application from newrelic.core.config import global_settings from newrelic.core.infinite_tracing_pb2 import AttributeValue, Span -from newrelic.packages import six settings = global_settings() @@ -281,9 +280,8 @@ def condition(*args, **kwargs): _create_channel = StreamingRpc.create_channel def create_channel(self, *args, **kwargs): - ret = _create_channel(self, *args, **kwargs) + _create_channel(self, *args, **kwargs) connect_event.set() - return ret monkeypatch.setattr(StreamingRpc, "condition", condition) monkeypatch.setattr(StreamingRpc, "create_channel", create_channel) @@ -330,9 +328,6 @@ def connect_complete(): _test() -@conditional_decorator( - condition=six.PY2, decorator=pytest.mark.xfail(reason="Test frequently times out on Py2.", strict=False) -) def test_no_data_loss_on_reconnect(mock_grpc_server, app, buffer_empty_event, batching, spans_processed_event): """ Test for data loss when channel is closed by the server while waiting for more data in a request iterator. @@ -356,6 +351,7 @@ def test_no_data_loss_on_reconnect(mock_grpc_server, app, buffer_empty_event, ba span = Span(intrinsics={}, agent_attributes={}, user_attributes={}) + @retry(attempts=5, wait=2) # This test is flakey so add a retry. @override_generic_settings( settings, { @@ -389,12 +385,12 @@ def _test(): # Wait for OK status code to close the channel start_time = time.time() while not (request_iterator._stream and request_iterator._stream.done()): - assert time.time() - start_time < 5, "Timed out waiting for OK status code." + assert time.time() - start_time < 15, "Timed out waiting for OK status code." time.sleep(0.5) # Put new span and wait until buffer has been emptied and either sent or lost stream_buffer.put(span) - assert spans_processed_event.wait(timeout=5), "Data lost in stream buffer iterator." + assert spans_processed_event.wait(timeout=15), "Data lost in stream buffer iterator." _test() diff --git a/tests/agent_streaming/test_streaming_rpc.py b/tests/agent_streaming/test_streaming_rpc.py index 3ab74086ef..8f9a5e70d7 100644 --- a/tests/agent_streaming/test_streaming_rpc.py +++ b/tests/agent_streaming/test_streaming_rpc.py @@ -54,7 +54,7 @@ def test_correct_settings(mock_grpc_server, compression_setting, gRPC_compressio }, ) def _test(): - endpoint = "localhost:%s" % mock_grpc_server + endpoint = f"localhost:{mock_grpc_server}" stream_buffer = StreamBuffer(1) rpc = StreamingRpc( @@ -74,7 +74,7 @@ def _test(): def test_close_before_connect(mock_grpc_server, batching): - endpoint = "localhost:%s" % mock_grpc_server + endpoint = f"localhost:{mock_grpc_server}" stream_buffer = StreamBuffer(0, batching=batching) rpc = StreamingRpc(endpoint, stream_buffer, DEFAULT_METADATA, record_metric, ssl=False) @@ -89,7 +89,7 @@ def test_close_before_connect(mock_grpc_server, batching): def test_close_while_connected(mock_grpc_server, buffer_empty_event, batching): - endpoint = "localhost:%s" % mock_grpc_server + endpoint = f"localhost:{mock_grpc_server}" stream_buffer = StreamBuffer(1, batching=batching) rpc = StreamingRpc(endpoint, stream_buffer, DEFAULT_METADATA, record_metric, ssl=False) @@ -130,7 +130,7 @@ def condition(*args, **kwargs): user_attributes={}, ) - endpoint = "localhost:%s" % mock_grpc_server + endpoint = f"localhost:{mock_grpc_server}" stream_buffer = StreamBuffer(1, batching=batching) rpc = StreamingRpc(endpoint, stream_buffer, DEFAULT_METADATA, record_metric, ssl=False) @@ -158,7 +158,7 @@ def test_rpc_serialization_and_deserialization( ): """StreamingRPC sends deserializable span to correct endpoint.""" - endpoint = "localhost:%s" % mock_grpc_server + endpoint = f"localhost:{mock_grpc_server}" stream_buffer = StreamBuffer(1, batching=batching) span = Span( diff --git a/tests/agent_unittests/_test_import_hook.py b/tests/agent_unittests/_test_import_hook.py index 6afbb06374..e5c0a60a03 100644 --- a/tests/agent_unittests/_test_import_hook.py +++ b/tests/agent_unittests/_test_import_hook.py @@ -15,10 +15,10 @@ def run(): pass -class A(object): +class A(): def run(): pass -class B(object): +class B(): def run(): pass diff --git a/tests/agent_unittests/conftest.py b/tests/agent_unittests/conftest.py index 1504d1b8d9..4e9bfa5597 100644 --- a/tests/agent_unittests/conftest.py +++ b/tests/agent_unittests/conftest.py @@ -15,9 +15,13 @@ import sys import tempfile -import pytest +from importlib import reload -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +import pytest +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 newrelic_caplog as caplog, ) @@ -37,15 +41,7 @@ ) -try: - # python 2.x - reload -except NameError: - # python 3.x - from importlib import reload - - -class FakeProtos(object): +class FakeProtos(): Span = object() SpanBatch = object() diff --git a/tests/agent_unittests/test_agent.py b/tests/agent_unittests/test_agent.py index 55eea099cc..e099a2f0bd 100644 --- a/tests/agent_unittests/test_agent.py +++ b/tests/agent_unittests/test_agent.py @@ -18,7 +18,7 @@ from testing_support.fixtures import override_generic_settings -class FakeApplication(object): +class FakeApplication(): name = 'Fake' def __init__(self, *args, **kwargs): diff --git a/tests/agent_unittests/test_agent_connect.py b/tests/agent_unittests/test_agent_connect.py index eb944c072a..ca257b3ddf 100644 --- a/tests/agent_unittests/test_agent_connect.py +++ b/tests/agent_unittests/test_agent_connect.py @@ -13,26 +13,27 @@ # limitations under the License. import pytest +from testing_support.fixtures import failing_endpoint, override_generic_settings +from testing_support.validators.validate_internal_metrics import ( + validate_internal_metrics, +) + from newrelic.core.application import Application from newrelic.core.config import global_settings from newrelic.network.exceptions import ForceAgentDisconnect -from testing_support.fixtures import ( - override_generic_settings, - failing_endpoint -) -from testing_support.validators.validate_internal_metrics import validate_internal_metrics - - SETTINGS = global_settings() -@override_generic_settings(SETTINGS, { - 'developer_mode': True, -}) -@failing_endpoint('preconnect', raises=ForceAgentDisconnect) +@override_generic_settings( + SETTINGS, + { + "developer_mode": True, + }, +) +@failing_endpoint("preconnect", raises=ForceAgentDisconnect) def test_http_gone_stops_connect(): - app = Application('Python Agent Test (agent_unittests-connect)') + app = Application("Python Agent Test (agent_unittests-connect)") app.connect_to_data_collector(None) # The agent must not reattempt a connection after a ForceAgentDisconnect. @@ -48,28 +49,55 @@ def test_http_gone_stops_connect(): ] -@override_generic_settings(SETTINGS, { - 'developer_mode': True, -}) +@override_generic_settings( + SETTINGS, + { + "developer_mode": True, + }, +) @pytest.mark.parametrize("feature_setting,subfeature_setting", _logging_settings_matrix) def test_logging_connect_supportability_metrics(feature_setting, subfeature_setting): metric_value = "enabled" if feature_setting and subfeature_setting else "disabled" - @override_generic_settings(SETTINGS, { - "application_logging.enabled": feature_setting, - "application_logging.forwarding.enabled": subfeature_setting, - "application_logging.metrics.enabled": subfeature_setting, - "application_logging.local_decorating.enabled": subfeature_setting, - }) - @validate_internal_metrics([ - ("Supportability/Logging/Forwarding/Python/%s" % metric_value, 1), - ("Supportability/Logging/LocalDecorating/Python/%s" % metric_value, 1), - ("Supportability/Logging/Metrics/Python/%s" % metric_value, 1), - ]) + @override_generic_settings( + SETTINGS, + { + "application_logging.enabled": feature_setting, + "application_logging.forwarding.enabled": subfeature_setting, + "application_logging.metrics.enabled": subfeature_setting, + "application_logging.local_decorating.enabled": subfeature_setting, + }, + ) + @validate_internal_metrics( + [ + (f"Supportability/Logging/Forwarding/Python/{metric_value}", 1), + (f"Supportability/Logging/LocalDecorating/Python/{metric_value}", 1), + (f"Supportability/Logging/Metrics/Python/{metric_value}", 1), + ] + ) def test(): - app = Application('Python Agent Test (agent_unittests-connect)') + app = Application("Python Agent Test (agent_unittests-connect)") app.connect_to_data_collector(None) assert app._active_session - + test() + + +@override_generic_settings( + SETTINGS, + { + "developer_mode": True, + "ai_monitoring.streaming.enabled": False, + }, +) +@validate_internal_metrics( + [ + ("Supportability/Python/ML/Streaming/Disabled", 1), + ] +) +def test_ml_streaming_disabled_supportability_metrics(): + app = Application("Python Agent Test (agent_unittests-connect)") + app.connect_to_data_collector(None) + + assert app._active_session diff --git a/tests/agent_unittests/test_agent_protocol.py b/tests/agent_unittests/test_agent_protocol.py index ba75358abd..839e479bf9 100644 --- a/tests/agent_unittests/test_agent_protocol.py +++ b/tests/agent_unittests/test_agent_protocol.py @@ -35,14 +35,13 @@ NetworkInterfaceException, RetryDataForRequest, ) -from newrelic.packages import six Request = namedtuple("Request", ("method", "path", "params", "headers", "payload")) # Global constants used in tests APP_NAME = "test_app" -IP_ADDRESS = AWS = AZURE = GCP = PCF = BOOT_ID = DOCKER = KUBERNETES = None +IP_ADDRESS = AWS = AZURE = ECS = GCP = PCF = BOOT_ID = DOCKER = KUBERNETES = None BROWSER_MONITORING_DEBUG = "debug" BROWSER_MONITORING_LOADER = "loader" CAPTURE_PARAMS = "capture_params" @@ -117,9 +116,10 @@ def clear_sent_values(): @pytest.fixture(autouse=True) def override_utilization(monkeypatch): - global AWS, AZURE, GCP, PCF, BOOT_ID, DOCKER, KUBERNETES + global AWS, AZURE, ECS, GCP, PCF, BOOT_ID, DOCKER, KUBERNETES AWS = {"id": "foo", "type": "bar", "zone": "baz"} AZURE = {"location": "foo", "name": "bar", "vmId": "baz", "vmSize": "boo"} + ECS = {"ecsDockerId": "foobar"} GCP = {"id": 1, "machineType": "trmntr-t1000", "name": "arnold", "zone": "abc"} PCF = {"cf_instance_guid": "1", "cf_instance_ip": "7", "memory_limit": "0"} BOOT_ID = "cca356a7d72737f645a10c122ebbe906" @@ -134,8 +134,12 @@ def detect(cls): output = BOOT_ID elif name.startswith("AWS"): output = AWS + elif name.startswith("ECS"): + output = ECS elif name.startswith("Azure"): output = AZURE + elif name.startswith("ECS"): + output = ECS elif name.startswith("GCP"): output = GCP elif name.startswith("PCF"): @@ -170,7 +174,7 @@ def test_send(status_code): HttpClientRecorder.STATUS_CODE = status_code settings = finalize_application_settings( { - "request_headers_map": {"custom-header": u"value"}, + "request_headers_map": {"custom-header": "value"}, # pylint: disable=W1406 "agent_run_id": "RUN_TOKEN", } ) @@ -196,7 +200,7 @@ def test_send(status_code): assert request.headers == { "Content-Type": "application/json", - "custom-header": u"value", + "custom-header": "value", } assert request.payload == b"[1,2,3]" @@ -290,6 +294,7 @@ def test_close_connection(): def connect_payload_asserts( payload, with_aws=True, + with_ecs=True, with_gcp=True, with_pcf=True, with_azure=True, @@ -297,7 +302,8 @@ def connect_payload_asserts( with_kubernetes=True, ): payload_data = payload[0] - assert isinstance(payload_data["agent_version"], type(u"")) + + assert isinstance(payload_data["agent_version"], str) assert payload_data["app_name"] == PAYLOAD_APP_NAME assert payload_data["display_host"] == DISPLAY_NAME assert payload_data["environment"] == ENVIRONMENT @@ -311,9 +317,10 @@ def connect_payload_asserts( assert len(payload_data["security_settings"]) == 2 assert payload_data["security_settings"]["capture_params"] == CAPTURE_PARAMS assert payload_data["security_settings"]["transaction_tracer"] == {"record_sql": RECORD_SQL} - assert len(payload_data["settings"]) == 2 + assert len(payload_data["settings"]) == 3 assert payload_data["settings"]["browser_monitoring.loader"] == (BROWSER_MONITORING_LOADER) assert payload_data["settings"]["browser_monitoring.debug"] == (BROWSER_MONITORING_DEBUG) + assert payload_data["settings"]["ai_monitoring.enabled"] is False utilization_len = 5 @@ -325,7 +332,7 @@ def connect_payload_asserts( else: assert "ip_address" not in payload_data["utilization"] - utilization_len = utilization_len + any([with_aws, with_pcf, with_gcp, with_azure, with_docker, with_kubernetes]) + utilization_len = utilization_len + any([with_aws, with_ecs, with_pcf, with_gcp, with_azure, with_docker, with_kubernetes]) assert len(payload_data["utilization"]) == utilization_len assert payload_data["utilization"]["hostname"] == HOST @@ -342,11 +349,13 @@ def connect_payload_asserts( assert harvest_limits["error_event_data"] == ERROR_EVENT_DATA vendors_len = 0 - if any([with_aws, with_pcf, with_gcp, with_azure]): vendors_len += 1 - if with_docker: + if with_ecs: + vendors_len += 1 + + if with_docker and not with_ecs: vendors_len += 1 if with_kubernetes: @@ -365,7 +374,10 @@ def connect_payload_asserts( elif with_azure: assert payload_data["utilization"]["vendors"]["azure"] == AZURE - if with_docker: + if with_ecs: + assert payload_data["utilization"]["vendors"]["ecs"] == ECS + + if with_docker and not with_ecs: assert payload_data["utilization"]["vendors"]["docker"] == DOCKER if with_kubernetes: @@ -375,24 +387,25 @@ def connect_payload_asserts( @pytest.mark.parametrize( - "with_aws,with_pcf,with_gcp,with_azure,with_docker,with_kubernetes,with_ip", + "with_aws,with_ecs,with_pcf,with_gcp,with_azure,with_docker,with_kubernetes,with_ip", [ - (False, False, False, False, False, False, False), - (False, False, False, False, False, False, True), - (True, False, False, False, True, True, True), - (False, True, False, False, True, True, True), - (False, False, True, False, True, True, True), - (False, False, False, True, True, True, True), - (True, False, False, False, False, False, True), - (False, True, False, False, False, False, True), - (False, False, True, False, False, False, True), - (False, False, False, True, False, False, True), - (True, True, True, True, True, True, True), - (True, True, True, True, True, False, True), - (True, True, True, True, False, True, True), + (False, False, False, False, False, False, False, False), + (False, False, False, False, False, False, False, True), + (True, True, False, False, False, True, False, True), + (True, True, False, False, False, True, True, True), + (False, False, True, False, False, True, True, True), + (False, False, False, True, False, True, True, True), + (False, False, False, False, True, True, True, True), + (True, True, False, False, False, False, False, True), + (False, False, True, False, False, False, False, True), + (False, False, False, True, False, False, False, True), + (False, False, False, False, True, False, False, True), + (True, True, True, True, True, True, True, True), + (True, True, True, True, True, True, False, True), + (True, True, True, True, True, False, True, True), ], ) -def test_connect(with_aws, with_pcf, with_gcp, with_azure, with_docker, with_kubernetes, with_ip): +def test_connect(with_aws, with_ecs, with_pcf, with_gcp, with_azure, with_docker, with_kubernetes, with_ip): global AWS, AZURE, GCP, PCF, BOOT_ID, DOCKER, KUBERNETES, IP_ADDRESS if not with_aws: AWS = Exception @@ -402,6 +415,8 @@ def test_connect(with_aws, with_pcf, with_gcp, with_azure, with_docker, with_kub GCP = Exception if not with_azure: AZURE = Exception + if not with_ecs: + ECS = Exception if not with_docker: DOCKER = Exception if not with_kubernetes: @@ -459,6 +474,7 @@ def test_connect(with_aws, with_pcf, with_gcp, with_azure, with_docker, with_kub with_pcf=with_pcf, with_gcp=with_gcp, with_azure=with_azure, + with_ecs=with_ecs, with_docker=with_docker, with_kubernetes=with_kubernetes, ) @@ -479,7 +495,7 @@ def test_connect(with_aws, with_pcf, with_gcp, with_azure, with_docker, with_kub assert agent_settings_payload["proxy_host"] == "None" assert agent_settings_payload["attributes.include"] == "[]" assert agent_settings_payload["feature_flag"] == str(set()) - assert isinstance(agent_settings_payload["attribute_filter"], six.string_types) + assert isinstance(agent_settings_payload["attribute_filter"], str) # Verify that the connection is closed assert HttpClientRecorder.STATE == 0 @@ -537,6 +553,7 @@ def test_serverless_protocol_finalize(capsys): assert data["metadata"]["foo"] == "bar" assert data["metadata"]["agent_version"] != "x" + assert data["metadata"]["agent_language"] == "python" def test_audit_logging(): @@ -563,8 +580,9 @@ def test_audit_logging(): ) def test_ca_bundle_path(monkeypatch, ca_bundle_path): # Pretend CA certificates are not available - class DefaultVerifyPaths(object): + class DefaultVerifyPaths(): cafile = None + capath = None def __init__(self, *args, **kwargs): pass diff --git a/tests/agent_unittests/test_check_environment.py b/tests/agent_unittests/test_check_environment.py index 7f74f7c702..f34884efae 100644 --- a/tests/agent_unittests/test_check_environment.py +++ b/tests/agent_unittests/test_check_environment.py @@ -30,7 +30,7 @@ def test_check_environment_failing(content): os.makedirs(uwsgi_dir) with open(init_file, 'w') as f: for key, value in content.items(): - f.write("%s = %s" % (key, value)) + f.write(f"{key} = {value}") sys.path.insert(0, temp_dir) import uwsgi diff --git a/tests/agent_unittests/test_connect_response_fields.py b/tests/agent_unittests/test_connect_response_fields.py index 617cc7ce23..452be53f00 100644 --- a/tests/agent_unittests/test_connect_response_fields.py +++ b/tests/agent_unittests/test_connect_response_fields.py @@ -12,41 +12,41 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest import functools import time +import pytest from testing_support.fixtures import override_generic_settings -from newrelic.core.config import global_settings -from newrelic.core.agent_protocol import AgentProtocol + from newrelic.common.agent_http import DeveloperModeClient from newrelic.common.encoding_utils import json_encode - +from newrelic.core.agent_protocol import AgentProtocol +from newrelic.core.config import global_settings DEFAULT = object() LINKED_APPLICATIONS = [] ENVIRONMENT = [] NOW = time.time() EMPTY_SAMPLES = { - 'reservoir_size': 100, - 'events_seen': 0, + "reservoir_size": 100, + "events_seen": 0, } _all_endpoints = ( - ('send_metric_data', (NOW, NOW + 1, ())), - ('send_transaction_events', (EMPTY_SAMPLES, ())), - ('send_custom_events', (EMPTY_SAMPLES, ())), - ('send_error_events', (EMPTY_SAMPLES, ())), - ('send_transaction_traces', ([[]],)), - ('send_sql_traces', ([[]],)), - ('get_agent_commands', ()), - ('send_profile_data', ([[]],)), - ('send_errors', ([[]],)), - ('send_agent_command_results', ({0: {}},)), - ('agent_settings', ({},)), - ('send_span_events', (EMPTY_SAMPLES, ())), - ('shutdown_session', ()), + ("send_metric_data", (NOW, NOW + 1, ())), + ("send_transaction_events", (EMPTY_SAMPLES, ())), + ("send_custom_events", (EMPTY_SAMPLES, ())), + ("send_error_events", (EMPTY_SAMPLES, ())), + ("send_transaction_traces", ([[]],)), + ("send_sql_traces", ([[]],)), + ("get_agent_commands", ()), + ("send_profile_data", ([[]],)), + ("send_errors", ([[]],)), + ("send_agent_command_results", ({0: {}},)), + ("agent_settings", ({},)), + ("send_span_events", (EMPTY_SAMPLES, ())), + ("shutdown_session", ()), ) @@ -85,22 +85,17 @@ def send_request( ) -@pytest.mark.parametrize('headers_map_present', (True, False)) +@pytest.mark.parametrize("headers_map_present", (True, False)) def test_no_blob_behavior(headers_map_present): if headers_map_present: - connect_response_fields = {u"request_headers_map": None} - client_cls = functools.partial( - CustomTestClient, connect_response_fields=connect_response_fields) + connect_response_fields = {"request_headers_map": None} + client_cls = functools.partial(CustomTestClient, connect_response_fields=connect_response_fields) else: - client_cls = functools.partial( - CustomTestClient, connect_response_fields=DEFAULT) + client_cls = functools.partial(CustomTestClient, connect_response_fields=DEFAULT) protocol = AgentProtocol.connect( - 'app_name', - LINKED_APPLICATIONS, - ENVIRONMENT, - global_settings(), - client_cls=client_cls) + "app_name", LINKED_APPLICATIONS, ENVIRONMENT, global_settings(), client_cls=client_cls + ) protocol.send("shutdown") @@ -111,19 +106,14 @@ def test_no_blob_behavior(headers_map_present): def test_blob(): - request_headers_map = {u'X-Foo': u'Bar'} - connect_response_fields = {u"request_headers_map": request_headers_map} + request_headers_map = {"X-Foo": "Bar"} + connect_response_fields = {"request_headers_map": request_headers_map} - client_cls = functools.partial( - CustomTestClient, - connect_response_fields=connect_response_fields) + client_cls = functools.partial(CustomTestClient, connect_response_fields=connect_response_fields) protocol = AgentProtocol.connect( - 'app_name', - LINKED_APPLICATIONS, - ENVIRONMENT, - global_settings(), - client_cls=client_cls) + "app_name", LINKED_APPLICATIONS, ENVIRONMENT, global_settings(), client_cls=client_cls + ) protocol.send("shutdown") @@ -134,52 +124,71 @@ def test_blob(): } -@override_generic_settings(global_settings(), { - 'developer_mode': True, -}) +@override_generic_settings( + global_settings(), + { + "developer_mode": True, + }, +) def test_server_side_config_precedence(): connect_response_fields = { - u'agent_config': {u'span_events.enabled': True}, - u'span_events.enabled': False, + "agent_config": {"span_events.enabled": True}, + "span_events.enabled": False, } - client_cls = functools.partial( - CustomTestClient, - connect_response_fields=connect_response_fields) + client_cls = functools.partial(CustomTestClient, connect_response_fields=connect_response_fields) protocol = AgentProtocol.connect( - 'app_name', - LINKED_APPLICATIONS, - ENVIRONMENT, - global_settings(), - client_cls=client_cls) + "app_name", LINKED_APPLICATIONS, ENVIRONMENT, global_settings(), client_cls=client_cls + ) assert protocol.configuration.span_events.enabled is False -@override_generic_settings(global_settings(), { - 'developer_mode': True, -}) -@pytest.mark.parametrize("connect_response_fields", -( - {}, - {"span_event_harvest_config": {"report_period_ms": 60000, "harvest_limit": 123}}, - {"span_event_harvest_config": {}}) +@override_generic_settings( + global_settings(), + { + "developer_mode": True, + }, +) +@pytest.mark.parametrize( + "connect_response_fields", + ( + {}, + {"span_event_harvest_config": {"report_period_ms": 60000, "harvest_limit": 123}}, + {"span_event_harvest_config": {}}, + ), ) def test_span_event_harvest_config(connect_response_fields): - client_cls = functools.partial( - CustomTestClient, - connect_response_fields=connect_response_fields) + client_cls = functools.partial(CustomTestClient, connect_response_fields=connect_response_fields) protocol = AgentProtocol.connect( - 'app_name', - LINKED_APPLICATIONS, - ENVIRONMENT, - global_settings(), - client_cls=client_cls) + "app_name", LINKED_APPLICATIONS, ENVIRONMENT, global_settings(), client_cls=client_cls + ) if connect_response_fields and connect_response_fields["span_event_harvest_config"]: expected = 123 else: from newrelic.core.config import SPAN_EVENT_RESERVOIR_SIZE + expected = SPAN_EVENT_RESERVOIR_SIZE assert protocol.configuration.event_harvest_config.harvest_limits.span_event_data == expected + + +@override_generic_settings( + global_settings(), + { + "developer_mode": True, + }, +) +@pytest.mark.parametrize("connect_response_fields", ({}, {"collect_ai": True}, {"collect_ai": False})) +def test_account_level_aim(connect_response_fields): + client_cls = functools.partial(CustomTestClient, connect_response_fields=connect_response_fields) + + protocol = AgentProtocol.connect( + "app_name", LINKED_APPLICATIONS, ENVIRONMENT, global_settings(), client_cls=client_cls + ) + + if connect_response_fields and connect_response_fields["collect_ai"]: + assert protocol.configuration.ai_monitoring.enabled == connect_response_fields["collect_ai"] + else: + assert protocol.configuration.ai_monitoring.enabled is False diff --git a/tests/agent_unittests/test_encoding_utils.py b/tests/agent_unittests/test_encoding_utils.py new file mode 100644 index 0000000000..397f2fa2ef --- /dev/null +++ b/tests/agent_unittests/test_encoding_utils.py @@ -0,0 +1,52 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from newrelic.common.encoding_utils import camel_case, snake_case + + +@pytest.mark.parametrize("input_,expected,upper", [ + ("", "", False), + ("", "", True), + ("my_string", "myString", False), + ("my_string", "MyString", True), + ("LeaveCase", "LeaveCase", False), + ("correctCase", "CorrectCase", True), + ("UPPERcaseLETTERS", "UPPERcaseLETTERS", False), + ("UPPERcaseLETTERS", "UPPERcaseLETTERS", True), + ("lowerCASEletters", "lowerCASEletters", False), + ("lowerCASEletters", "LowerCASEletters", True), + ("very_long_snake_string", "VeryLongSnakeString", True), + ("kebab-case", "kebab-case", False), +]) +def test_camel_case(input_, expected, upper): + output = camel_case(input_, upper=upper) + assert output == expected + + +@pytest.mark.parametrize("input_,expected", [ + ("", ""), + ("", ""), + ("my_string", "my_string"), + ("myString", "my_string"), + ("MyString", "my_string"), + ("UPPERcaseLETTERS", "uppercase_letters"), + ("lowerCASEletters", "lower_caseletters"), + ("VeryLongCamelString", "very_long_camel_string"), + ("kebab-case", "kebab-case"), +]) +def test_snake_case(input_, expected): + output = snake_case(input_) + assert output == expected diff --git a/tests/agent_unittests/test_environment.py b/tests/agent_unittests/test_environment.py index b2c639adc2..22a102cd14 100644 --- a/tests/agent_unittests/test_environment.py +++ b/tests/agent_unittests/test_environment.py @@ -15,12 +15,16 @@ import sys import pytest +from testing_support.fixtures import override_generic_settings +from newrelic.core.config import global_settings from newrelic.core.environment import environment_settings +settings = global_settings() + def module(version): - class Module(object): + class Module(): pass if version: @@ -44,10 +48,27 @@ def test_plugin_list(): # Check that bogus plugins don't get reported assert "newrelic.hooks.newrelic" not in plugin_list # Check that plugin that should get reported has version info. - assert "pytest (%s)" % (pytest.__version__) in plugin_list + assert f"pytest ({pytest.__version__})" in plugin_list + + +@override_generic_settings(settings, {"package_reporting.enabled": False}) +def test_plugin_list_when_package_reporting_disabled(): + # Let's pretend we fired an import hook + import newrelic.hooks.adapter_gunicorn # noqa: F401 + + environment_info = environment_settings() + + for key, plugin_list in environment_info: + if key == "Plugin List": + break + else: + assert False, "'Plugin List' not found" + + # Check that bogus plugins don't get reported + assert plugin_list == [] -class NoIteratorDict(object): +class NoIteratorDict(): def __init__(self, d): self.d = d @@ -92,6 +113,17 @@ def test_plugin_list_uses_no_sys_modules_iterator(monkeypatch): "1.2.3", "4.5.6", ), + # New replacement module uvicorn_worker should function the same + ( + { + "gunicorn": module("1.2.3"), + "uvicorn": module("4.5.6"), + "uvicorn_worker": object(), + }, + "gunicorn (uvicorn)", + "1.2.3", + "4.5.6", + ), ({"uvicorn": object()}, "uvicorn", None, None), ( { diff --git a/tests/agent_unittests/test_full_uri_payloads.py b/tests/agent_unittests/test_full_uri_payloads.py index 3ba321e34f..054b3e1e73 100644 --- a/tests/agent_unittests/test_full_uri_payloads.py +++ b/tests/agent_unittests/test_full_uri_payloads.py @@ -20,14 +20,17 @@ from testing_support.fixtures import collector_agent_registration_fixture from newrelic.core.agent_protocol import AgentProtocol from newrelic.common.agent_http import HttpClient -from newrelic.core.config import global_settings +from newrelic.core.config import global_settings, _environ_as_bool + +DEVELOPER_MODE = _environ_as_bool("NEW_RELIC_DEVELOPER_MODE", False) or "NEW_RELIC_LICENSE_KEY" not in os.environ +SKIP_IF_DEVELOPER_MODE = pytest.mark.skipif(DEVELOPER_MODE, reason="Cannot connect to collector in developer mode") class FullUriClient(HttpClient): def send_request( self, method="POST", path="/agent_listener/invoke_raw_method", *args, **kwargs ): - path = "https://" + self._host + path + path = f"https://{self._host}{path}" return super(FullUriClient, self).send_request(method, path, *args, **kwargs) @@ -55,10 +58,7 @@ def session(application): } -@pytest.mark.skipif( - "NEW_RELIC_LICENSE_KEY" not in os.environ, - reason="License key is not expected to be valid", -) +@SKIP_IF_DEVELOPER_MODE @pytest.mark.parametrize( "method,payload", [ @@ -86,10 +86,7 @@ def test_full_uri_payload(session, method, payload): protocol.send(method, payload) -@pytest.mark.skipif( - "NEW_RELIC_LICENSE_KEY" not in os.environ, - reason="License key is not expected to be valid", -) +@SKIP_IF_DEVELOPER_MODE def test_full_uri_connect(): # An exception will be raised here if there's a problem with the response AgentProtocol.connect( diff --git a/tests/agent_unittests/test_harvest_loop.py b/tests/agent_unittests/test_harvest_loop.py index 3056221079..a3eaf7b5ff 100644 --- a/tests/agent_unittests/test_harvest_loop.py +++ b/tests/agent_unittests/test_harvest_loop.py @@ -32,7 +32,7 @@ from newrelic.core.function_node import FunctionNode from newrelic.core.log_event_node import LogEventNode from newrelic.core.root_node import RootNode -from newrelic.core.stats_engine import CustomMetrics, SampledDataSet +from newrelic.core.stats_engine import CustomMetrics, SampledDataSet, DimensionalMetrics from newrelic.core.transaction_node import TransactionNode from newrelic.network.exceptions import RetryDataForRequest @@ -49,6 +49,11 @@ def transaction_node(request): event = create_custom_event("Custom", {}) custom_events.add(event) + ml_events = SampledDataSet(capacity=num_events) + for _ in range(num_events): + event = create_custom_event("Custom", {}) + ml_events.add(event) + log_events = SampledDataSet(capacity=num_events) for _ in range(num_events): event = LogEventNode(1653609717, "WARNING", "A", {}) @@ -122,10 +127,12 @@ def transaction_node(request): errors=errors, slow_sql=(), custom_events=custom_events, + ml_events=ml_events, log_events=log_events, apdex_t=0.5, suppress_apdex=False, custom_metrics=CustomMetrics(), + dimensional_metrics=DimensionalMetrics(), guid="4485b89db608aece", cpu_time=0.0, suppress_transaction_trace=False, @@ -136,6 +143,10 @@ def transaction_node(request): synthetics_job_id=None, synthetics_monitor_id=None, synthetics_header=None, + synthetics_type=None, + synthetics_initiator=None, + synthetics_attributes=None, + synthetics_info_header=None, is_part_of_cat=False, trip_id="4485b89db608aece", path_hash=None, @@ -818,6 +829,7 @@ def test_flexible_events_harvested(allowlist_event): app._stats_engine.log_events.add(LogEventNode(1653609717, "WARNING", "A", {})) app._stats_engine.span_events.add("span event") app._stats_engine.record_custom_metric("CustomMetric/Int", 1) + app._stats_engine.record_dimensional_metric("DimensionalMetric/Int", 1, tags={"tag": "tag"}) assert app._stats_engine.transaction_events.num_seen == 1 assert app._stats_engine.error_events.num_seen == 1 @@ -825,6 +837,7 @@ def test_flexible_events_harvested(allowlist_event): assert app._stats_engine.log_events.num_seen == 1 assert app._stats_engine.span_events.num_seen == 1 assert app._stats_engine.record_custom_metric("CustomMetric/Int", 1) + assert app._stats_engine.record_dimensional_metric("DimensionalMetric/Int", 1, tags={"tag": "tag"}) app.harvest(flexible=True) @@ -844,7 +857,8 @@ def test_flexible_events_harvested(allowlist_event): assert app._stats_engine.span_events.num_seen == num_seen assert ("CustomMetric/Int", "") in app._stats_engine.stats_table - assert app._stats_engine.metrics_count() > 1 + assert ("DimensionalMetric/Int", frozenset({("tag", "tag")})) in app._stats_engine.dimensional_stats_table + assert app._stats_engine.metrics_count() > 3 @pytest.mark.parametrize( diff --git a/tests/agent_unittests/test_http_client.py b/tests/agent_unittests/test_http_client.py index a5c340d6a6..9d8d398ba2 100644 --- a/tests/agent_unittests/test_http_client.py +++ b/tests/agent_unittests/test_http_client.py @@ -18,11 +18,12 @@ import ssl import zlib +from io import StringIO + import pytest -from testing_support.mock_external_http_server import ( - BaseHTTPServer, - MockExternalHTTPServer, -) + +from http.server import BaseHTTPRequestHandler, HTTPServer +from testing_support.mock_external_http_server import MockExternalHTTPServer from newrelic.common import certs from newrelic.common.agent_http import ( @@ -39,11 +40,6 @@ from newrelic.network.exceptions import NetworkInterfaceException from newrelic.packages.urllib3.util import Url -try: - from StringIO import StringIO -except ImportError: - from io import StringIO - SERVER_CERT = os.path.join(os.path.dirname(__file__), "cert.pem") @@ -51,7 +47,7 @@ def echo_full_request(self): self.server.connections.append(self.connection) request_line = str(self.requestline).encode("utf-8") - headers = "\n".join("%s: %s" % (k.lower(), v) for k, v in self.headers.items()) + headers = "\n".join(f"{k.lower()}: {v}" for k, v in self.headers.items()) self.send_response(200) self.end_headers() self.wfile.write(request_line) @@ -81,12 +77,12 @@ def do_CONNECT(self): handler = type( "ResponseHandler", ( - BaseHTTPServer.BaseHTTPRequestHandler, + BaseHTTPRequestHandler, object, ), {"do_GET": handler, "do_POST": handler, "do_CONNECT": do_CONNECT}, ) - self.httpd = BaseHTTPServer.HTTPServer(("localhost", self.port), handler) + self.httpd = HTTPServer(("localhost", self.port), handler) self.httpd.connections = [] self.httpd.connect_host = None self.httpd.connect_port = None @@ -199,7 +195,7 @@ def test_http_no_payload(server, method): assert connection.pool is None # Verify request line - assert data[0].startswith(method + " /agent_listener/invoke_raw_method ") + assert data[0].startswith(f"{method} /agent_listener/invoke_raw_method ") # Verify headers user_agent_header = "" @@ -234,7 +230,7 @@ def test_non_ok_response(client_cls, server): assert internal_metrics == { "Supportability/Python/Collector/Failures": [1, 0, 0, 0, 0, 0], "Supportability/Python/Collector/Failures/direct": [1, 0, 0, 0, 0, 0], - "Supportability/Python/Collector/HTTPError/%d" % status: [1, 0, 0, 0, 0, 0], + f"Supportability/Python/Collector/HTTPError/{status}": [1, 0, 0, 0, 0, 0], } else: assert not internal_metrics @@ -325,7 +321,7 @@ def test_http_payload_compression(server, client_cls, method, threshold): # Verify the compressed payload length is recorded assert internal_metrics["Supportability/Python/Collector/method1/ZLIB/Bytes"][:2] == [1, payload_byte_len] assert internal_metrics["Supportability/Python/Collector/ZLIB/Bytes"][:2] == [2, payload_byte_len*2] - + assert len(internal_metrics) == 8 else: # Verify no ZLIB compression metrics were sent @@ -366,11 +362,14 @@ def test_cert_path(server): def test_default_cert_path(monkeypatch, system_certs_available): if system_certs_available: cert_file = "foo" + ca_path = "/usr/certs" else: cert_file = None + ca_path = None - class DefaultVerifyPaths(object): + class DefaultVerifyPaths(): cafile = cert_file + capath = ca_path def __init__(self, *args, **kwargs): pass @@ -420,8 +419,8 @@ def test_ssl_via_ssl_proxy(server, auth): if proxy_user: auth_expected = proxy_user if proxy_pass: - auth_expected = auth_expected + ":" + proxy_pass - auth_expected = "Basic " + base64.b64encode(auth_expected.encode("utf-8")).decode("utf-8") + auth_expected = f"{auth_expected}:{proxy_pass}" + auth_expected = f"Basic {base64.b64encode(auth_expected.encode('utf-8')).decode('utf-8')}" assert proxy_auth == auth_expected else: assert not proxy_auth @@ -488,8 +487,8 @@ def test_ssl_via_non_ssl_proxy(insecure_server, auth): if proxy_user: auth_expected = proxy_user if proxy_pass: - auth_expected = auth_expected + ":" + proxy_pass - auth_expected = "Basic " + base64.b64encode(auth_expected.encode("utf-8")).decode("utf-8") + auth_expected = f"{auth_expected}:{proxy_pass}" + auth_expected = f"Basic {base64.b64encode(auth_expected.encode('utf-8')).decode('utf-8')}" assert insecure_server.httpd.connect_headers["proxy-authorization"] == auth_expected else: assert "proxy-authorization" not in insecure_server.httpd.connect_headers @@ -629,8 +628,8 @@ def test_audit_logging(server, insecure_server, client_cls, proxy_host, exceptio connection = "direct" assert internal_metrics == { "Supportability/Python/Collector/Failures": [1, 0, 0, 0, 0, 0], - "Supportability/Python/Collector/Failures/%s" % connection: [1, 0, 0, 0, 0, 0], - "Supportability/Python/Collector/Exception/%s" % exc: [1, 0, 0, 0, 0, 0], + f"Supportability/Python/Collector/Failures/{connection}": [1, 0, 0, 0, 0, 0], + f"Supportability/Python/Collector/Exception/{exc}": [1, 0, 0, 0, 0, 0], } else: assert not internal_metrics diff --git a/tests/agent_unittests/test_import_hook.py b/tests/agent_unittests/test_import_hook.py index fa414484c2..65ad885d35 100644 --- a/tests/agent_unittests/test_import_hook.py +++ b/tests/agent_unittests/test_import_hook.py @@ -12,10 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys - import newrelic.api.import_hook as import_hook -import newrelic.packages.six as six import pytest from newrelic.config import _module_function_glob @@ -41,25 +38,19 @@ def test_import_hook_finder(monkeypatch): } monkeypatch.setattr(import_hook, "_import_hooks", registered_hooks) - # Finding a module that does not exist and is not registered returns None. - module = finder.find_module("module_does_not_exist") + # Finding a module that does not exist returns None, whether or not it is registered. + module = finder.find_spec("module_does_not_exist") assert module is None - # Finding a module that does not exist and is registered behaves - # differently on python 2 vs python 3. - if six.PY2: - with pytest.raises(ImportError): - module = finder.find_module("registered_but_does_not_exist") - else: - module = finder.find_module("registered_but_does_not_exist") - assert module is None + module = finder.find_spec("registered_but_does_not_exist") + assert module is None # Finding a module that exists, but is not registered returns None. - module = finder.find_module("newrelic") + module = finder.find_spec("newrelic") assert module is None # Finding a module that exists, and is registered, finds that module. - module = finder.find_module("newrelic.api") + module = finder.find_spec("newrelic.api") assert module is not None diff --git a/tests/agent_unittests/test_package_version_utils.py b/tests/agent_unittests/test_package_version_utils.py index d80714d778..8b2abeda86 100644 --- a/tests/agent_unittests/test_package_version_utils.py +++ b/tests/agent_unittests/test_package_version_utils.py @@ -13,6 +13,7 @@ # limitations under the License. import sys +import warnings import pytest from testing_support.validators.validate_function_called import validate_function_called @@ -20,15 +21,24 @@ from newrelic.common.package_version_utils import ( NULL_VERSIONS, VERSION_ATTRS, + _get_package_version, get_package_version, get_package_version_tuple, ) +# Notes: +# importlib.metadata was a provisional addition to the std library in PY38 and PY39 +# while pkg_resources was deprecated. +# importlib.metadata is no longer provisional in PY310+. It added some attributes +# such as distribution_packages and removed pkg_resources. + IS_PY38_PLUS = sys.version_info[:2] >= (3, 8) +IS_PY310_PLUS = sys.version_info[:2] >= (3, 10) SKIP_IF_NOT_IMPORTLIB_METADATA = pytest.mark.skipif(not IS_PY38_PLUS, reason="importlib.metadata is not supported.") SKIP_IF_IMPORTLIB_METADATA = pytest.mark.skipif( IS_PY38_PLUS, reason="importlib.metadata is preferred over pkg_resources." ) +SKIP_IF_NOT_PY310_PLUS = pytest.mark.skipif(not IS_PY310_PLUS, reason="These features were added in 3.10+") @pytest.fixture(scope="function", autouse=True) @@ -40,6 +50,12 @@ def patched_pytest_module(monkeypatch): yield pytest +@pytest.fixture(scope="function", autouse=True) +def cleared_package_version_cache(): + """Ensure cache is empty before every test to exercise code paths.""" + _get_package_version.cache_clear() + + @pytest.mark.parametrize( "attr,value,expected_value", ( @@ -49,13 +65,23 @@ def patched_pytest_module(monkeypatch): ("version_tuple", [3, 1, "0b2"], "3.1.0b2"), ), ) -def test_get_package_version(attr, value, expected_value): +def test_get_package_version(monkeypatch, attr, value, expected_value): # There is no file/module here, so we monkeypatch # pytest instead for our purposes - setattr(pytest, attr, value) + monkeypatch.setattr(pytest, attr, value, raising=False) version = get_package_version("pytest") assert version == expected_value - delattr(pytest, attr) + + +def test_skips_version_callables(monkeypatch): + # There is no file/module here, so we monkeypatch + # pytest instead for our purposes + monkeypatch.setattr(pytest, "version", lambda x: "1.2.3.4", raising=False) + monkeypatch.setattr(pytest, "version_tuple", [3, 1, "0b2"], raising=False) + + version = get_package_version("pytest") + + assert version == "3.1.0b2" @pytest.mark.parametrize( @@ -67,13 +93,12 @@ def test_get_package_version(attr, value, expected_value): ("version_tuple", [3, 1, "0b2"], (3, 1, "0b2")), ), ) -def test_get_package_version_tuple(attr, value, expected_value): +def test_get_package_version_tuple(monkeypatch, attr, value, expected_value): # There is no file/module here, so we monkeypatch # pytest instead for our purposes - setattr(pytest, attr, value) + monkeypatch.setattr(pytest, attr, value, raising=False) version = get_package_version_tuple("pytest") assert version == expected_value - delattr(pytest, attr) @SKIP_IF_NOT_IMPORTLIB_METADATA @@ -83,8 +108,45 @@ def test_importlib_metadata(): assert version not in NULL_VERSIONS, version +@SKIP_IF_NOT_PY310_PLUS +@validate_function_called("importlib.metadata", "packages_distributions") +def test_mapping_import_to_distribution_packages(): + version = get_package_version("pytest") + assert version not in NULL_VERSIONS, version + + @SKIP_IF_IMPORTLIB_METADATA @validate_function_called("pkg_resources", "get_distribution") def test_pkg_resources_metadata(): version = get_package_version("pytest") assert version not in NULL_VERSIONS, version + + +def _getattr_deprecation_warning(attr): + if attr == "__version__": + warnings.warn("Testing deprecation warnings.", DeprecationWarning) + return "3.2.1" + else: + raise NotImplementedError() + + +def test_deprecation_warning_suppression(monkeypatch, recwarn): + # Add fake module to be deleted later + monkeypatch.setattr(pytest, "__getattr__", _getattr_deprecation_warning, raising=False) + + assert get_package_version("pytest") == "3.2.1" + + assert not recwarn.list, "Warnings not suppressed." + + +def test_version_caching(monkeypatch): + # Add fake module to be deleted later + sys.modules["mymodule"] = sys.modules["pytest"] + monkeypatch.setattr(pytest, "__version__", "1.0.0", raising=False) + version = get_package_version("mymodule") + assert version not in NULL_VERSIONS, version + + # Ensure after deleting that the call to _get_package_version still completes because of caching + del sys.modules["mymodule"] + version = get_package_version("mymodule") + assert version not in NULL_VERSIONS, version diff --git a/tests/agent_unittests/test_region_aware_settings.py b/tests/agent_unittests/test_region_aware_settings.py index 7b47640497..a1449822bb 100644 --- a/tests/agent_unittests/test_region_aware_settings.py +++ b/tests/agent_unittests/test_region_aware_settings.py @@ -19,24 +19,24 @@ """ NO_REGION_KEY = '66c637a29c3982469a3fe8d1982d002c4a' -INI_FILE_NO_REGION_KEY = """ +INI_FILE_NO_REGION_KEY = f""" [newrelic] -license_key = %s -""" % NO_REGION_KEY +license_key = {NO_REGION_KEY} +""" INI_FILE_NO_REGION_KEY = INI_FILE_NO_REGION_KEY.encode('utf-8') EU01_KEY = 'eu01xx66c637a29c3982469a3fe8d1982d002c4a' -INI_FILE_EU01_KEY = """ +INI_FILE_EU01_KEY = f""" [newrelic] -license_key = %s -""" % EU01_KEY +license_key = {EU01_KEY} +""" INI_FILE_EU01_KEY = INI_FILE_EU01_KEY.encode('utf-8') -INI_FILE_HOST_OVERRIDE = """ +INI_FILE_HOST_OVERRIDE = f""" [newrelic] host = staging-collector.newrelic.com -license_key = %s -""" % EU01_KEY +license_key = {EU01_KEY} +""" INI_FILE_HOST_OVERRIDE = INI_FILE_HOST_OVERRIDE.encode('utf-8') STAGING_HOST = 'staging-collector.newrelic.com' diff --git a/tests/agent_unittests/test_sampler_metrics.py b/tests/agent_unittests/test_sampler_metrics.py index 3f3b42a690..a2f666c22d 100644 --- a/tests/agent_unittests/test_sampler_metrics.py +++ b/tests/agent_unittests/test_sampler_metrics.py @@ -20,7 +20,6 @@ from testing_support.fixtures import override_generic_settings from newrelic.core.config import global_settings -from newrelic.packages import six from newrelic.samplers.cpu_usage import cpu_usage_data_source from newrelic.samplers.gc_data import garbage_collector_data_source from newrelic.samplers.memory_usage import memory_usage_data_source @@ -52,36 +51,28 @@ def memory_data_source(): PID = os.getpid() -if six.PY2: - EXPECTED_GC_METRICS = ( - "GC/objects/%d/all" % PID, - "GC/objects/%d/generation/0" % PID, - "GC/objects/%d/generation/1" % PID, - "GC/objects/%d/generation/2" % PID, - ) -else: - EXPECTED_GC_METRICS = ( - "GC/objects/%d/all" % PID, - "GC/objects/%d/generation/0" % PID, - "GC/objects/%d/generation/1" % PID, - "GC/objects/%d/generation/2" % PID, - "GC/collections/%d/all" % PID, - "GC/collections/%d/0" % PID, - "GC/collections/%d/1" % PID, - "GC/collections/%d/2" % PID, - "GC/collected/%d/all" % PID, - "GC/collected/%d/0" % PID, - "GC/collected/%d/1" % PID, - "GC/collected/%d/2" % PID, - "GC/uncollectable/%d/all" % PID, - "GC/uncollectable/%d/0" % PID, - "GC/uncollectable/%d/1" % PID, - "GC/uncollectable/%d/2" % PID, - "GC/time/%d/all" % PID, - "GC/time/%d/0" % PID, - "GC/time/%d/1" % PID, - "GC/time/%d/2" % PID, - ) +EXPECTED_GC_METRICS = ( + f"GC/objects/{PID}/all", + f"GC/objects/{PID}/generation/0", + f"GC/objects/{PID}/generation/1", + f"GC/objects/{PID}/generation/2", + f"GC/collections/{PID}/all", + f"GC/collections/{PID}/0", + f"GC/collections/{PID}/1", + f"GC/collections/{PID}/2", + f"GC/collected/{PID}/all", + f"GC/collected/{PID}/0", + f"GC/collected/{PID}/1", + f"GC/collected/{PID}/2", + f"GC/uncollectable/{PID}/all", + f"GC/uncollectable/{PID}/0", + f"GC/uncollectable/{PID}/1", + f"GC/uncollectable/{PID}/2", + f"GC/time/{PID}/all", + f"GC/time/{PID}/0", + f"GC/time/{PID}/1", + f"GC/time/{PID}/2", +) @pytest.mark.xfail( @@ -152,14 +143,22 @@ def test_cpu_metrics_collection(cpu_data_source): EXPECTED_MEMORY_METRICS = ( "Memory/Physical", - "Memory/Physical/%d" % PID, "Memory/Physical/Utilization", - "Memory/Physical/Utilization/%d" % PID, + f"Memory/Physical/{PID}", + f"Memory/Physical/Utilization/{PID}", ) -def test_memory_metrics_collection(memory_data_source): - metrics_table = set(m[0] for m in (memory_data_source() or ())) +@pytest.mark.parametrize("enabled", (True, False)) +def test_memory_metrics_collection(memory_data_source, enabled): + @override_generic_settings(settings, {"memory_runtime_pid_metrics.enabled": enabled}) + def _test(): + metrics_table = set(m[0] for m in (memory_data_source() or ())) + if enabled: + for metric in EXPECTED_MEMORY_METRICS: + assert metric in metrics_table + else: + assert EXPECTED_MEMORY_METRICS[0] in metrics_table + assert EXPECTED_MEMORY_METRICS[1] in metrics_table - for metric in EXPECTED_MEMORY_METRICS: - assert metric in metrics_table + _test() diff --git a/tests/agent_unittests/test_trace_cache.py b/tests/agent_unittests/test_trace_cache.py index e0f7db84fa..71046efb14 100644 --- a/tests/agent_unittests/test_trace_cache.py +++ b/tests/agent_unittests/test_trace_cache.py @@ -21,7 +21,7 @@ _TEST_CONCURRENT_ITERATION_TC_SIZE = 20 -class DummyTrace(object): +class DummyTrace(): pass diff --git a/tests/agent_unittests/test_utilization_settings.py b/tests/agent_unittests/test_utilization_settings.py index 8af4bcbf1b..84b370e6b1 100644 --- a/tests/agent_unittests/test_utilization_settings.py +++ b/tests/agent_unittests/test_utilization_settings.py @@ -15,6 +15,8 @@ import os import tempfile +from importlib import reload + import pytest # these will be reloaded for each test @@ -34,12 +36,6 @@ global_settings, ) -try: - # python 2.x - reload -except NameError: - # python 3.x - from importlib import reload INI_FILE_WITHOUT_UTIL_CONF = b""" [newrelic] @@ -76,7 +72,7 @@ # Tests for loading settings and testing for values precedence -class Environ(object): +class Environ(): def __init__(self, env_dict): self.env_dict = {} for key in env_dict.keys(): @@ -118,6 +114,22 @@ def reset(wrapped, instance, args, kwargs): return reset +@reset_agent_config(INI_FILE_WITHOUT_UTIL_CONF, {"NEW_RELIC_HOST": "collector.newrelic.com"}) +def test_otlp_host_port_default(): + settings = global_settings() + assert settings.otlp_host == "otlp.nr-data.net" + assert settings.otlp_port == 0 + + +@reset_agent_config( + INI_FILE_WITHOUT_UTIL_CONF, {"NEW_RELIC_OTLP_HOST": "custom-otlp.nr-data.net", "NEW_RELIC_OTLP_PORT": 443} +) +def test_otlp_port_override(): + settings = global_settings() + assert settings.otlp_host == "custom-otlp.nr-data.net" + assert settings.otlp_port == 443 + + @reset_agent_config(INI_FILE_WITHOUT_UTIL_CONF, ENV_WITHOUT_UTIL_CONF) def test_heroku_default(): settings = global_settings() diff --git a/tests/agent_unittests/test_wrappers.py b/tests/agent_unittests/test_wrappers.py new file mode 100644 index 0000000000..eccee4df5b --- /dev/null +++ b/tests/agent_unittests/test_wrappers.py @@ -0,0 +1,81 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from newrelic.common.object_wrapper import function_wrapper + + +@pytest.fixture(scope="function") +def wrapper(): + @function_wrapper + def _wrapper(wrapped, instance, args, kwargs): + return wrapped(*args, **kwargs) + + return _wrapper + + +@pytest.fixture(scope="function") +def wrapped_function(wrapper): + @wrapper + def wrapped(): + return True + + return wrapped + + +def test_nr_prefix_attributes(wrapped_function): + wrapped_function._nr_attr = 1 + vars_ = vars(wrapped_function) + + assert wrapped_function._nr_attr == 1, "_nr_ attributes should be stored on wrapper object and retrievable." + assert "_nr_attr" not in vars_, "_nr_ attributes should NOT appear in __dict__." + + +def test_self_prefix_attributes(wrapped_function): + wrapped_function._self_attr = 1 + vars_ = vars(wrapped_function) + + assert wrapped_function._self_attr == 1, "_self_ attributes should be stored on wrapper object and retrievable." + assert "_nr_attr" not in vars_, "_self_ attributes should NOT appear in __dict__." + + +def test_prefixed_attributes_share_namespace(wrapped_function): + wrapped_function._nr_attr = 1 + wrapped_function._self_attr = 2 + + assert ( + wrapped_function._nr_attr == 2 + ), "_nr_ attributes share a namespace with _self_ attributes and should be overwritten." + + +def test_wrapped_function_attributes(wrapped_function): + wrapped_function._other_attr = 1 + vars_ = vars(wrapped_function) + + assert wrapped_function._other_attr == 1, "All other attributes should be stored on wrapped object and retrievable." + assert "_other_attr" in vars_, "Other types of attributes SHOULD appear in __dict__." + + assert wrapped_function() + + +def test_multiple_wrapper_last_object(wrapper): + def wrapped(): + pass + + wrapper_1 = wrapper(wrapped) + wrapper_2 = wrapper(wrapper_1) + + assert wrapper_2._nr_last_object is wrapped, "Last object in chain should be the wrapped function." + assert wrapper_2._nr_next_object is wrapper_1, "Next object in chain should be the middle function." diff --git a/tests/application_celery/_target_application.py b/tests/application_celery/_target_application.py new file mode 100644 index 0000000000..d5aa6a7a7e --- /dev/null +++ b/tests/application_celery/_target_application.py @@ -0,0 +1,59 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from celery import Celery, shared_task +from testing_support.validators.validate_distributed_trace_accepted import ( + validate_distributed_trace_accepted, +) + +from newrelic.api.transaction import current_transaction + +app = Celery( + "tasks", + broker_url="memory://", + result_backend="cache+memory://", + worker_hijack_root_logger=False, + pool="solo", + broker_heartbeat=0, +) + + +@app.task +def add(x, y): + return x + y + + +@app.task +def tsum(nums): + return sum(nums) + + +@app.task +def nested_add(x, y): + return add(x, y) + + +@shared_task +def shared_task_add(x, y): + return x + y + + +@app.task +@validate_distributed_trace_accepted(transport_type="AMQP") +def assert_dt(): + # Basic checks for DT delegated to task + txn = current_transaction() + assert txn, "No transaction active." + assert txn.name == "_target_application.assert_dt", f"Transaction name does not match: {txn.name}" + return 1 diff --git a/tests/application_celery/conftest.py b/tests/application_celery/conftest.py index 49f0fe4776..90bc93c719 100644 --- a/tests/application_celery/conftest.py +++ b/tests/application_celery/conftest.py @@ -11,20 +11,41 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (application_celery)', - default_settings=_default_settings) + app_name="Python Agent Test (application_celery)", default_settings=_default_settings +) + + +@pytest.fixture(scope="session") +def celery_config(): + # Used by celery pytest plugin to configure Celery instance + return { + "broker_url": "memory://", + "result_backend": "cache+memory://", + } + + +@pytest.fixture(scope="session") +def celery_worker_parameters(): + # Used by celery pytest plugin to configure worker instance + return {"shutdown_timeout": 120} + + +@pytest.fixture(scope="session", autouse=True) +def celery_worker_available(celery_session_worker): + yield celery_session_worker diff --git a/tests/application_celery/test_celery.py b/tests/application_celery/test_application.py similarity index 52% rename from tests/application_celery/test_celery.py rename to tests/application_celery/test_application.py index c2f9177fa5..c9e5bb3b2a 100644 --- a/tests/application_celery/test_celery.py +++ b/tests/application_celery/test_application.py @@ -12,20 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -from newrelic.api.background_task import background_task -from newrelic.api.transaction import ignore_transaction, end_of_transaction - -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_code_level_metrics import validate_code_level_metrics +from _target_application import add, nested_add, shared_task_add, tsum +from testing_support.validators.validate_code_level_metrics import ( + validate_code_level_metrics, +) +from testing_support.validators.validate_transaction_count import ( + validate_transaction_count, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) -from tasks import add, tsum +from newrelic.api.background_task import background_task +from newrelic.api.transaction import end_of_transaction, ignore_transaction @validate_transaction_metrics( - name='test_celery:test_celery_task_as_function_trace', - scoped_metrics=[('Function/tasks.add', 1)], - background_task=True) -@validate_code_level_metrics("tasks", "add") + name="test_application:test_celery_task_as_function_trace", + scoped_metrics=[("Function/_target_application.add", 1)], + background_task=True, +) +@validate_code_level_metrics("_target_application", "add") @background_task() def test_celery_task_as_function_trace(): """ @@ -37,12 +44,8 @@ def test_celery_task_as_function_trace(): assert result == 7 -@validate_transaction_metrics( - name='tasks.add', - group='Celery', - scoped_metrics=[], - background_task=True) -@validate_code_level_metrics("tasks", "add") +@validate_transaction_metrics(name="_target_application.add", group="Celery", scoped_metrics=[], background_task=True) +@validate_code_level_metrics("_target_application", "add") def test_celery_task_as_background_task(): """ Calling add() outside of a transaction means the agent will create @@ -53,12 +56,13 @@ def test_celery_task_as_background_task(): result = add(3, 4) assert result == 7 + @validate_transaction_metrics( - name='test_celery:test_celery_tasks_multiple_function_traces', - scoped_metrics=[('Function/tasks.add', 1), - ('Function/tasks.tsum', 1)], - background_task=True) -@validate_code_level_metrics("tasks", "tsum") + name="test_application:test_celery_tasks_multiple_function_traces", + scoped_metrics=[("Function/_target_application.add", 1), ("Function/_target_application.tsum", 1)], + background_task=True, +) +@validate_code_level_metrics("_target_application", "tsum") @background_task() def test_celery_tasks_multiple_function_traces(): add_result = add(5, 6) @@ -74,8 +78,8 @@ def test_celery_tasks_ignore_transaction(): No transaction is recorded, due to the call to ignore_transaction(), so no validation fixture is used. The purpose of this test is to make sure the agent doesn't throw an error. - """ + add_result = add(1, 2) assert add_result == 3 @@ -86,16 +90,17 @@ def test_celery_tasks_ignore_transaction(): @validate_transaction_metrics( - name='test_celery:test_celery_tasks_end_transaction', - scoped_metrics=[('Function/tasks.add', 1)], - background_task=True) + name="test_application:test_celery_tasks_end_transaction", + scoped_metrics=[("Function/_target_application.add", 1)], + background_task=True, +) @background_task() def test_celery_tasks_end_transaction(): """ Only functions that run before the call to end_of_transaction() are included in the transaction. - """ + add_result = add(1, 2) assert add_result == 3 @@ -103,3 +108,36 @@ def test_celery_tasks_end_transaction(): tsum_result = tsum([1, 2, 3]) assert tsum_result == 6 + + +@validate_transaction_metrics( + name="_target_application.nested_add", + group="Celery", + scoped_metrics=[("Function/_target_application.add", 1)], + background_task=True, +) +@validate_transaction_count(1) +@validate_code_level_metrics("_target_application", "nested_add") +def test_celery_nested_tasks(): + """ + Celery tasks run inside other celery tasks should not start a new transactions, + and should create a function trace instead. + """ + + add_result = nested_add(1, 2) + assert add_result == 3 + + +@validate_transaction_metrics( + name="_target_application.shared_task_add", group="Celery", scoped_metrics=[], background_task=True +) +@validate_code_level_metrics("_target_application", "shared_task_add") +def test_celery_shared_task_as_background_task(): + """ + Calling shared_task_add() outside of a transaction means the agent will create + a background transaction (with a group of 'Celery') and record shared_task_add() + as a background task. + + """ + result = shared_task_add(3, 4) + assert result == 7 diff --git a/tests/application_celery/test_distributed_tracing.py b/tests/application_celery/test_distributed_tracing.py new file mode 100644 index 0000000000..2b6eccf542 --- /dev/null +++ b/tests/application_celery/test_distributed_tracing.py @@ -0,0 +1,69 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from _target_application import add, assert_dt +from testing_support.fixtures import override_application_settings +from testing_support.validators.validate_transaction_count import ( + validate_transaction_count, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + + +@validate_transaction_metrics( + name="_target_application.assert_dt", + group="Celery", + rollup_metrics=[ + ("Supportability/DistributedTrace/AcceptPayload/Success", None), + ("Supportability/TraceContext/Accept/Success", 1), + ], + background_task=True, + index=-2, +) +@validate_transaction_metrics( + name="test_distributed_tracing:test_celery_task_distributed_tracing_enabled", + background_task=True, +) +@validate_transaction_count(2) +@background_task() +def test_celery_task_distributed_tracing_enabled(): + result = assert_dt.apply_async() + result = result.get() + assert result == 1 + + +@override_application_settings({"distributed_tracing.enabled": False}) +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + rollup_metrics=[ + ("Supportability/DistributedTrace/AcceptPayload/Success", None), + ("Supportability/TraceContext/Accept/Success", None), # No trace context should be accepted + ], + background_task=True, + index=-2, +) +@validate_transaction_metrics( + name="test_distributed_tracing:test_celery_task_distributed_tracing_disabled", + background_task=True, +) +@validate_transaction_count(2) +@background_task() +def test_celery_task_distributed_tracing_disabled(): + result = add.apply_async((1, 2)) + result = result.get() + assert result == 3 diff --git a/tests/application_celery/test_celery_max_tasks_per_child.py b/tests/application_celery/test_max_tasks_per_child.py similarity index 65% rename from tests/application_celery/test_celery_max_tasks_per_child.py rename to tests/application_celery/test_max_tasks_per_child.py index bf75ef589d..b7ec1a778e 100644 --- a/tests/application_celery/test_celery_max_tasks_per_child.py +++ b/tests/application_celery/test_max_tasks_per_child.py @@ -13,27 +13,31 @@ # limitations under the License. import pytest - from billiard import get_context from billiard.pool import Worker +from testing_support.validators.validate_function_called import validate_function_called -from testing_support.validators.validate_function_called import ( - validate_function_called) +from newrelic.common.object_wrapper import transient_function_wrapper class OnExit(Exception): pass -@validate_function_called('newrelic.core.agent', 'Agent.shutdown_agent') -def test_max_tasks_per_child(): +@transient_function_wrapper("newrelic.core.agent", "Agent.shutdown_agent") +def mock_agent_shutdown(wrapped, instance, args, kwargs): + # Prevent agent from actually shutting down and blocking further tests + pass + +@mock_agent_shutdown +@validate_function_called("newrelic.core.agent", "Agent.shutdown_agent") +def test_max_tasks_per_child(): def on_exit(*args, **kwargs): raise OnExit() ctx = get_context() - worker = Worker(ctx.SimpleQueue(), ctx.SimpleQueue(), None, - maxtasks=1, on_exit=on_exit) + worker = Worker(ctx.SimpleQueue(), ctx.SimpleQueue(), None, maxtasks=1, on_exit=on_exit) with pytest.raises(OnExit): worker._do_exit(None, 0) diff --git a/tests/application_celery/test_task_methods.py b/tests/application_celery/test_task_methods.py new file mode 100644 index 0000000000..509129b093 --- /dev/null +++ b/tests/application_celery/test_task_methods.py @@ -0,0 +1,328 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from _target_application import add, tsum +from testing_support.validators.validate_code_level_metrics import ( + validate_code_level_metrics, +) +from testing_support.validators.validate_transaction_count import ( + validate_transaction_count, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +import celery + + +FORGONE_TASK_METRICS = [("Function/_target_application.add", None), ("Function/_target_application.tsum", None)] + + +@pytest.fixture(scope="module", autouse=True, params=[False, True], ids=["unpatched", "patched"]) +def with_worker_optimizations(request, celery_worker_available): + if request.param: + celery.app.trace.setup_worker_optimizations(celery_worker_available.app) + + yield request.param + celery.app.trace.reset_worker_optimizations() + + +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, +) +@validate_code_level_metrics("_target_application", "add") +@validate_transaction_count(1) +def test_celery_task_call(): + """ + Executes task in local process and returns the result directly. + """ + result = add(3, 4) + assert result == 7 + + +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, +) +@validate_code_level_metrics("_target_application", "add") +@validate_transaction_count(1) +def test_celery_task_apply(): + """ + Executes task in local process and returns an EagerResult. + """ + result = add.apply((3, 4)) + result = result.get() + assert result == 7 + + +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, +) +@validate_code_level_metrics("_target_application", "add") +@validate_transaction_count(1) +def test_celery_task_delay(): + """ + Executes task on worker process and returns an AsyncResult. + """ + result = add.delay(3, 4) + result = result.get() + assert result == 7 + + +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, +) +@validate_code_level_metrics("_target_application", "add") +@validate_transaction_count(1) +def test_celery_task_apply_async(): + """ + Executes task on worker process and returns an AsyncResult. + """ + result = add.apply_async((3, 4)) + result = result.get() + assert result == 7 + + +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, +) +@validate_code_level_metrics("_target_application", "add") +@validate_transaction_count(1) +def test_celery_app_send_task(celery_session_app): + """ + Executes task on worker process and returns an AsyncResult. + """ + result = celery_session_app.send_task("_target_application.add", (3, 4)) + result = result.get() + assert result == 7 + + +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, +) +@validate_code_level_metrics("_target_application", "add") +@validate_transaction_count(1) +def test_celery_task_signature(): + """ + Executes task on worker process and returns an AsyncResult. + """ + result = add.s(3, 4).delay() + result = result.get() + assert result == 7 + + +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, +) +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, + index=-2, +) +@validate_code_level_metrics("_target_application", "add") +@validate_code_level_metrics("_target_application", "add", index=-2) +@validate_transaction_count(2) +def test_celery_task_link(): + """ + Executes multiple tasks on worker process and returns an AsyncResult. + """ + result = add.apply_async((3, 4), link=[add.s(5)]) + result = result.get() + assert result == 7 # Linked task result won't be returned + + +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, +) +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, + index=-2, +) +@validate_code_level_metrics("_target_application", "add") +@validate_code_level_metrics("_target_application", "add", index=-2) +@validate_transaction_count(2) +def test_celery_chain(): + """ + Executes multiple tasks on worker process and returns an AsyncResult. + """ + result = celery.chain(add.s(3, 4), add.s(5))() + + result = result.get() + assert result == 12 + + +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, +) +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, + index=-2, +) +@validate_code_level_metrics("_target_application", "add") +@validate_code_level_metrics("_target_application", "add", index=-2) +@validate_transaction_count(2) +def test_celery_group(): + """ + Executes multiple tasks on worker process and returns an AsyncResult. + """ + result = celery.group(add.s(3, 4), add.s(1, 2))() + result = result.get() + assert result == [7, 3] + + +@validate_transaction_metrics( + name="_target_application.tsum", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, +) +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, + index=-2, +) +@validate_transaction_metrics( + name="_target_application.add", + group="Celery", + scoped_metrics=FORGONE_TASK_METRICS, + rollup_metrics=FORGONE_TASK_METRICS, + background_task=True, + index=-3, +) +@validate_code_level_metrics("_target_application", "tsum") +@validate_code_level_metrics("_target_application", "add", index=-2) +@validate_code_level_metrics("_target_application", "add", index=-3) +@validate_transaction_count(3) +def test_celery_chord(): + """ + Executes 2 add tasks, followed by a tsum task on the worker process and returns an AsyncResult. + """ + result = celery.chord([add.s(3, 4), add.s(1, 2)])(tsum.s()) + result = result.get() + assert result == 10 + + +@validate_transaction_metrics( + name="celery.map/_target_application.tsum", + group="Celery", + scoped_metrics=[("Function/_target_application.tsum", 2)], + rollup_metrics=[("Function/_target_application.tsum", 2)], + background_task=True, +) +@validate_code_level_metrics("_target_application", "tsum", count=3) +@validate_transaction_count(1) +def test_celery_task_map(): + """ + Executes map task on worker process with original task as a subtask and returns an AsyncResult. + """ + result = tsum.map([(3, 4), (1, 2)]).apply() + result = result.get() + assert result == [7, 3] + + +@validate_transaction_metrics( + name="celery.starmap/_target_application.add", + group="Celery", + scoped_metrics=[("Function/_target_application.add", 2)], + rollup_metrics=[("Function/_target_application.add", 2)], + background_task=True, +) +@validate_code_level_metrics("_target_application", "add", count=3) +@validate_transaction_count(1) +def test_celery_task_starmap(): + """ + Executes starmap task on worker process with original task as a subtask and returns an AsyncResult. + """ + result = add.starmap([(3, 4), (1, 2)]).apply_async() + result = result.get() + assert result == [7, 3] + + +@validate_transaction_metrics( + name="celery.starmap/_target_application.add", + group="Celery", + scoped_metrics=[("Function/_target_application.add", 1)], + rollup_metrics=[("Function/_target_application.add", 1)], + background_task=True, +) +@validate_transaction_metrics( + name="celery.starmap/_target_application.add", + group="Celery", + scoped_metrics=[("Function/_target_application.add", 1)], + rollup_metrics=[("Function/_target_application.add", 1)], + background_task=True, + index=-2, +) +@validate_code_level_metrics("_target_application", "add", count=2) +@validate_code_level_metrics("_target_application", "add", count=2, index=-2) +@validate_transaction_count(2) +def test_celery_task_chunks(): + """ + Executes multiple tasks on worker process and returns an AsyncResult. + """ + result = add.chunks([(3, 4), (1, 2)], n=1).apply_async() + result = result.get() + assert result == [[7], [3]] diff --git a/tests/application_celery/test_wrappers.py b/tests/application_celery/test_wrappers.py new file mode 100644 index 0000000000..1bca1b4361 --- /dev/null +++ b/tests/application_celery/test_wrappers.py @@ -0,0 +1,46 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from _target_application import add + +import celery + +from newrelic.common.object_wrapper import _NRBoundFunctionWrapper + + +FORGONE_TASK_METRICS = [("Function/_target_application.add", None), ("Function/_target_application.tsum", None)] + + +def test_task_wrapping_detection(): + """ + Ensure celery detects our monkeypatching properly and will run our instrumentation + on __call__ and runs that instead of micro-optimizing it away to a run() call. + + If this is not working, most other tests in this file will fail as the different ways + of running celery tasks will not all run our instrumentation. + """ + assert celery.app.trace.task_has_custom(add, "__call__") + + +def test_worker_optimizations_preserve_instrumentation(celery_worker_available): + is_instrumented = lambda: isinstance(celery.app.task.BaseTask.__call__, _NRBoundFunctionWrapper) + + celery.app.trace.reset_worker_optimizations() + assert is_instrumented(), "Instrumentation not initially applied." + + celery.app.trace.setup_worker_optimizations(celery_worker_available.app) + assert is_instrumented(), "setup_worker_optimizations removed instrumentation." + + celery.app.trace.reset_worker_optimizations() + assert is_instrumented(), "reset_worker_optimizations removed instrumentation." diff --git a/tests/application_gearman/conftest.py b/tests/application_gearman/conftest.py index 6a38806e2e..83fa925600 100644 --- a/tests/application_gearman/conftest.py +++ b/tests/application_gearman/conftest.py @@ -12,19 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (application_gearman)', - default_settings=_default_settings) + app_name="Python Agent Test (application_gearman)", default_settings=_default_settings +) diff --git a/tests/application_gearman/test_gearman.py b/tests/application_gearman/test_gearman.py index 7ddc13fdc3..af9fca1293 100644 --- a/tests/application_gearman/test_gearman.py +++ b/tests/application_gearman/test_gearman.py @@ -12,7 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from __future__ import print_function +# ============================================================================================================= +# This framework has not had recent releases and falls outside our support window, so these tests are disabled. +# ============================================================================================================= import os import threading @@ -20,16 +22,18 @@ import gearman from newrelic.api.background_task import background_task +from testing_support.db_settings import gearman_settings worker_thread = None worker_event = threading.Event() gm_client = None -GEARMAND_HOST = os.environ.get("GEARMAND_PORT_4730_TCP_ADDR", "localhost") -GEARMAND_PORT = os.environ.get("GEARMAND_PORT_4730_TCP_PORT", "4730") +GEARMAND_SETTINGS = gearman_settings()[0] +GEARMAND_HOST = GEARMAND_SETTINGS["host"] +GEARMAND_PORT = GEARMAND_SETTINGS["port"] -GEARMAND_ADDR = "%s:%s" % (GEARMAND_HOST, GEARMAND_PORT) +GEARMAND_ADDR = f"{GEARMAND_HOST}:{GEARMAND_PORT}" class GearmanWorker(gearman.GearmanWorker): diff --git a/tests/component_djangorestframework/conftest.py b/tests/component_djangorestframework/conftest.py index a4b37571dc..8a0265dbab 100644 --- a/tests/component_djangorestframework/conftest.py +++ b/tests/component_djangorestframework/conftest.py @@ -12,20 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, - 'debug.log_autorum_middleware': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_autorum_middleware": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (component_djangorestframework)', - default_settings=_default_settings) + app_name="Python Agent Test (component_djangorestframework)", default_settings=_default_settings +) diff --git a/tests/component_djangorestframework/test_application.py b/tests/component_djangorestframework/test_application.py index 9ed60aa33d..829914aec2 100644 --- a/tests/component_djangorestframework/test_application.py +++ b/tests/component_djangorestframework/test_application.py @@ -12,190 +12,167 @@ # See the License for the specific language governing permissions and # limitations under the License. +import django import pytest import webtest +from testing_support.fixtures import function_not_called, override_generic_settings +from testing_support.validators.validate_code_level_metrics import ( + validate_code_level_metrics, +) +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) -from newrelic.packages import six from newrelic.core.config import global_settings -from testing_support.fixtures import ( - override_generic_settings, - function_not_called) -from testing_support.validators.validate_transaction_errors import validate_transaction_errors -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_code_level_metrics import validate_code_level_metrics -import django - -DJANGO_VERSION = tuple(map(int, django.get_version().split('.')[:2])) +DJANGO_VERSION = tuple(map(int, django.get_version().split(".")[:2])) - -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def target_application(): from wsgi import application + test_application = webtest.TestApp(application) return test_application if DJANGO_VERSION >= (1, 10): - url_module_path = 'django.urls.resolvers' + url_module_path = "django.urls.resolvers" # Django 1.10 new style middleware removed individual process_* methods. # All middleware in Django 1.10+ is called through the __call__ methods on # middlwares. - process_request_method = '' - process_view_method = '' - process_response_method = '' + process_request_method = "" + process_view_method = "" + process_response_method = "" else: - url_module_path = 'django.core.urlresolvers' - process_request_method = '.process_request' - process_view_method = '.process_view' - process_response_method = '.process_response' + url_module_path = "django.core.urlresolvers" + process_request_method = ".process_request" + process_view_method = ".process_view" + process_response_method = ".process_response" if DJANGO_VERSION >= (2, 0): - url_resolver_cls = 'URLResolver' + url_resolver_cls = "URLResolver" else: - url_resolver_cls = 'RegexURLResolver' + url_resolver_cls = "RegexURLResolver" _scoped_metrics = [ - ('Function/django.core.handlers.wsgi:WSGIHandler.__call__', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - (('Function/django.middleware.common:' - 'CommonMiddleware' + process_request_method), 1), - (('Function/django.contrib.sessions.middleware:' - 'SessionMiddleware' + process_request_method), 1), - (('Function/django.contrib.auth.middleware:' - 'AuthenticationMiddleware' + process_request_method), 1), - (('Function/django.contrib.messages.middleware:' - 'MessageMiddleware' + process_request_method), 1), - (('Function/%s:' % url_module_path + - '%s.resolve' % url_resolver_cls), 1), - (('Function/django.middleware.csrf:' - 'CsrfViewMiddleware' + process_view_method), 1), - (('Function/django.contrib.messages.middleware:' - 'MessageMiddleware' + process_response_method), 1), - (('Function/django.middleware.csrf:' - 'CsrfViewMiddleware' + process_response_method), 1), - (('Function/django.contrib.sessions.middleware:' - 'SessionMiddleware' + process_response_method), 1), - (('Function/django.middleware.common:' - 'CommonMiddleware' + process_response_method), 1), + ("Function/django.core.handlers.wsgi:WSGIHandler.__call__", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + (f"Function/django.middleware.common:CommonMiddleware{process_request_method}", 1), + (f"Function/django.contrib.sessions.middleware:SessionMiddleware{process_request_method}", 1), + (f"Function/django.contrib.auth.middleware:AuthenticationMiddleware{process_request_method}", 1), + (f"Function/django.contrib.messages.middleware:MessageMiddleware{process_request_method}", 1), + (f"Function/{url_module_path}:{url_resolver_cls}.resolve", 1), + (f"Function/django.middleware.csrf:CsrfViewMiddleware{process_view_method}", 1), + (f"Function/django.contrib.messages.middleware:MessageMiddleware{process_response_method}", 1), + (f"Function/django.middleware.csrf:CsrfViewMiddleware{process_response_method}", 1), + (f"Function/django.contrib.sessions.middleware:SessionMiddleware{process_response_method}", 1), + (f"Function/django.middleware.common:CommonMiddleware{process_response_method}", 1), ] _test_application_index_scoped_metrics = list(_scoped_metrics) -_test_application_index_scoped_metrics.append(('Function/views:index', 1)) +_test_application_index_scoped_metrics.append(("Function/views:index", 1)) if DJANGO_VERSION >= (1, 5): - _test_application_index_scoped_metrics.extend([ - ('Function/django.http.response:HttpResponse.close', 1)]) + _test_application_index_scoped_metrics.extend([("Function/django.http.response:HttpResponse.close", 1)]) @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('views:index', - scoped_metrics=_test_application_index_scoped_metrics) +@validate_transaction_metrics("views:index", scoped_metrics=_test_application_index_scoped_metrics) @validate_code_level_metrics("views", "index") def test_application_index(target_application): - response = target_application.get('') - response.mustcontain('INDEX RESPONSE') + response = target_application.get("") + response.mustcontain("INDEX RESPONSE") _test_application_view_scoped_metrics = list(_scoped_metrics) -_test_application_view_scoped_metrics.append(('Function/urls:View.get', 1)) +_test_application_view_scoped_metrics.append(("Function/urls:View.get", 1)) if DJANGO_VERSION >= (1, 5): - _test_application_view_scoped_metrics.extend([ - ('Function/rest_framework.response:Response.close', 1)]) + _test_application_view_scoped_metrics.extend([("Function/rest_framework.response:Response.close", 1)]) @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('urls:View.get', - scoped_metrics=_test_application_view_scoped_metrics) +@validate_transaction_metrics("urls:View.get", scoped_metrics=_test_application_view_scoped_metrics) @validate_code_level_metrics("urls.View", "get") def test_application_view(target_application): - response = target_application.get('/view/') + response = target_application.get("/view/") assert response.status_int == 200 - response.mustcontain('restframework view response') + response.mustcontain("restframework view response") _test_application_view_error_scoped_metrics = list(_scoped_metrics) -_test_application_view_error_scoped_metrics.append( - ('Function/urls:ViewError.get', 1)) +_test_application_view_error_scoped_metrics.append(("Function/urls:ViewError.get", 1)) -@validate_transaction_errors(errors=['urls:Error']) -@validate_transaction_metrics('urls:ViewError.get', - scoped_metrics=_test_application_view_error_scoped_metrics) +@validate_transaction_errors(errors=["urls:Error"]) +@validate_transaction_metrics("urls:ViewError.get", scoped_metrics=_test_application_view_error_scoped_metrics) @validate_code_level_metrics("urls.ViewError", "get") def test_application_view_error(target_application): - target_application.get('/view_error/', status=500) + target_application.get("/view_error/", status=500) _test_application_view_handle_error_scoped_metrics = list(_scoped_metrics) -_test_application_view_handle_error_scoped_metrics.append( - ('Function/urls:ViewHandleError.get', 1)) +_test_application_view_handle_error_scoped_metrics.append(("Function/urls:ViewHandleError.get", 1)) -@pytest.mark.parametrize('status,should_record', [(418, True), (200, False)]) -@pytest.mark.parametrize('use_global_exc_handler', [True, False]) +@pytest.mark.parametrize("status,should_record", [(418, True), (200, False)]) +@pytest.mark.parametrize("use_global_exc_handler", [True, False]) @validate_code_level_metrics("urls.ViewHandleError", "get") -def test_application_view_handle_error(status, should_record, - use_global_exc_handler, target_application): - errors = ['urls:Error'] if should_record else [] +def test_application_view_handle_error(status, should_record, use_global_exc_handler, target_application): + errors = ["urls:Error"] if should_record else [] @validate_transaction_errors(errors=errors) - @validate_transaction_metrics('urls:ViewHandleError.get', - scoped_metrics=_test_application_view_handle_error_scoped_metrics) + @validate_transaction_metrics( + "urls:ViewHandleError.get", scoped_metrics=_test_application_view_handle_error_scoped_metrics + ) def _test(): - response = target_application.get( - '/view_handle_error/%s/%s/' % (status, use_global_exc_handler), - status=status) + response = target_application.get(f"/view_handle_error/{status}/{use_global_exc_handler}/", status=status) if use_global_exc_handler: - response.mustcontain('exception was handled global') + response.mustcontain("exception was handled global") else: - response.mustcontain('exception was handled not global') + response.mustcontain("exception was handled not global") _test() -_test_api_view_view_name_get = 'urls:wrapped_view.get' +_test_api_view_view_name_get = "urls:wrapped_view.get" _test_api_view_scoped_metrics_get = list(_scoped_metrics) -_test_api_view_scoped_metrics_get.append( - ('Function/%s' % _test_api_view_view_name_get, 1)) +_test_api_view_scoped_metrics_get.append((f"Function/{_test_api_view_view_name_get}", 1)) @validate_transaction_errors(errors=[]) -@validate_transaction_metrics(_test_api_view_view_name_get, - scoped_metrics=_test_api_view_scoped_metrics_get) -@validate_code_level_metrics("urls.WrappedAPIView" if six.PY3 else "urls", "wrapped_view") +@validate_transaction_metrics(_test_api_view_view_name_get, scoped_metrics=_test_api_view_scoped_metrics_get) +@validate_code_level_metrics("urls.WrappedAPIView", "wrapped_view") def test_api_view_get(target_application): - response = target_application.get('/api_view/') - response.mustcontain('wrapped_view response') + response = target_application.get("/api_view/") + response.mustcontain("wrapped_view response") -_test_api_view_view_name_post = 'urls:wrapped_view.http_method_not_allowed' +_test_api_view_view_name_post = "urls:wrapped_view.http_method_not_allowed" _test_api_view_scoped_metrics_post = list(_scoped_metrics) -_test_api_view_scoped_metrics_post.append( - ('Function/%s' % _test_api_view_view_name_post, 1)) +_test_api_view_scoped_metrics_post.append((f"Function/{_test_api_view_view_name_post}", 1)) -@validate_transaction_errors( - errors=['rest_framework.exceptions:MethodNotAllowed']) -@validate_transaction_metrics(_test_api_view_view_name_post, - scoped_metrics=_test_api_view_scoped_metrics_post) +@validate_transaction_errors(errors=["rest_framework.exceptions:MethodNotAllowed"]) +@validate_transaction_metrics(_test_api_view_view_name_post, scoped_metrics=_test_api_view_scoped_metrics_post) def test_api_view_method_not_allowed(target_application): - target_application.post('/api_view/', status=405) + target_application.post("/api_view/", status=405) def test_application_view_agent_disabled(target_application): settings = global_settings() - @override_generic_settings(settings, {'enabled': False}) - @function_not_called('newrelic.core.stats_engine', - 'StatsEngine.record_transaction') + @override_generic_settings(settings, {"enabled": False}) + @function_not_called("newrelic.core.stats_engine", "StatsEngine.record_transaction") def _test(): - response = target_application.get('/view/') + response = target_application.get("/view/") assert response.status_int == 200 - response.mustcontain('restframework view response') + response.mustcontain("restframework view response") _test() diff --git a/tests/component_flask_rest/_test_application.py b/tests/component_flask_rest/_test_application.py index 44003de4cc..cbd23fab2b 100644 --- a/tests/component_flask_rest/_test_application.py +++ b/tests/component_flask_rest/_test_application.py @@ -39,8 +39,7 @@ def get(self, exception, code): elif 'CustomException' in exception: e = CustomException() else: - raise AssertionError('Unexpected exception received: %s' % - exception) + raise AssertionError(f'Unexpected exception received: {exception}') e.code = code raise e diff --git a/tests/component_flask_rest/conftest.py b/tests/component_flask_rest/conftest.py index ff00973ab6..7fa57c44c4 100644 --- a/tests/component_flask_rest/conftest.py +++ b/tests/component_flask_rest/conftest.py @@ -12,20 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, - 'debug.log_autorum_middleware': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_autorum_middleware": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (component_flask_rest)', - default_settings=_default_settings) + app_name="Python Agent Test (component_flask_rest)", default_settings=_default_settings +) diff --git a/tests/component_flask_rest/test_application.py b/tests/component_flask_rest/test_application.py index d463a0205e..58b3db46ea 100644 --- a/tests/component_flask_rest/test_application.py +++ b/tests/component_flask_rest/test_application.py @@ -29,9 +29,6 @@ from newrelic.common.object_names import callable_name from newrelic.core.config import global_settings -from newrelic.packages import six - -TEST_APPLICATION_PREFIX = "_test_application.create_app." if six.PY3 else "_test_application" @pytest.fixture(params=["flask_restful", "flask_restx"]) @@ -62,7 +59,7 @@ def application(request): ] -@validate_code_level_metrics(TEST_APPLICATION_PREFIX + ".IndexResource", "get") +@validate_code_level_metrics("_test_application.create_app..IndexResource", "get") @validate_transaction_errors(errors=[]) @validate_transaction_metrics("_test_application:index", scoped_metrics=_test_application_index_scoped_metrics) def test_application_index(application): @@ -88,11 +85,11 @@ def test_application_index(application): ], ) def test_application_raises(exception, status_code, ignore_status_code, propagate_exceptions, application): - @validate_code_level_metrics(TEST_APPLICATION_PREFIX + ".ExceptionResource", "get") + @validate_code_level_metrics("_test_application.create_app..ExceptionResource", "get") @validate_transaction_metrics("_test_application:exception", scoped_metrics=_test_application_raises_scoped_metrics) def _test(): try: - application.get("/exception/%s/%i" % (exception, status_code), status=status_code, expect_errors=True) + application.get(f"/exception/{exception}/{status_code}", status=status_code, expect_errors=True) except Exception as e: assert propagate_exceptions @@ -118,4 +115,4 @@ def test_application_outside_transaction(application): def _test(): application.get("/exception/werkzeug.exceptions:HTTPException/404", status=404) - _test() + _test() \ No newline at end of file diff --git a/tests/application_celery/tasks.py b/tests/component_graphqlserver/__init__.py similarity index 80% rename from tests/application_celery/tasks.py rename to tests/component_graphqlserver/__init__.py index 476221b558..8030baccf7 100644 --- a/tests/application_celery/tasks.py +++ b/tests/component_graphqlserver/__init__.py @@ -11,16 +11,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - -from celery import Celery - - -app = Celery('tasks') - -@app.task -def add(x, y): - return x + y - -@app.task -def tsum(nums): - return sum(nums) diff --git a/tests/component_graphqlserver/_target_schema_async.py b/tests/component_graphqlserver/_target_schema_async.py new file mode 100644 index 0000000000..aff587bc8c --- /dev/null +++ b/tests/component_graphqlserver/_target_schema_async.py @@ -0,0 +1,155 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from graphql import ( + GraphQLArgument, + GraphQLField, + GraphQLInt, + GraphQLList, + GraphQLNonNull, + GraphQLObjectType, + GraphQLSchema, + GraphQLString, + GraphQLUnionType, +) + +from ._target_schema_sync import books, libraries, magazines + +storage = [] + + +async def resolve_library(parent, info, index): + return libraries[index] + + +async def resolve_storage_add(parent, info, string): + storage.append(string) + return string + + +async def resolve_storage(parent, info): + return [storage.pop()] + + +async def resolve_search(parent, info, contains): + search_books = [b for b in books if contains in b["name"]] + search_magazines = [m for m in magazines if contains in m["name"]] + return search_books + search_magazines + + +Author = GraphQLObjectType( + "Author", + { + "first_name": GraphQLField(GraphQLString), + "last_name": GraphQLField(GraphQLString), + }, +) + +Book = GraphQLObjectType( + "Book", + { + "id": GraphQLField(GraphQLInt), + "name": GraphQLField(GraphQLString), + "isbn": GraphQLField(GraphQLString), + "author": GraphQLField(Author), + "branch": GraphQLField(GraphQLString), + }, +) + +Magazine = GraphQLObjectType( + "Magazine", + { + "id": GraphQLField(GraphQLInt), + "name": GraphQLField(GraphQLString), + "issue": GraphQLField(GraphQLInt), + "branch": GraphQLField(GraphQLString), + }, +) + + +Library = GraphQLObjectType( + "Library", + { + "id": GraphQLField(GraphQLInt), + "branch": GraphQLField(GraphQLString), + "book": GraphQLField(GraphQLList(Book)), + "magazine": GraphQLField(GraphQLList(Magazine)), + }, +) + +Storage = GraphQLList(GraphQLString) + + +async def resolve_hello(root, info): + return "Hello!" + + +async def resolve_echo(root, info, echo): + return echo + + +async def resolve_error(root, info): + raise RuntimeError("Runtime Error!") + + +hello_field = GraphQLField(GraphQLString, resolver=resolve_hello) +library_field = GraphQLField( + Library, + resolver=resolve_library, + args={"index": GraphQLArgument(GraphQLNonNull(GraphQLInt))}, +) +search_field = GraphQLField( + GraphQLList(GraphQLUnionType("Item", (Book, Magazine), resolve_type=resolve_search)), + args={"contains": GraphQLArgument(GraphQLNonNull(GraphQLString))}, +) +echo_field = GraphQLField( + GraphQLString, + resolver=resolve_echo, + args={"echo": GraphQLArgument(GraphQLNonNull(GraphQLString))}, +) +storage_field = GraphQLField( + Storage, + resolver=resolve_storage, +) +storage_add_field = GraphQLField( + GraphQLString, + resolver=resolve_storage_add, + args={"string": GraphQLArgument(GraphQLNonNull(GraphQLString))}, +) +error_field = GraphQLField(GraphQLString, resolver=resolve_error) +error_non_null_field = GraphQLField(GraphQLNonNull(GraphQLString), resolver=resolve_error) +error_middleware_field = GraphQLField(GraphQLString, resolver=resolve_hello) + +query = GraphQLObjectType( + name="Query", + fields={ + "hello": hello_field, + "library": library_field, + "search": search_field, + "echo": echo_field, + "storage": storage_field, + "error": error_field, + "error_non_null": error_non_null_field, + "error_middleware": error_middleware_field, + }, +) + +mutation = GraphQLObjectType( + name="Mutation", + fields={ + "storage_add": storage_add_field, + }, +) + +target_schema = GraphQLSchema(query=query, mutation=mutation) diff --git a/tests/component_graphqlserver/_test_graphql.py b/tests/component_graphqlserver/_test_graphql.py index 50b5621f9a..7a29b3a8fe 100644 --- a/tests/component_graphqlserver/_test_graphql.py +++ b/tests/component_graphqlserver/_test_graphql.py @@ -12,15 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. +from flask import Flask +from sanic import Sanic import json - import webtest -from flask import Flask -from framework_graphql._target_application import _target_application as schema + +from testing_support.asgi_testing import AsgiTest +from framework_graphql._target_schema_sync import target_schema as schema from graphql_server.flask import GraphQLView as FlaskView from graphql_server.sanic import GraphQLView as SanicView -from sanic import Sanic -from testing_support.asgi_testing import AsgiTest + +# Sanic +target_application = dict() def set_middlware(middleware, view_middleware): @@ -95,5 +98,4 @@ def flask_execute(query, middleware=None): return response - target_application["Flask"] = flask_execute diff --git a/tests/component_graphqlserver/conftest.py b/tests/component_graphqlserver/conftest.py index f62af82100..26613da4f1 100644 --- a/tests/component_graphqlserver/conftest.py +++ b/tests/component_graphqlserver/conftest.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, diff --git a/tests/component_graphqlserver/test_graphql.py b/tests/component_graphqlserver/test_graphql.py index e5566047ec..5cb24e2848 100644 --- a/tests/component_graphqlserver/test_graphql.py +++ b/tests/component_graphqlserver/test_graphql.py @@ -12,16 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. + import importlib import pytest from testing_support.fixtures import dt_enabled -from testing_support.validators.validate_transaction_errors import validate_transaction_errors -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_count import ( validate_transaction_count, ) +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.common.object_names import callable_name @@ -36,7 +41,7 @@ def is_graphql_2(): @pytest.fixture(scope="session", params=("Sanic", "Flask")) def target_application(request): - import _test_graphql + from . import _test_graphql framework = request.param version = importlib.import_module(framework.lower()).__version__ @@ -73,9 +78,9 @@ def test_basic(target_application): from graphql_server import __version__ as graphql_server_version FRAMEWORK_METRICS = [ - ("Python/Framework/GraphQL/%s" % graphql_version, 1), - ("Python/Framework/GraphQLServer/%s" % graphql_server_version, 1), - ("Python/Framework/%s/%s" % (framework, version), 1), + (f"Python/Framework/GraphQL/{graphql_version}", 1), + (f"Python/Framework/GraphQLServer/{graphql_server_version}", 1), + (f"Python/Framework/{framework}/{version}", 1), ] @validate_transaction_metrics( @@ -96,9 +101,9 @@ def test_query_and_mutation(target_application): from graphql_server import __version__ as graphql_server_version FRAMEWORK_METRICS = [ - ("Python/Framework/GraphQL/%s" % graphql_version, 1), - ("Python/Framework/GraphQLServer/%s" % graphql_server_version, 1), - ("Python/Framework/%s/%s" % (framework, version), 1), + (f"Python/Framework/GraphQL/{graphql_version}", 1), + (f"Python/Framework/GraphQLServer/{graphql_server_version}", 1), + (f"Python/Framework/{framework}/{version}", 1), ] _test_query_scoped_metrics = [ ("GraphQL/resolve/GraphQLServer/storage", 1), @@ -186,7 +191,7 @@ def test_middleware(target_application): _test_middleware_metrics = [ ("GraphQL/operation/GraphQLServer/query//hello", 1), ("GraphQL/resolve/GraphQLServer/hello", 1), - ("Function/test_graphql:example_middleware", 1), + ("Function/component_graphqlserver.test_graphql:example_middleware", 1), ] # Base span count 6: Transaction, View, Operation, Middleware, and 1 Resolver and Resolver function @@ -214,13 +219,13 @@ def test_exception_in_middleware(target_application): # Metrics _test_exception_scoped_metrics = [ - ("GraphQL/operation/GraphQLServer/query/MyQuery/%s" % field, 1), - ("GraphQL/resolve/GraphQLServer/%s" % field, 1), + (f"GraphQL/operation/GraphQLServer/query/MyQuery/{field}", 1), + (f"GraphQL/resolve/GraphQLServer/{field}", 1), ] _test_exception_rollup_metrics = [ ("Errors/all", 1), ("Errors/allWeb", 1), - ("Errors/WebTransaction/GraphQL/test_graphql:error_middleware", 1), + ("Errors/WebTransaction/GraphQL/component_graphqlserver.test_graphql:error_middleware", 1), ] + _test_exception_scoped_metrics # Attributes @@ -237,7 +242,7 @@ def test_exception_in_middleware(target_application): } @validate_transaction_metrics( - "test_graphql:error_middleware", + "component_graphqlserver.test_graphql:error_middleware", "GraphQL", scoped_metrics=_test_exception_scoped_metrics, rollup_metrics=_test_exception_rollup_metrics + _graphql_base_rollup_metrics, @@ -255,19 +260,19 @@ def _test(): @dt_enabled def test_exception_in_resolver(target_application, field): framework, version, target_application = target_application - query = "query MyQuery { %s }" % field + query = f"query MyQuery {{ {field} }}" - txn_name = "framework_graphql._target_application:resolve_error" + txn_name = "framework_graphql._target_schema_sync:resolve_error" # Metrics _test_exception_scoped_metrics = [ - ("GraphQL/operation/GraphQLServer/query/MyQuery/%s" % field, 1), - ("GraphQL/resolve/GraphQLServer/%s" % field, 1), + (f"GraphQL/operation/GraphQLServer/query/MyQuery/{field}", 1), + (f"GraphQL/resolve/GraphQLServer/{field}", 1), ] _test_exception_rollup_metrics = [ ("Errors/all", 1), ("Errors/allWeb", 1), - ("Errors/WebTransaction/GraphQL/%s" % txn_name, 1), + (f"Errors/WebTransaction/GraphQL/{txn_name}", 1), ] + _test_exception_scoped_metrics # Attributes @@ -328,7 +333,7 @@ def test_exception_in_validation(target_application, is_graphql_2, query, exc_cl _test_exception_rollup_metrics = [ ("Errors/all", 1), ("Errors/allWeb", 1), - ("Errors/WebTransaction/GraphQL/%s" % txn_name, 1), + (f"Errors/WebTransaction/GraphQL/{txn_name}", 1), ] + _test_exception_scoped_metrics # Attributes @@ -488,9 +493,9 @@ def _test(): def test_deepest_unique_path(target_application, query, expected_path): framework, version, target_application = target_application if expected_path == "/error": - txn_name = "framework_graphql._target_application:resolve_error" + txn_name = "framework_graphql._target_schema_sync:resolve_error" else: - txn_name = "query/%s" % expected_path + txn_name = f"query/{expected_path}" @validate_transaction_metrics( txn_name, diff --git a/tests/component_tastypie/conftest.py b/tests/component_tastypie/conftest.py index e38e3b2f31..c1bdf9a88b 100644 --- a/tests/component_tastypie/conftest.py +++ b/tests/component_tastypie/conftest.py @@ -12,20 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, - 'debug.log_autorum_middleware': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_autorum_middleware": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (component_tastypie)', - default_settings=_default_settings) + app_name="Python Agent Test (component_tastypie)", default_settings=_default_settings +) diff --git a/tests/component_tastypie/test_application.py b/tests/component_tastypie/test_application.py index 5f81d08319..9622890131 100644 --- a/tests/component_tastypie/test_application.py +++ b/tests/component_tastypie/test_application.py @@ -13,49 +13,50 @@ # limitations under the License. import pytest -import six import webtest - from tastypie import VERSION +from testing_support.fixtures import override_ignore_status_codes +from testing_support.validators.validate_code_level_metrics import ( + validate_code_level_metrics, +) +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from wsgi import application from newrelic.api.background_task import background_task from newrelic.api.transaction import end_of_transaction -from testing_support.fixtures import override_ignore_status_codes -from testing_support.validators.validate_transaction_errors import validate_transaction_errors -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_code_level_metrics import validate_code_level_metrics - -from wsgi import application - test_application = webtest.TestApp(application) _test_application_scoped_metrics = [ - ('Function/django.core.handlers.wsgi:WSGIHandler.__call__', 1), - ('Function/django.http.response:HttpResponse.close', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), + ("Function/django.core.handlers.wsgi:WSGIHandler.__call__", 1), + ("Function/django.http.response:HttpResponse.close", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), ] _test_application_index_scoped_metrics = list(_test_application_scoped_metrics) -_test_application_index_scoped_metrics.append( - ('Function/views:index', 1)) +_test_application_index_scoped_metrics.append(("Function/views:index", 1)) @validate_code_level_metrics("views", "index") @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('views:index', - scoped_metrics=_test_application_index_scoped_metrics) +@validate_transaction_metrics("views:index", scoped_metrics=_test_application_index_scoped_metrics) def test_application_index(): - response = test_application.get('/index/') + response = test_application.get("/index/") assert response.status_code == 200 - response.mustcontain('INDEX RESPONSE') + response.mustcontain("INDEX RESPONSE") -class TastyPieFullDebugMode(object): +class TastyPieFullDebugMode(): def __init__(self, tastypie_full_debug): from django.conf import settings + self.settings = settings self.tastypie_full_debug = tastypie_full_debug @@ -68,156 +69,121 @@ def __exit__(self, *args, **kwargs): _test_api_base_scoped_metrics = [ - ('Function/django.core.handlers.wsgi:WSGIHandler.__call__', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), + ("Function/django.core.handlers.wsgi:WSGIHandler.__call__", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/tastypie.resources:Resource.wrap_view..wrapper", 1), + ("Function/django.urls.resolvers:URLResolver.resolve", 1), ] -if six.PY3: - _test_api_base_scoped_metrics.append( - ('Function/tastypie.resources:Resource.wrap_view..wrapper', 1)) -else: - _test_api_base_scoped_metrics.append( - ('Function/tastypie.resources:wrapper', 1)) - -# django < 1.12 used the RegexURLResolver class and this was updated to URLResolver in later versions -if VERSION <= (0, 14, 3) and not six.PY3: - _test_api_base_scoped_metrics.append(('Function/django.urls.resolvers:RegexURLResolver.resolve', 1)) -else: - _test_api_base_scoped_metrics.append(('Function/django.urls.resolvers:URLResolver.resolve', 1)) - +_test_application_not_found_scoped_metrics = list(_test_api_base_scoped_metrics) -_test_application_not_found_scoped_metrics = list( - _test_api_base_scoped_metrics) - -@pytest.mark.parametrize('api_version', ['v1', 'v2']) -@pytest.mark.parametrize('tastypie_full_debug', [True, False]) +@pytest.mark.parametrize("api_version", ["v1", "v2"]) +@pytest.mark.parametrize("tastypie_full_debug", [True, False]) def test_not_found(api_version, tastypie_full_debug): - - _test_application_not_found_scoped_metrics = list( - _test_api_base_scoped_metrics) + _test_application_not_found_scoped_metrics = list(_test_api_base_scoped_metrics) if tastypie_full_debug: - _test_application_not_found_scoped_metrics.append( - ('Function/django.http.response:HttpResponse.close', 1)) + _test_application_not_found_scoped_metrics.append(("Function/django.http.response:HttpResponse.close", 1)) else: _test_application_not_found_scoped_metrics.append( - (('Function/django.http.response:' - 'HttpResponseNotFound.close'), 1)) + (("Function/django.http.response:" "HttpResponseNotFound.close"), 1) + ) _errors = [] if tastypie_full_debug: - _errors.append('tastypie.exceptions:NotFound') + _errors.append("tastypie.exceptions:NotFound") @validate_transaction_errors(errors=_errors) - @validate_transaction_metrics('api:SimpleResource.dispatch_detail', - scoped_metrics=_test_application_not_found_scoped_metrics) + @validate_transaction_metrics( + "api:SimpleResource.dispatch_detail", scoped_metrics=_test_application_not_found_scoped_metrics + ) def _test_not_found(): with TastyPieFullDebugMode(tastypie_full_debug) as debug_status: - test_application.get('/api/%s/simple/NotFound/' % api_version, - status=debug_status) + test_application.get(f"/api/{api_version}/simple/NotFound/", status=debug_status) _test_not_found() -_test_application_object_does_not_exist_scoped_metrics = list( - _test_api_base_scoped_metrics) +_test_application_object_does_not_exist_scoped_metrics = list(_test_api_base_scoped_metrics) -_test_application_object_does_not_exist_scoped_metrics.append( - ('Function/tastypie.http:HttpNotFound.close', 1)) +_test_application_object_does_not_exist_scoped_metrics.append(("Function/tastypie.http:HttpNotFound.close", 1)) -@pytest.mark.parametrize('api_version', ['v1', 'v2']) -@pytest.mark.parametrize('tastypie_full_debug', [True, False]) +@pytest.mark.parametrize("api_version", ["v1", "v2"]) +@pytest.mark.parametrize("tastypie_full_debug", [True, False]) @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('api:SimpleResource.dispatch_detail', - scoped_metrics=_test_application_object_does_not_exist_scoped_metrics) +@validate_transaction_metrics( + "api:SimpleResource.dispatch_detail", scoped_metrics=_test_application_object_does_not_exist_scoped_metrics +) def test_object_does_not_exist(api_version, tastypie_full_debug): with TastyPieFullDebugMode(tastypie_full_debug): - test_application.get( - '/api/%s/simple/ObjectDoesNotExist/' % api_version, status=404) + test_application.get(f"/api/{api_version}/simple/ObjectDoesNotExist/", status=404) -_test_application_raises_zerodivision = list(_test_api_base_scoped_metrics) -_test_application_raises_zerodivision_exceptions = [] +_test_application_raises_zerodivision_exceptions = ["builtins:ZeroDivisionError"] -if six.PY3: - _test_application_raises_zerodivision_exceptions.append( - 'builtins:ZeroDivisionError') -else: - _test_application_raises_zerodivision_exceptions.append( - 'exceptions:ZeroDivisionError') - -@pytest.mark.parametrize('api_version', ['v1', 'v2']) -@pytest.mark.parametrize('tastypie_full_debug', [True, False]) -@validate_transaction_errors( - errors=_test_application_raises_zerodivision_exceptions) +@pytest.mark.parametrize("api_version", ["v1", "v2"]) +@pytest.mark.parametrize("tastypie_full_debug", [True, False]) +@validate_transaction_errors(errors=_test_application_raises_zerodivision_exceptions) def test_raises_zerodivision(api_version, tastypie_full_debug): - _test_application_raises_zerodivision = list(_test_api_base_scoped_metrics) - if tastypie_full_debug: _test_application_raises_zerodivision.append( - (('Function/django.core.handlers.exception:' - 'handle_uncaught_exception'), 1)) + (("Function/django.core.handlers.exception:handle_uncaught_exception"), 1) + ) else: - _test_application_raises_zerodivision.append( - ('Function/tastypie.http:HttpApplicationError.close', 1)) + _test_application_raises_zerodivision.append(("Function/tastypie.http:HttpApplicationError.close", 1)) - @validate_transaction_metrics('api:SimpleResource.dispatch_detail', - scoped_metrics=_test_application_raises_zerodivision) + @validate_transaction_metrics( + "api:SimpleResource.dispatch_detail", scoped_metrics=_test_application_raises_zerodivision + ) def _test_raises_zerodivision(): with TastyPieFullDebugMode(tastypie_full_debug): - test_application.get( - '/api/%s/simple/ZeroDivisionError/' % api_version, - status=500) + test_application.get(f"/api/{api_version}/simple/ZeroDivisionError/", status=500) _test_raises_zerodivision() -@pytest.mark.parametrize('api_version', ['v1', 'v2']) -@pytest.mark.parametrize('tastypie_full_debug', [True, False]) +@pytest.mark.parametrize("api_version", ["v1", "v2"]) +@pytest.mark.parametrize("tastypie_full_debug", [True, False]) @override_ignore_status_codes(set()) # don't ignore any status codes -@validate_transaction_errors(errors=['tastypie.exceptions:NotFound']) -@validate_transaction_metrics('api:SimpleResource.dispatch_detail', - scoped_metrics=_test_application_not_found_scoped_metrics) +@validate_transaction_errors(errors=["tastypie.exceptions:NotFound"]) +@validate_transaction_metrics( + "api:SimpleResource.dispatch_detail", scoped_metrics=_test_application_not_found_scoped_metrics +) def test_record_404_errors(api_version, tastypie_full_debug): - - _test_application_not_found_scoped_metrics = list( - _test_api_base_scoped_metrics) + _test_application_not_found_scoped_metrics = list(_test_api_base_scoped_metrics) if tastypie_full_debug: - _test_application_not_found_scoped_metrics.append( - ('Function/django.http.response:HttpResponse.close', 1)) + _test_application_not_found_scoped_metrics.append(("Function/django.http.response:HttpResponse.close", 1)) else: _test_application_not_found_scoped_metrics.append( - (('Function/django.http.response:' - 'HttpResponseNotFound.close'), 1)) + (("Function/django.http.response:" "HttpResponseNotFound.close"), 1) + ) - @validate_transaction_metrics('api:SimpleResource.dispatch_detail', - scoped_metrics=_test_application_not_found_scoped_metrics) + @validate_transaction_metrics( + "api:SimpleResource.dispatch_detail", scoped_metrics=_test_application_not_found_scoped_metrics + ) def _test_not_found(): with TastyPieFullDebugMode(tastypie_full_debug) as debug_status: - test_application.get('/api/%s/simple/NotFound/' % api_version, - status=debug_status) + test_application.get(f"/api/{api_version}/simple/NotFound/", status=debug_status) _test_not_found() -@pytest.mark.parametrize('api_version', ['v1', 'v2']) -@pytest.mark.parametrize('tastypie_full_debug', [True, False]) +@pytest.mark.parametrize("api_version", ["v1", "v2"]) +@pytest.mark.parametrize("tastypie_full_debug", [True, False]) @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('test_application:test_ended_txn_name', - background_task=True) +@validate_transaction_metrics("test_application:test_ended_txn_name", background_task=True) @background_task() def test_ended_txn_name(api_version, tastypie_full_debug): # if the transaction has ended, do not change the transaction name end_of_transaction() with TastyPieFullDebugMode(tastypie_full_debug) as debug_status: - test_application.get('/api/%s/simple/NotFound/' % api_version, - status=debug_status) + test_application.get(f"/api/{api_version}/simple/NotFound/", status=debug_status) diff --git a/tests/coroutines_asyncio/conftest.py b/tests/coroutines_asyncio/conftest.py index 5d3d843d01..94726182cd 100644 --- a/tests/coroutines_asyncio/conftest.py +++ b/tests/coroutines_asyncio/conftest.py @@ -12,13 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixture.event_loop import event_loop -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixture.event_loop import ( # noqa: F401; pylint: disable=W0611 + event_loop, +) +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, diff --git a/tests/cross_agent/conftest.py b/tests/cross_agent/conftest.py index d21ebf236e..cba1ea98b0 100644 --- a/tests/cross_agent/conftest.py +++ b/tests/cross_agent/conftest.py @@ -12,20 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, - 'debug.log_autorum_middleware': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_autorum_middleware": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (cross_agent_tests)', - default_settings=_default_settings) + app_name="Python Agent Test (cross_agent_tests)", default_settings=_default_settings +) diff --git a/tests/cross_agent/fixtures/docker_container_id_v2/README.md b/tests/cross_agent/fixtures/docker_container_id_v2/README.md new file mode 100644 index 0000000000..ea6cc25035 --- /dev/null +++ b/tests/cross_agent/fixtures/docker_container_id_v2/README.md @@ -0,0 +1,6 @@ +These tests cover parsing of Docker container IDs on Linux hosts out of +`/proc/self/mountinfo` (or `/proc//mountinfo` more generally). + +The `cases.json` file lists each filename in this directory containing +example `/proc/self/mountinfo` content, and the expected Docker container ID that +should be parsed from that file. diff --git a/tests/cross_agent/fixtures/docker_container_id_v2/cases.json b/tests/cross_agent/fixtures/docker_container_id_v2/cases.json new file mode 100644 index 0000000000..83d6360a31 --- /dev/null +++ b/tests/cross_agent/fixtures/docker_container_id_v2/cases.json @@ -0,0 +1,36 @@ +[ + { + "filename": "docker-20.10.16.txt", + "containerId": "84cf3472a20d1bfb4b50e48b6ff50d96dfcd812652d76dd907951e6f98997bce", + "expectedMetrics": null + }, + { + "filename": "docker-24.0.2.txt", + "containerId": "b0a24eed1b031271d8ba0784b8f354b3da892dfd08bbcf14dd7e8a1cf9292f65", + "expectedMetrics": null + }, + { + "filename": "empty.txt", + "containerId": null, + "expectedMetrics": null + }, + { + "filename": "invalid-characters.txt", + "containerId": null, + "expectedMetrics": null + }, + { + "filename": "docker-too-long.txt", + "containerId": null, + "expectedMetrics": null + }, + { + "filename": "invalid-length.txt", + "containerId": null, + "expectedMetrics": { + "Supportability/utilization/docker/error": { + "callCount": 1 + } + } + } +] diff --git a/tests/cross_agent/fixtures/docker_container_id_v2/docker-20.10.16.txt b/tests/cross_agent/fixtures/docker_container_id_v2/docker-20.10.16.txt new file mode 100644 index 0000000000..ce2b1bedf6 --- /dev/null +++ b/tests/cross_agent/fixtures/docker_container_id_v2/docker-20.10.16.txt @@ -0,0 +1,24 @@ +519 413 0:152 / / rw,relatime master:180 - overlay overlay rw,lowerdir=/var/lib/docker/overlay2/l/YCID3333O5VYPYDNTQRZX4GI67:/var/lib/docker/overlay2/l/G7H4TULAFM2UBFRL7QFQPUNXY5:/var/lib/docker/overlay2/l/RLC4GCL75VGXXXYJJO57STHIYN:/var/lib/docker/overlay2/l/YOZKNWFAP6YX74XEKPHX4KG4UN:/var/lib/docker/overlay2/l/46EQ6YX5PQQZ4Z3WCSMQ6Z4YWI:/var/lib/docker/overlay2/l/KGKX3Z5ZMOCDWOFKBS2FSHMQMQ:/var/lib/docker/overlay2/l/CKFYAF4TXZD4RCE6RG6UNL5WVI,upperdir=/var/lib/docker/overlay2/358c429f7b04ee5a228b94efaebe3413a98fcc676b726f078fe875727e3bddd2/diff,workdir=/var/lib/docker/overlay2/358c429f7b04ee5a228b94efaebe3413a98fcc676b726f078fe875727e3bddd2/work +520 519 0:155 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw +521 519 0:156 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +522 521 0:157 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +523 519 0:158 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro +524 523 0:30 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +525 521 0:154 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw +526 521 0:159 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +527 519 254:1 /docker/volumes/3237dea4f8022f1addd7b6f072a9c847eb3e5b8df0d599f462ba7040884d4618/_data /data rw,relatime master:28 - ext4 /dev/vda1 rw +528 519 254:1 /docker/containers/84cf3472a20d1bfb4b50e48b6ff50d96dfcd812652d76dd907951e6f98997bce/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw +529 519 254:1 /docker/containers/84cf3472a20d1bfb4b50e48b6ff50d96dfcd812652d76dd907951e6f98997bce/hostname /etc/hostname rw,relatime - ext4 /dev/vda1 rw +530 519 254:1 /docker/containers/84cf3472a20d1bfb4b50e48b6ff50d96dfcd812652d76dd907951e6f98997bce/hosts /etc/hosts rw,relatime - ext4 /dev/vda1 rw +414 521 0:157 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +415 520 0:155 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw +416 520 0:155 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw +417 520 0:155 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw +418 520 0:155 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw +419 520 0:155 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw +420 520 0:160 / /proc/acpi ro,relatime - tmpfs tmpfs ro +421 520 0:156 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +422 520 0:156 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +423 520 0:156 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +424 520 0:156 /null /proc/sched_debug rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +425 523 0:161 / /sys/firmware ro,relatime - tmpfs tmpfs ro diff --git a/tests/cross_agent/fixtures/docker_container_id_v2/docker-24.0.2.txt b/tests/cross_agent/fixtures/docker_container_id_v2/docker-24.0.2.txt new file mode 100644 index 0000000000..1725e7726a --- /dev/null +++ b/tests/cross_agent/fixtures/docker_container_id_v2/docker-24.0.2.txt @@ -0,0 +1,21 @@ +1014 1013 0:269 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw +1019 1013 0:270 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +1020 1019 0:271 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +1021 1013 0:272 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro +1022 1021 0:30 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +1023 1019 0:268 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw +1024 1019 0:273 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +1025 1013 254:1 /docker/containers/b0a24eed1b031271d8ba0784b8f354b3da892dfd08bbcf14dd7e8a1cf9292f65/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw,discard +1026 1013 254:1 /docker/containers/b0a24eed1b031271d8ba0784b8f354b3da892dfd08bbcf14dd7e8a1cf9292f65/hostname /etc/hostname rw,relatime - ext4 /dev/vda1 rw,discard +1027 1013 254:1 /docker/containers/b0a24eed1b031271d8ba0784b8f354b3da892dfd08bbcf14dd7e8a1cf9292f65/hosts /etc/hosts rw,relatime - ext4 /dev/vda1 rw,discard +717 1019 0:271 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +718 1014 0:269 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw +719 1014 0:269 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw +720 1014 0:269 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw +721 1014 0:269 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw +723 1014 0:269 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw +726 1014 0:274 / /proc/acpi ro,relatime - tmpfs tmpfs ro +727 1014 0:270 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +728 1014 0:270 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +729 1014 0:270 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +730 1021 0:275 / /sys/firmware ro,relatime - tmpfs tmpfs ro diff --git a/tests/cross_agent/fixtures/docker_container_id_v2/docker-too-long.txt b/tests/cross_agent/fixtures/docker_container_id_v2/docker-too-long.txt new file mode 100644 index 0000000000..608eaf7a49 --- /dev/null +++ b/tests/cross_agent/fixtures/docker_container_id_v2/docker-too-long.txt @@ -0,0 +1,21 @@ +1014 1013 0:269 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw +1019 1013 0:270 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +1020 1019 0:271 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +1021 1013 0:272 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro +1022 1021 0:30 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +1023 1019 0:268 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw +1024 1019 0:273 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +1025 1013 254:1 /docker/containers/3ccfa00432798ff38f85839de1e396f771b4acbe9f4ddea0a761c39b9790a7821/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw,discard +1026 1013 254:1 /docker/containers/3ccfa00432798ff38f85839de1e396f771b4acbe9f4ddea0a761c39b9790a7821/hostname /etc/hostname rw,relatime - ext4 /dev/vda1 rw,discard +1027 1013 254:1 /docker/containers/3ccfa00432798ff38f85839de1e396f771b4acbe9f4ddea0a761c39b9790a7821/hosts /etc/hosts rw,relatime - ext4 /dev/vda1 rw,discard +717 1019 0:271 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +718 1014 0:269 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw +719 1014 0:269 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw +720 1014 0:269 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw +721 1014 0:269 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw +723 1014 0:269 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw +726 1014 0:274 / /proc/acpi ro,relatime - tmpfs tmpfs ro +727 1014 0:270 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +728 1014 0:270 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +729 1014 0:270 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +730 1021 0:275 / /sys/firmware ro,relatime - tmpfs tmpfs ro diff --git a/tests/cross_agent/fixtures/docker_container_id_v2/empty.txt b/tests/cross_agent/fixtures/docker_container_id_v2/empty.txt new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/cross_agent/fixtures/docker_container_id_v2/invalid-characters.txt b/tests/cross_agent/fixtures/docker_container_id_v2/invalid-characters.txt new file mode 100644 index 0000000000..b561475ac6 --- /dev/null +++ b/tests/cross_agent/fixtures/docker_container_id_v2/invalid-characters.txt @@ -0,0 +1,21 @@ +1014 1013 0:269 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw +1019 1013 0:270 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +1020 1019 0:271 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +1021 1013 0:272 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro +1022 1021 0:30 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +1023 1019 0:268 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw +1024 1019 0:273 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +1025 1013 254:1 /docker/containers/WRONGINCORRECTINVALIDCHARSERRONEOUSBADPHONYBROKEN2TERRIBLENOPE55/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw,discard +1026 1013 254:1 /docker/containers/WRONGINCORRECTINVALIDCHARSERRONEOUSBADPHONYBROKEN2TERRIBLENOPE55/hostname /etc/hostname rw,relatime - ext4 /dev/vda1 rw,discard +1027 1013 254:1 /docker/containers/WRONGINCORRECTINVALIDCHARSERRONEOUSBADPHONYBROKEN2TERRIBLENOPE55/hosts /etc/hosts rw,relatime - ext4 /dev/vda1 rw,discard +717 1019 0:271 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +718 1014 0:269 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw +719 1014 0:269 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw +720 1014 0:269 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw +721 1014 0:269 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw +723 1014 0:269 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw +726 1014 0:274 / /proc/acpi ro,relatime - tmpfs tmpfs ro +727 1014 0:270 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +728 1014 0:270 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +729 1014 0:270 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +730 1021 0:275 / /sys/firmware ro,relatime - tmpfs tmpfs ro diff --git a/tests/cross_agent/fixtures/docker_container_id_v2/invalid-length.txt b/tests/cross_agent/fixtures/docker_container_id_v2/invalid-length.txt new file mode 100644 index 0000000000..a8987df707 --- /dev/null +++ b/tests/cross_agent/fixtures/docker_container_id_v2/invalid-length.txt @@ -0,0 +1,21 @@ +1014 1013 0:269 / /proc rw,nosuid,nodev,noexec,relatime - proc proc rw +1019 1013 0:270 / /dev rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +1020 1019 0:271 / /dev/pts rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +1021 1013 0:272 / /sys ro,nosuid,nodev,noexec,relatime - sysfs sysfs ro +1022 1021 0:30 / /sys/fs/cgroup ro,nosuid,nodev,noexec,relatime - cgroup2 cgroup rw +1023 1019 0:268 / /dev/mqueue rw,nosuid,nodev,noexec,relatime - mqueue mqueue rw +1024 1019 0:273 / /dev/shm rw,nosuid,nodev,noexec,relatime - tmpfs shm rw,size=65536k +1025 1013 254:1 /docker/containers/47cbd16b77c5/resolv.conf /etc/resolv.conf rw,relatime - ext4 /dev/vda1 rw,discard +1026 1013 254:1 /docker/containers/47cbd16b77c5/hostname /etc/hostname rw,relatime - ext4 /dev/vda1 rw,discard +1027 1013 254:1 /docker/containers/47cbd16b77c5/hosts /etc/hosts rw,relatime - ext4 /dev/vda1 rw,discard +717 1019 0:271 /0 /dev/console rw,nosuid,noexec,relatime - devpts devpts rw,gid=5,mode=620,ptmxmode=666 +718 1014 0:269 /bus /proc/bus ro,nosuid,nodev,noexec,relatime - proc proc rw +719 1014 0:269 /fs /proc/fs ro,nosuid,nodev,noexec,relatime - proc proc rw +720 1014 0:269 /irq /proc/irq ro,nosuid,nodev,noexec,relatime - proc proc rw +721 1014 0:269 /sys /proc/sys ro,nosuid,nodev,noexec,relatime - proc proc rw +723 1014 0:269 /sysrq-trigger /proc/sysrq-trigger ro,nosuid,nodev,noexec,relatime - proc proc rw +726 1014 0:274 / /proc/acpi ro,relatime - tmpfs tmpfs ro +727 1014 0:270 /null /proc/kcore rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +728 1014 0:270 /null /proc/keys rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +729 1014 0:270 /null /proc/timer_list rw,nosuid - tmpfs tmpfs rw,size=65536k,mode=755 +730 1021 0:275 / /sys/firmware ro,relatime - tmpfs tmpfs ro diff --git a/tests/cross_agent/fixtures/ecs_container_id/ecs_mock_server.py b/tests/cross_agent/fixtures/ecs_container_id/ecs_mock_server.py new file mode 100644 index 0000000000..5c7853f4b9 --- /dev/null +++ b/tests/cross_agent/fixtures/ecs_container_id/ecs_mock_server.py @@ -0,0 +1,114 @@ +import pytest +import json +from testing_support.mock_external_http_server import MockExternalHTTPServer + +STANDARD_RESPONSE = { + "DockerId": "1e1698469422439ea356071e581e8545-2769485393", + "Name": "fargateapp", + "DockerName": "fargateapp", + "Image": "123456789012.dkr.ecr.us-west-2.amazonaws.com/fargatetest:latest", + "ImageID": "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcd", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:123456789012:cluster/testcluster", + "com.amazonaws.ecs.container-name": "fargateapp", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:123456789012:task/testcluster/1e1698469422439ea356071e581e8545", + "com.amazonaws.ecs.task-definition-family": "fargatetestapp", + "com.amazonaws.ecs.task-definition-version": "7", + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": {"CPU": 2}, + "CreatedAt": "2024-04-25T17:38:31.073208914Z", + "StartedAt": "2024-04-25T17:38:31.073208914Z", + "Type": "NORMAL", + "LogDriver": "awslogs", + "LogOptions": { + "awslogs-create-group": "true", + "awslogs-group": "/ecs/fargatetestapp", + "awslogs-region": "us-west-2", + "awslogs-stream": "ecs/fargateapp/1e1698469422439ea356071e581e8545", + }, + "ContainerARN": "arn:aws:ecs:us-west-2:123456789012:container/testcluster/1e1698469422439ea356071e581e8545/050256a5-a7f3-461c-a16f-aca4eae37b01", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": ["10.10.10.10"], + "AttachmentIndex": 0, + "MACAddress": "06:d7:3f:49:1d:a7", + "IPv4SubnetCIDRBlock": "10.10.10.0/20", + "DomainNameServers": ["10.10.10.2"], + "DomainNameSearchList": ["us-west-2.compute.internal"], + "PrivateDNSName": "ip-10-10-10-10.us-west-2.compute.internal", + "SubnetGatewayIpv4Address": "10.10.10.1/20", + } + ], + "Snapshotter": "overlayfs", +} + +NO_ID_RESPONSE = { + "Name": "fargateapp", + "DockerName": "fargateapp", + "Image": "123456789012.dkr.ecr.us-west-2.amazonaws.com/fargatetest:latest", + "ImageID": "sha256:1234567890abcdef1234567890abcdef1234567890abcdef1234567890abcd", + "Labels": { + "com.amazonaws.ecs.cluster": "arn:aws:ecs:us-west-2:123456789012:cluster/testcluster", + "com.amazonaws.ecs.container-name": "fargateapp", + "com.amazonaws.ecs.task-arn": "arn:aws:ecs:us-west-2:123456789012:task/testcluster/1e1698469422439ea356071e581e8545", + "com.amazonaws.ecs.task-definition-family": "fargatetestapp", + "com.amazonaws.ecs.task-definition-version": "7", + }, + "DesiredStatus": "RUNNING", + "KnownStatus": "RUNNING", + "Limits": {"CPU": 2}, + "CreatedAt": "2024-04-25T17:38:31.073208914Z", + "StartedAt": "2024-04-25T17:38:31.073208914Z", + "Type": "NORMAL", + "LogDriver": "awslogs", + "LogOptions": { + "awslogs-create-group": "true", + "awslogs-group": "/ecs/fargatetestapp", + "awslogs-region": "us-west-2", + "awslogs-stream": "ecs/fargateapp/1e1698469422439ea356071e581e8545", + }, + "ContainerARN": "arn:aws:ecs:us-west-2:123456789012:container/testcluster/1e1698469422439ea356071e581e8545/050256a5-a7f3-461c-a16f-aca4eae37b01", + "Networks": [ + { + "NetworkMode": "awsvpc", + "IPv4Addresses": ["10.10.10.10"], + "AttachmentIndex": 0, + "MACAddress": "06:d7:3f:49:1d:a7", + "IPv4SubnetCIDRBlock": "10.10.10.0/20", + "DomainNameServers": ["10.10.10.2"], + "DomainNameSearchList": ["us-west-2.compute.internal"], + "PrivateDNSName": "ip-10-10-10-10.us-west-2.compute.internal", + "SubnetGatewayIpv4Address": "10.10.10.1/20", + } + ], + "Snapshotter": "overlayfs", +} + + +def simple_get(self): + response = json.dumps(STANDARD_RESPONSE).encode("utf-8") + self.send_response(200) + self.end_headers() + self.wfile.write(response) + + +def bad_response_get(self): + response = json.dumps(NO_ID_RESPONSE).encode("utf-8") + self.send_response(200) + self.end_headers() + self.wfile.write(response) + + +@pytest.fixture(scope="function") +def mock_server(): + with MockExternalHTTPServer(handler=simple_get) as mock_server: + yield mock_server + + +@pytest.fixture(scope="function") +def bad_response_mock_server(): + with MockExternalHTTPServer(handler=bad_response_get) as bad_response_mock_server: + yield bad_response_mock_server diff --git a/tests/cross_agent/fixtures/rum_client_config.json b/tests/cross_agent/fixtures/rum_client_config.json deleted file mode 100644 index 8f6e7cbbbe..0000000000 --- a/tests/cross_agent/fixtures/rum_client_config.json +++ /dev/null @@ -1,91 +0,0 @@ -[ - { - "testname":"all fields present", - - "apptime_milliseconds":5, - "queuetime_milliseconds":3, - "browser_monitoring.attributes.enabled":true, - "transaction_name":"WebTransaction/brink/of/glory", - "license_key":"0000111122223333444455556666777788889999", - "connect_reply": - { - "beacon":"my_beacon", - "browser_key":"my_browser_key", - "application_id":"my_application_id", - "error_beacon":"my_error_beacon", - "js_agent_file":"my_js_agent_file" - }, - "user_attributes":{"alpha":"beta"}, - "expected": - { - "beacon":"my_beacon", - "licenseKey":"my_browser_key", - "applicationID":"my_application_id", - "transactionName":"Z1VSZENQX0JTUUZbXF4fUkJYX1oeXVQdVV9fQkk=", - "queueTime":3, - "applicationTime":5, - "atts":"SxJFEgtKE1BeQlpTEQoSUlVFUBNMTw==", - "errorBeacon":"my_error_beacon", - "agent":"my_js_agent_file" - } - }, - { - "testname":"browser_monitoring.attributes.enabled disabled", - - "apptime_milliseconds":5, - "queuetime_milliseconds":3, - "browser_monitoring.attributes.enabled":false, - "transaction_name":"WebTransaction/brink/of/glory", - "license_key":"0000111122223333444455556666777788889999", - "connect_reply": - { - "beacon":"my_beacon", - "browser_key":"my_browser_key", - "application_id":"my_application_id", - "error_beacon":"my_error_beacon", - "js_agent_file":"my_js_agent_file" - }, - "user_attributes":{"alpha":"beta"}, - "expected": - { - "beacon":"my_beacon", - "licenseKey":"my_browser_key", - "applicationID":"my_application_id", - "transactionName":"Z1VSZENQX0JTUUZbXF4fUkJYX1oeXVQdVV9fQkk=", - "queueTime":3, - "applicationTime":5, - "atts":"", - "errorBeacon":"my_error_beacon", - "agent":"my_js_agent_file" - } - }, - { - "testname":"empty js_agent_file", - "apptime_milliseconds":5, - "queuetime_milliseconds":3, - "browser_monitoring.attributes.enabled":true, - "transaction_name":"WebTransaction/brink/of/glory", - "license_key":"0000111122223333444455556666777788889999", - "connect_reply": - { - "beacon":"my_beacon", - "browser_key":"my_browser_key", - "application_id":"my_application_id", - "error_beacon":"my_error_beacon", - "js_agent_file":"" - }, - "user_attributes":{"alpha":"beta"}, - "expected": - { - "beacon":"my_beacon", - "licenseKey":"my_browser_key", - "applicationID":"my_application_id", - "transactionName":"Z1VSZENQX0JTUUZbXF4fUkJYX1oeXVQdVV9fQkk=", - "queueTime":3, - "applicationTime":5, - "atts":"SxJFEgtKE1BeQlpTEQoSUlVFUBNMTw==", - "errorBeacon":"my_error_beacon", - "agent":"" - } - } -] diff --git a/tests/cross_agent/fixtures/rum_footer_insertion_location/close-body-in-comment.html b/tests/cross_agent/fixtures/rum_footer_insertion_location/close-body-in-comment.html deleted file mode 100644 index e32df24204..0000000000 --- a/tests/cross_agent/fixtures/rum_footer_insertion_location/close-body-in-comment.html +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - Comment contains a close body tag - - -

The quick brown fox jumps over the lazy dog.

- - EXPECTED_RUM_FOOTER_LOCATION - diff --git a/tests/cross_agent/fixtures/rum_footer_insertion_location/dynamic-iframe.html b/tests/cross_agent/fixtures/rum_footer_insertion_location/dynamic-iframe.html deleted file mode 100644 index 5e1acc86b5..0000000000 --- a/tests/cross_agent/fixtures/rum_footer_insertion_location/dynamic-iframe.html +++ /dev/null @@ -1,35 +0,0 @@ - - - - - - Dynamic iframe Generation - - -

The quick brown fox jumps over the lazy dog.

- - - EXPECTED_RUM_FOOTER_LOCATION - diff --git a/tests/cross_agent/test_agent_attributes.py b/tests/cross_agent/test_agent_attributes.py index c254be7728..527b31a753 100644 --- a/tests/cross_agent/test_agent_attributes.py +++ b/tests/cross_agent/test_agent_attributes.py @@ -40,7 +40,8 @@ def _default_settings(): 'browser_monitoring.attributes.exclude': [], } -FIXTURE = os.path.join(os.curdir, 'fixtures', 'attribute_configuration.json') +CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) +FIXTURE = os.path.join(CURRENT_DIR, 'fixtures', 'attribute_configuration.json') def _load_tests(): with open(FIXTURE, 'r') as fh: diff --git a/tests/cross_agent/test_boot_id_utilization_data.py b/tests/cross_agent/test_boot_id_utilization_data.py index ea5b26a9e7..42f58c12cf 100644 --- a/tests/cross_agent/test_boot_id_utilization_data.py +++ b/tests/cross_agent/test_boot_id_utilization_data.py @@ -48,7 +48,7 @@ def _parametrize_test(test): _boot_id_tests = [_parametrize_test(t) for t in _load_tests()] -class MockedBootIdEndpoint(object): +class MockedBootIdEndpoint(): def __init__(self, boot_id): self.boot_id = boot_id diff --git a/tests/cross_agent/test_cat_map.py b/tests/cross_agent/test_cat_map.py index 67c5ab8151..b20485c83b 100644 --- a/tests/cross_agent/test_cat_map.py +++ b/tests/cross_agent/test_cat_map.py @@ -18,42 +18,53 @@ can be found in test/framework_tornado_r3/test_cat_map.py """ -import webtest -import pytest import json import os -try: - from urllib2 import urlopen # Py2.X -except ImportError: - from urllib.request import urlopen # Py3.X +import pytest +import webtest + +from urllib.request import urlopen -from newrelic.packages import six +from testing_support.fixtures import ( + make_cross_agent_headers, + override_application_name, + override_application_settings, + validate_analytics_catmap_data, +) +from testing_support.mock_external_http_server import ( + MockExternalHTTPHResponseHeadersServer, +) +from testing_support.validators.validate_tt_parameters import validate_tt_parameters from newrelic.api.external_trace import ExternalTrace -from newrelic.api.transaction import (get_browser_timing_header, - set_transaction_name, get_browser_timing_footer, set_background_task, - current_transaction) +from newrelic.api.transaction import ( + current_transaction, + get_browser_timing_header, + set_background_task, + set_transaction_name, +) from newrelic.api.wsgi_application import wsgi_application -from newrelic.common.encoding_utils import obfuscate, json_encode - -from testing_support.fixtures import (override_application_settings, - override_application_name, validate_tt_parameters, - make_cross_agent_headers, validate_analytics_catmap_data) -from testing_support.mock_external_http_server import ( - MockExternalHTTPHResponseHeadersServer) +from newrelic.common.encoding_utils import json_encode, obfuscate -ENCODING_KEY = '1234567890123456789012345678901234567890' +ENCODING_KEY = "1234567890123456789012345678901234567890" CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, 'fixtures')) +JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, "fixtures")) OUTBOUD_REQUESTS = {} -_parameters_list = ["name", "appName", "transactionName", "transactionGuid", - "inboundPayload", "outboundRequests", "expectedIntrinsicFields", - "nonExpectedIntrinsicFields"] +_parameters_list = [ + "name", + "appName", + "transactionName", + "transactionGuid", + "inboundPayload", + "outboundRequests", + "expectedIntrinsicFields", + "nonExpectedIntrinsicFields", +] -@pytest.fixture(scope='module') +@pytest.fixture(scope="module") def server(): with MockExternalHTTPHResponseHeadersServer() as _server: yield _server @@ -61,8 +72,8 @@ def server(): def load_tests(): result = [] - path = os.path.join(JSON_DIR, 'cat_map.json') - with open(path, 'r') as fh: + path = os.path.join(JSON_DIR, "cat_map.json") + with open(path, "r") as fh: tests = json.load(fh) for test in tests: @@ -77,57 +88,49 @@ def load_tests(): @wsgi_application() def target_wsgi_application(environ, start_response): - status = '200 OK' + status = "200 OK" - txn_name = environ.get('txn') - if six.PY2: - txn_name = txn_name.decode('UTF-8') - txn_name = txn_name.split('/', 3) + txn_name = environ.get("txn").split("/", 3) - guid = environ.get('guid') - old_cat = environ.get('old_cat') == 'True' + guid = environ.get("guid") + old_cat = environ.get("old_cat") == "True" txn = current_transaction() txn.guid = guid for req in OUTBOUD_REQUESTS: # Change the transaction name before making an outbound call. - outgoing_name = req['outboundTxnName'].split('/', 3) - if outgoing_name[0] != 'WebTransaction': + outgoing_name = req["outboundTxnName"].split("/", 3) + if outgoing_name[0] != "WebTransaction": set_background_task(True) set_transaction_name(outgoing_name[2], group=outgoing_name[1]) - expected_outbound_header = obfuscate( - json_encode(req['expectedOutboundPayload']), ENCODING_KEY) - generated_outbound_header = dict( - ExternalTrace.generate_request_headers(txn)) + expected_outbound_header = obfuscate(json_encode(req["expectedOutboundPayload"]), ENCODING_KEY) + generated_outbound_header = dict(ExternalTrace.generate_request_headers(txn)) # A 500 error is returned because 'assert' statements in the wsgi app # are ignored. if old_cat: - if (expected_outbound_header != - generated_outbound_header['X-NewRelic-Transaction']): - status = '500 Outbound Headers Check Failed.' + if expected_outbound_header != generated_outbound_header["X-NewRelic-Transaction"]: + status = "500 Outbound Headers Check Failed." else: - if 'X-NewRelic-Transaction' in generated_outbound_header: - status = '500 Outbound Headers Check Failed.' - r = urlopen(environ['server_url']) + if "X-NewRelic-Transaction" in generated_outbound_header: + status = "500 Outbound Headers Check Failed." + r = urlopen(environ["server_url"]) # nosec B310 r.read(10) # Set the final transaction name. - if txn_name[0] != 'WebTransaction': + if txn_name[0] != "WebTransaction": set_background_task(True) set_transaction_name(txn_name[2], group=txn_name[1]) - text = '%s

RESPONSE

%s' + text = "%s

RESPONSE

" - output = (text % (get_browser_timing_header(), - get_browser_timing_footer())).encode('UTF-8') + output = (text % get_browser_timing_header()).encode("UTF-8") - response_headers = [('Content-type', 'text/html; charset=utf-8'), - ('Content-Length', str(len(output)))] + response_headers = [("Content-type", "text/html; charset=utf-8"), ("Content-Length", str(len(output)))] start_response(status, response_headers) return [output] @@ -137,26 +140,35 @@ def target_wsgi_application(environ, start_response): @pytest.mark.parametrize(_parameters, load_tests()) -@pytest.mark.parametrize('old_cat', (True, False)) -def test_cat_map(name, appName, transactionName, transactionGuid, - inboundPayload, outboundRequests, expectedIntrinsicFields, - nonExpectedIntrinsicFields, old_cat, server): +@pytest.mark.parametrize("old_cat", (True, False)) +def test_cat_map( + name, + appName, + transactionName, + transactionGuid, + inboundPayload, + outboundRequests, + expectedIntrinsicFields, + nonExpectedIntrinsicFields, + old_cat, + server, +): global OUTBOUD_REQUESTS OUTBOUD_REQUESTS = outboundRequests or {} _custom_settings = { - 'cross_process_id': '1#1', - 'encoding_key': ENCODING_KEY, - 'trusted_account_ids': [1], - 'cross_application_tracer.enabled': True, - 'distributed_tracing.enabled': not old_cat, - 'transaction_tracer.transaction_threshold': 0.0, + "cross_process_id": "1#1", + "encoding_key": ENCODING_KEY, + "trusted_account_ids": [1], + "cross_application_tracer.enabled": True, + "distributed_tracing.enabled": not old_cat, + "transaction_tracer.transaction_threshold": 0.0, } if expectedIntrinsicFields and old_cat: _external_node_params = { - 'path_hash': expectedIntrinsicFields['nr.pathHash'], - 'trip_id': expectedIntrinsicFields['nr.tripId'], + "path_hash": expectedIntrinsicFields["nr.pathHash"], + "trip_id": expectedIntrinsicFields["nr.tripId"], } else: _external_node_params = [] @@ -167,38 +179,39 @@ def test_cat_map(name, appName, transactionName, transactionGuid, expectedIntrinsicFields = {} @validate_tt_parameters(required_params=_external_node_params) - @validate_analytics_catmap_data(transactionName, - expected_attributes=expectedIntrinsicFields, - non_expected_attributes=nonExpectedIntrinsicFields) + @validate_analytics_catmap_data( + transactionName, expected_attributes=expectedIntrinsicFields, non_expected_attributes=nonExpectedIntrinsicFields + ) @override_application_settings(_custom_settings) @override_application_name(appName) def run_cat_test(): - - if six.PY2: - txn_name = transactionName.encode('UTF-8') - guid = transactionGuid.encode('UTF-8') - else: - txn_name = transactionName - guid = transactionGuid + txn_name = transactionName + guid = transactionGuid # Only generate old cat style headers. This will test to make sure we # are properly ignoring these headers when the agent is using better # cat. - headers = make_cross_agent_headers(inboundPayload, ENCODING_KEY, '1#1') - response = target_application.get('/', headers=headers, - extra_environ={'txn': txn_name, 'guid': guid, - 'old_cat': str(old_cat), - 'server_url': 'http://localhost:%d' % server.port}) + headers = make_cross_agent_headers(inboundPayload, ENCODING_KEY, "1#1") + response = target_application.get( + "/", + headers=headers, + extra_environ={ + "txn": txn_name, + "guid": guid, + "old_cat": str(old_cat), + "server_url": f"http://localhost:{server.port}", + }, + ) # Validation of analytic data happens in the decorator. - assert response.status == '200 OK' + assert response.status == "200 OK" content = response.html.html.body.p.string # Validate actual body content as sanity check. - assert content == 'RESPONSE' + assert content == "RESPONSE" run_cat_test() diff --git a/tests/cross_agent/test_collector_hostname.py b/tests/cross_agent/test_collector_hostname.py index 2ce39a1ec8..a43e91262c 100644 --- a/tests/cross_agent/test_collector_hostname.py +++ b/tests/cross_agent/test_collector_hostname.py @@ -18,14 +18,9 @@ import sys import tempfile -import pytest +from importlib import reload -try: - # python 2.x - reload -except NameError: - # python 3.x - from importlib import reload +import pytest CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) @@ -68,9 +63,9 @@ def _test_collector_hostname( os.environ["NEW_RELIC_LICENSE_KEY"] = env_key if config_file_key: - ini_contents += "\nlicense_key = %s" % config_file_key + ini_contents += f"\nlicense_key = {config_file_key}" if config_override_host: - ini_contents += "\nhost = %s" % config_override_host + ini_contents += f"\nhost = {config_override_host}" import newrelic.config as config import newrelic.core.config as core_config diff --git a/tests/cross_agent/test_datstore_instance.py b/tests/cross_agent/test_datastore_instance.py similarity index 52% rename from tests/cross_agent/test_datstore_instance.py rename to tests/cross_agent/test_datastore_instance.py index aa095400fa..79a95e0be1 100644 --- a/tests/cross_agent/test_datstore_instance.py +++ b/tests/cross_agent/test_datastore_instance.py @@ -14,34 +14,40 @@ import json import os + import pytest from newrelic.api.background_task import background_task -from newrelic.api.database_trace import (register_database_client, - enable_datastore_instance_feature) +from newrelic.api.database_trace import register_database_client from newrelic.api.transaction import current_transaction from newrelic.core.database_node import DatabaseNode from newrelic.core.stats_engine import StatsEngine -FIXTURE = os.path.join(os.curdir, - 'fixtures', 'datastores', 'datastore_instances.json') +CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) +FIXTURE = os.path.join(CURRENT_DIR, "fixtures", "datastores", "datastore_instances.json") -_parameters_list = ['name', 'system_hostname', 'db_hostname', - 'product', 'port', 'unix_socket', 'database_path', - 'expected_instance_metric'] +_parameters_list = [ + "name", + "system_hostname", + "db_hostname", + "product", + "port", + "unix_socket", + "database_path", + "expected_instance_metric", +] -_parameters = ','.join(_parameters_list) +_parameters = ",".join(_parameters_list) def _load_tests(): - with open(FIXTURE, 'r') as fh: + with open(FIXTURE, "r") as fh: js = fh.read() return json.loads(js) def _parametrize_test(test): - return tuple([test.get(f, None if f != 'db_hostname' else 'localhost') - for f in _parameters_list]) + return tuple([test.get(f, None if f != "db_hostname" else "localhost") for f in _parameters_list]) _datastore_tests = [_parametrize_test(t) for t in _load_tests()] @@ -49,45 +55,44 @@ def _parametrize_test(test): @pytest.mark.parametrize(_parameters, _datastore_tests) @background_task() -def test_datastore_instance(name, system_hostname, db_hostname, - product, port, unix_socket, database_path, - expected_instance_metric, monkeypatch): +def test_datastore_instance( + name, system_hostname, db_hostname, product, port, unix_socket, database_path, expected_instance_metric, monkeypatch +): - monkeypatch.setattr('newrelic.common.system_info.gethostname', - lambda: system_hostname) + monkeypatch.setattr("newrelic.common.system_info.gethostname", lambda: system_hostname) - class FakeModule(): + class FakeModule: pass register_database_client(FakeModule, product) - enable_datastore_instance_feature(FakeModule) port_path_or_id = port or database_path or unix_socket - node = DatabaseNode(dbapi2_module=FakeModule, - sql='', - children=[], - start_time=0, - end_time=1, - duration=1, - exclusive=1, - stack_trace=None, - sql_format='obfuscated', - connect_params=None, - cursor_params=None, - sql_parameters=None, - execute_params=None, - host=db_hostname, - port_path_or_id=port_path_or_id, - database_name=database_path, - guid=None, - agent_attributes={}, - user_attributes={}, + node = DatabaseNode( + dbapi2_module=FakeModule, + sql="", + children=[], + start_time=0, + end_time=1, + duration=1, + exclusive=1, + stack_trace=None, + sql_format="obfuscated", + connect_params=None, + cursor_params=None, + sql_parameters=None, + execute_params=None, + host=db_hostname, + port_path_or_id=port_path_or_id, + database_name=database_path, + guid=None, + agent_attributes={}, + user_attributes={}, ) empty_stats = StatsEngine() transaction = current_transaction() - unscoped_scope = '' + unscoped_scope = "" # Check 'Datastore/instance' metric to confirm that: # 1. metric name is reported correctly diff --git a/tests/cross_agent/test_distributed_tracing.py b/tests/cross_agent/test_distributed_tracing.py index 0ff46eea24..715d22fbef 100644 --- a/tests/cross_agent/test_distributed_tracing.py +++ b/tests/cross_agent/test_distributed_tracing.py @@ -14,54 +14,70 @@ import json import os + import pytest import webtest +from testing_support.fixtures import override_application_settings, validate_attributes +from testing_support.validators.validate_error_event_attributes import ( + validate_error_event_attributes, +) +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_event_attributes import ( + validate_transaction_event_attributes, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.transaction import current_transaction from newrelic.api.wsgi_application import wsgi_application from newrelic.common.encoding_utils import DistributedTracePayload from newrelic.common.object_wrapper import transient_function_wrapper -from testing_support.fixtures import (override_application_settings, - validate_error_event_attributes, validate_attributes) -from testing_support.validators.validate_span_events import ( - validate_span_events) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes - CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, 'fixtures', - 'distributed_tracing')) - -_parameters_list = ['account_id', 'comment', 'expected_metrics', - 'force_sampled_true', 'inbound_payloads', 'intrinsics', - 'major_version', 'minor_version', 'outbound_payloads', - 'raises_exception', 'span_events_enabled', 'test_name', - 'transport_type', 'trusted_account_key', 'web_transaction'] -_parameters = ','.join(_parameters_list) +JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, "fixtures", "distributed_tracing")) + +_parameters_list = [ + "account_id", + "comment", + "expected_metrics", + "force_sampled_true", + "inbound_payloads", + "intrinsics", + "major_version", + "minor_version", + "outbound_payloads", + "raises_exception", + "span_events_enabled", + "test_name", + "transport_type", + "trusted_account_key", + "web_transaction", +] +_parameters = ",".join(_parameters_list) def load_tests(): result = [] - path = os.path.join(JSON_DIR, 'distributed_tracing.json') - with open(path, 'r') as fh: + path = os.path.join(JSON_DIR, "distributed_tracing.json") + with open(path, "r") as fh: tests = json.load(fh) for test in tests: values = (test.get(param, None) for param in _parameters_list) - param = pytest.param(*values, id=test.get('test_name')) + param = pytest.param(*values, id=test.get("test_name")) result.append(param) return result def override_compute_sampled(override): - @transient_function_wrapper('newrelic.core.adaptive_sampler', - 'AdaptiveSampler.compute_sampled') + @transient_function_wrapper("newrelic.core.adaptive_sampler", "AdaptiveSampler.compute_sampled") def _override_compute_sampled(wrapped, instance, args, kwargs): if override: return True return wrapped(*args, **kwargs) + return _override_compute_sampled @@ -70,58 +86,54 @@ def assert_payload(payload, payload_assertions, major_version, minor_version): # flatten payload so it matches the test: # payload['d']['ac'] -> payload['d.ac'] - d = payload.pop('d') + d = payload.pop("d") for key, value in d.items(): - payload['d.%s' % key] = value + payload[f"d.{key}"] = value - for expected in payload_assertions.get('expected', []): + for expected in payload_assertions.get("expected", []): assert expected in payload - for unexpected in payload_assertions.get('unexpected', []): + for unexpected in payload_assertions.get("unexpected", []): assert unexpected not in payload - for key, value in payload_assertions.get('exact', {}).items(): + for key, value in payload_assertions.get("exact", {}).items(): assert key in payload if isinstance(value, list): value = tuple(value) assert payload[key] == value - assert payload['v'][0] == major_version - assert payload['v'][1] == minor_version + assert payload["v"][0] == major_version + assert payload["v"][1] == minor_version @wsgi_application() def target_wsgi_application(environ, start_response): - status = '200 OK' - output = b'hello world' - response_headers = [('Content-type', 'text/html; charset=utf-8'), - ('Content-Length', str(len(output)))] + status = "200 OK" + output = b"hello world" + response_headers = [("Content-type", "text/html; charset=utf-8"), ("Content-Length", str(len(output)))] txn = current_transaction() - txn.set_transaction_name(test_settings['test_name']) + txn.set_transaction_name(test_settings["test_name"]) - if not test_settings['web_transaction']: + if not test_settings["web_transaction"]: txn.background_task = True - if test_settings['raises_exception']: + if test_settings["raises_exception"]: try: 1 / 0 except ZeroDivisionError: txn.notice_error() - extra_inbound_payloads = test_settings['extra_inbound_payloads'] + extra_inbound_payloads = test_settings["extra_inbound_payloads"] for payload, expected_result in extra_inbound_payloads: - result = txn.accept_distributed_trace_payload(payload, - test_settings['transport_type']) + result = txn.accept_distributed_trace_payload(payload, test_settings["transport_type"]) assert result is expected_result - outbound_payloads = test_settings['outbound_payloads'] + outbound_payloads = test_settings["outbound_payloads"] if outbound_payloads: for payload_assertions in outbound_payloads: payload = txn._create_distributed_trace_payload() - assert_payload(payload, payload_assertions, - test_settings['major_version'], - test_settings['minor_version']) + assert_payload(payload, payload_assertions, test_settings["major_version"], test_settings["minor_version"]) start_response(status, response_headers) return [output] @@ -131,14 +143,26 @@ def target_wsgi_application(environ, start_response): @pytest.mark.parametrize(_parameters, load_tests()) -def test_distributed_tracing(account_id, comment, expected_metrics, - force_sampled_true, inbound_payloads, intrinsics, major_version, - minor_version, outbound_payloads, raises_exception, - span_events_enabled, test_name, transport_type, trusted_account_key, - web_transaction): +def test_distributed_tracing( + account_id, + comment, + expected_metrics, + force_sampled_true, + inbound_payloads, + intrinsics, + major_version, + minor_version, + outbound_payloads, + raises_exception, + span_events_enabled, + test_name, + transport_type, + trusted_account_key, + web_transaction, +): extra_inbound_payloads = [] - if transport_type != 'HTTP': + if transport_type != "HTTP": # Since wsgi_application calls accept_distributed_trace_payload # automatically with transport_type='HTTP', we must defer this call # until we can specify the transport type. @@ -153,78 +177,68 @@ def test_distributed_tracing(account_id, comment, expected_metrics, global test_settings test_settings = { - 'test_name': test_name, - 'web_transaction': web_transaction, - 'raises_exception': raises_exception, - 'extra_inbound_payloads': extra_inbound_payloads, - 'outbound_payloads': outbound_payloads, - 'transport_type': transport_type, - 'major_version': major_version, - 'minor_version': minor_version, + "test_name": test_name, + "web_transaction": web_transaction, + "raises_exception": raises_exception, + "extra_inbound_payloads": extra_inbound_payloads, + "outbound_payloads": outbound_payloads, + "transport_type": transport_type, + "major_version": major_version, + "minor_version": minor_version, } override_settings = { - 'distributed_tracing.enabled': True, - 'span_events.enabled': span_events_enabled, - 'account_id': account_id, - 'trusted_account_key': trusted_account_key, + "distributed_tracing.enabled": True, + "span_events.enabled": span_events_enabled, + "account_id": account_id, + "trusted_account_key": trusted_account_key, } - common_required = intrinsics['common']['expected'] - common_forgone = intrinsics['common']['unexpected'] - common_exact = intrinsics['common'].get('exact', {}) - - txn_intrinsics = intrinsics.get('Transaction', {}) - txn_event_required = {'agent': [], 'user': [], - 'intrinsic': txn_intrinsics.get('expected', [])} - txn_event_required['intrinsic'].extend(common_required) - txn_event_forgone = {'agent': [], 'user': [], - 'intrinsic': txn_intrinsics.get('unexpected', [])} - txn_event_forgone['intrinsic'].extend(common_forgone) - txn_event_exact = {'agent': {}, 'user': {}, - 'intrinsic': txn_intrinsics.get('exact', {})} - txn_event_exact['intrinsic'].update(common_exact) + common_required = intrinsics["common"]["expected"] + common_forgone = intrinsics["common"]["unexpected"] + common_exact = intrinsics["common"].get("exact", {}) + + txn_intrinsics = intrinsics.get("Transaction", {}) + txn_event_required = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("expected", [])} + txn_event_required["intrinsic"].extend(common_required) + txn_event_forgone = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("unexpected", [])} + txn_event_forgone["intrinsic"].extend(common_forgone) + txn_event_exact = {"agent": {}, "user": {}, "intrinsic": txn_intrinsics.get("exact", {})} + txn_event_exact["intrinsic"].update(common_exact) headers = {} if inbound_payloads: payload = DistributedTracePayload(inbound_payloads[0]) - headers['newrelic'] = payload.http_safe() - - @validate_transaction_metrics(test_name, - rollup_metrics=expected_metrics, - background_task=not web_transaction) - @validate_transaction_event_attributes( - txn_event_required, txn_event_forgone, txn_event_exact) - @validate_attributes('intrinsic', common_required, common_forgone) + headers["newrelic"] = payload.http_safe() + + @validate_transaction_metrics(test_name, rollup_metrics=expected_metrics, background_task=not web_transaction) + @validate_transaction_event_attributes(txn_event_required, txn_event_forgone, txn_event_exact) + @validate_attributes("intrinsic", common_required, common_forgone) @override_compute_sampled(force_sampled_true) @override_application_settings(override_settings) def _test(): - response = test_application.get('/', headers=headers) - assert 'X-NewRelic-App-Data' not in response.headers + response = test_application.get("/", headers=headers) + assert "X-NewRelic-App-Data" not in response.headers - if 'Span' in intrinsics: - span_intrinsics = intrinsics.get('Span') - span_expected = span_intrinsics.get('expected', []) + if "Span" in intrinsics: + span_intrinsics = intrinsics.get("Span") + span_expected = span_intrinsics.get("expected", []) span_expected.extend(common_required) - span_unexpected = span_intrinsics.get('unexpected', []) + span_unexpected = span_intrinsics.get("unexpected", []) span_unexpected.extend(common_forgone) - span_exact = span_intrinsics.get('exact', {}) + span_exact = span_intrinsics.get("exact", {}) span_exact.update(common_exact) - _test = validate_span_events(exact_intrinsics=span_exact, - expected_intrinsics=span_expected, - unexpected_intrinsics=span_unexpected)(_test) + _test = validate_span_events( + exact_intrinsics=span_exact, expected_intrinsics=span_expected, unexpected_intrinsics=span_unexpected + )(_test) elif not span_events_enabled: _test = validate_span_events(count=0)(_test) if raises_exception: - error_event_required = {'agent': [], 'user': [], - 'intrinsic': common_required} - error_event_forgone = {'agent': [], 'user': [], - 'intrinsic': common_forgone} - error_event_exact = {'agent': {}, 'user': {}, - 'intrinsic': common_exact} - _test = validate_error_event_attributes(error_event_required, - error_event_forgone, error_event_exact)(_test) + error_event_required = {"agent": [], "user": [], "intrinsic": common_required} + error_event_forgone = {"agent": [], "user": [], "intrinsic": common_forgone} + error_event_exact = {"agent": {}, "user": {}, "intrinsic": common_exact} + _test = validate_error_event_attributes(error_event_required, error_event_forgone, error_event_exact)(_test) _test() diff --git a/tests/cross_agent/test_docker.py b/tests/cross_agent/test_docker_container_id.py similarity index 65% rename from tests/cross_agent/test_docker.py rename to tests/cross_agent/test_docker_container_id.py index 9bc1a73630..f14e80fcd8 100644 --- a/tests/cross_agent/test_docker.py +++ b/tests/cross_agent/test_docker_container_id.py @@ -19,7 +19,8 @@ import newrelic.common.utilization as u -DOCKER_FIXTURE = os.path.join(os.curdir, 'fixtures', 'docker_container_id') +CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) +DOCKER_FIXTURE = os.path.join(CURRENT_DIR, 'fixtures', 'docker_container_id') def _load_docker_test_attributes(): @@ -38,13 +39,23 @@ def _load_docker_test_attributes(): return docker_test_attributes +def mock_open(mock_file): + def _mock_open(filename, mode): + if filename == "/proc/self/mountinfo": + raise FileNotFoundError() + elif filename == "/proc/self/cgroup": + return mock_file + raise RuntimeError() + return _mock_open + + @pytest.mark.parametrize('filename, containerId', _load_docker_test_attributes()) -def test_docker_container_id(filename, containerId): +def test_docker_container_id_v1(monkeypatch, filename, containerId): path = os.path.join(DOCKER_FIXTURE, filename) with open(path, 'rb') as f: - with mock.patch.object(u, 'open', create=True, return_value=f): - if containerId is not None: - assert u.DockerUtilization.detect() == {'id': containerId} - else: - assert u.DockerUtilization.detect() is None + monkeypatch.setattr(u, "open", mock_open(f), raising=False) + if containerId is not None: + assert u.DockerUtilization.detect() == {'id': containerId} + else: + assert u.DockerUtilization.detect() is None diff --git a/tests/cross_agent/test_docker_container_id_v2.py b/tests/cross_agent/test_docker_container_id_v2.py new file mode 100644 index 0000000000..eee4e1305a --- /dev/null +++ b/tests/cross_agent/test_docker_container_id_v2.py @@ -0,0 +1,61 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import mock +import os +import pytest + +import newrelic.common.utilization as u + +CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) +DOCKER_FIXTURE = os.path.join(CURRENT_DIR, 'fixtures', 'docker_container_id_v2') + + +def _load_docker_test_attributes(): + """Returns a list of docker test attributes in the form: + [(, ), ...] + + """ + docker_test_attributes = [] + test_cases = os.path.join(DOCKER_FIXTURE, 'cases.json') + with open(test_cases, 'r') as fh: + js = fh.read() + json_list = json.loads(js) + for json_record in json_list: + docker_test_attributes.append( + (json_record['filename'], json_record['containerId'])) + return docker_test_attributes + + +def mock_open(mock_file): + def _mock_open(filename, mode): + if filename == "/proc/self/cgroup": + raise FileNotFoundError() + elif filename == "/proc/self/mountinfo": + return mock_file + raise RuntimeError() + return _mock_open + + +@pytest.mark.parametrize('filename, containerId', + _load_docker_test_attributes()) +def test_docker_container_id_v2(monkeypatch, filename, containerId): + path = os.path.join(DOCKER_FIXTURE, filename) + with open(path, 'rb') as f: + monkeypatch.setattr(u, "open", mock_open(f), raising=False) + if containerId is not None: + assert u.DockerUtilization.detect() == {'id': containerId} + else: + assert u.DockerUtilization.detect() is None diff --git a/tests/cross_agent/test_ecs_data.py b/tests/cross_agent/test_ecs_data.py new file mode 100644 index 0000000000..9ec32d3b5f --- /dev/null +++ b/tests/cross_agent/test_ecs_data.py @@ -0,0 +1,54 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import pytest +import newrelic.common.utilization as u +from fixtures.ecs_container_id.ecs_mock_server import mock_server, bad_response_mock_server +from test_pcf_utilization_data import Environ + + +@pytest.mark.parametrize("env_key", ["ECS_CONTAINER_METADATA_URI_V4", "ECS_CONTAINER_METADATA_URI"]) +def test_ecs_docker_container_id(env_key, mock_server): + mock_endpoint = "http://localhost:%d" % mock_server.port + env_dict = {env_key: mock_endpoint} + + with Environ(env_dict): + data = u.ECSUtilization.detect() + + assert data == {"ecsDockerId": "1e1698469422439ea356071e581e8545-2769485393"} + + +@pytest.mark.parametrize( + "env_dict", [{"ECS_CONTAINER_METADATA_URI_V4": "http:/invalid-uri"}, {"ECS_CONTAINER_METADATA_URI_V4": None}] +) +def test_ecs_docker_container_id_bad_uri(env_dict, mock_server): + with Environ(env_dict): + data = u.ECSUtilization.detect() + + assert data is None + + +def test_ecs_docker_container_id_bad_response(bad_response_mock_server): + mock_endpoint = "http://localhost:%d" % bad_response_mock_server.port + env_dict = {"ECS_CONTAINER_METADATA_URI": mock_endpoint} + + with Environ(env_dict): + data = u.ECSUtilization.detect() + + assert data is None + + +def test_ecs_container_id_no_metadata_env_vars(): + assert u.ECSUtilization.detect() is None diff --git a/tests/cross_agent/test_labels_and_rollups.py b/tests/cross_agent/test_labels_and_rollups.py index d333ec35ba..15ebb1e369 100644 --- a/tests/cross_agent/test_labels_and_rollups.py +++ b/tests/cross_agent/test_labels_and_rollups.py @@ -21,7 +21,8 @@ from testing_support.fixtures import override_application_settings -FIXTURE = os.path.join(os.curdir, 'fixtures', 'labels.json') +CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) +FIXTURE = os.path.join(CURRENT_DIR, 'fixtures', 'labels.json') def _load_tests(): with open(FIXTURE, 'r') as fh: diff --git a/tests/cross_agent/test_lambda_event_source.py b/tests/cross_agent/test_lambda_event_source.py index 511294cf6f..bea041f1a3 100644 --- a/tests/cross_agent/test_lambda_event_source.py +++ b/tests/cross_agent/test_lambda_event_source.py @@ -14,65 +14,70 @@ import json import os + import pytest +from testing_support.fixtures import override_application_settings +from testing_support.validators.validate_transaction_event_attributes import ( + validate_transaction_event_attributes, +) from newrelic.api.lambda_handler import lambda_handler -from testing_support.fixtures import override_application_settings -from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -FIXTURE_DIR = os.path.normpath(os.path.join(CURRENT_DIR, 'fixtures')) -FIXTURE = os.path.join(FIXTURE_DIR, 'lambda_event_source.json') +FIXTURE_DIR = os.path.normpath(os.path.join(CURRENT_DIR, "fixtures")) +FIXTURE = os.path.join(FIXTURE_DIR, "lambda_event_source.json") tests = {} events = {} def _load_tests(): - with open(FIXTURE, 'r') as fh: + with open(FIXTURE, "r") as fh: for test in json.loads(fh.read()): - test_name = test.pop('name') + test_name = test.pop("name") - test_file = test_name + '.json' - path = os.path.join(FIXTURE_DIR, 'lambda_event_source', test_file) - with open(path, 'r') as fh: + test_file = f"{test_name}.json" + path = os.path.join(FIXTURE_DIR, "lambda_event_source", test_file) + with open(path, "r") as fh: events[test_name] = json.loads(fh.read()) tests[test_name] = test return tests.keys() -class Context(object): - aws_request_id = 'cookies' - invoked_function_arn = 'arn' - function_name = 'cats' - function_version = '$LATEST' +class Context(): + aws_request_id = "cookies" + invoked_function_arn = "arn" + function_name = "cats" + function_version = "$LATEST" memory_limit_in_mb = 128 @lambda_handler() def handler(event, context): return { - 'statusCode': '200', - 'body': '{}', - 'headers': { - 'Content-Type': 'application/json', - 'Content-Length': 2, + "statusCode": "200", + "body": "{}", + "headers": { + "Content-Type": "application/json", + "Content-Length": 2, }, } -@pytest.mark.parametrize('test_name', _load_tests()) +# The lambda_hander has been deprecated for 3+ years +@pytest.mark.skip(reason="The lambda_handler has been deprecated") +@pytest.mark.parametrize("test_name", _load_tests()) def test_lambda_event_source(test_name): - _exact = {'user': {}, 'intrinsic': {}, 'agent': {}} + _exact = {"user": {}, "intrinsic": {}, "agent": {}} - expected_arn = tests[test_name].get('aws.lambda.eventSource.arn', None) + expected_arn = tests[test_name].get("aws.lambda.eventSource.arn", None) if expected_arn: - _exact['agent']['aws.lambda.eventSource.arn'] = expected_arn + _exact["agent"]["aws.lambda.eventSource.arn"] = expected_arn else: pytest.skip("Nothing to test!") return - @override_application_settings({'attributes.include': ['aws.*']}) + @override_application_settings({"attributes.include": ["aws.*"]}) @validate_transaction_event_attributes({}, exact_attrs=_exact) def _test(): handler(events[test_name], Context) diff --git a/tests/cross_agent/test_pcf_utilization_data.py b/tests/cross_agent/test_pcf_utilization_data.py index 28b56f7592..108e77d890 100644 --- a/tests/cross_agent/test_pcf_utilization_data.py +++ b/tests/cross_agent/test_pcf_utilization_data.py @@ -45,7 +45,7 @@ def _parametrize_test(test): _pcf_tests = [_parametrize_test(t) for t in _load_tests()] -class Environ(object): +class Environ(): def __init__(self, env_dict): env_dict = env_dict or {} cleaned_env_dict = {} @@ -66,7 +66,7 @@ def __exit__(self, *args, **kwargs): os.environ = INITIAL_ENV -class MockResponse(object): +class MockResponse(): def __init__(self, code, body): self.code = code diff --git a/tests/cross_agent/test_rules.py b/tests/cross_agent/test_rules.py index e37db787cd..ce2983c90e 100644 --- a/tests/cross_agent/test_rules.py +++ b/tests/cross_agent/test_rules.py @@ -16,23 +16,23 @@ import os import pytest -from newrelic.core.rules_engine import RulesEngine, NormalizationRule +from newrelic.api.application import application_instance +from newrelic.api.background_task import background_task +from newrelic.api.transaction import record_custom_metric +from newrelic.core.rules_engine import RulesEngine + +from testing_support.validators.validate_metric_payload import validate_metric_payload CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) FIXTURE = os.path.normpath(os.path.join( CURRENT_DIR, 'fixtures', 'rules.json')) + def _load_tests(): with open(FIXTURE, 'r') as fh: js = fh.read() return json.loads(js) -def _prepare_rules(test_rules): - # ensure all keys are present, if not present set to an empty string - for rule in test_rules: - for key in NormalizationRule._fields: - rule[key] = rule.get(key, '') - return test_rules def _make_case_insensitive(rules): # lowercase each rule @@ -42,14 +42,14 @@ def _make_case_insensitive(rules): rule['replacement'] = rule['replacement'].lower() return rules + @pytest.mark.parametrize('test_group', _load_tests()) def test_rules_engine(test_group): # FIXME: The test fixture assumes that matching is case insensitive when it # is not. To avoid errors, just lowercase all rules, inputs, and expected # values. - insense_rules = _make_case_insensitive(test_group['rules']) - test_rules = _prepare_rules(insense_rules) + test_rules = _make_case_insensitive(test_group['rules']) rules_engine = RulesEngine(test_rules) for test in test_group['tests']: @@ -66,3 +66,46 @@ def test_rules_engine(test_group): assert expected == '' else: assert result == expected + + +@pytest.mark.parametrize('test_group', _load_tests()) +def test_rules_engine_metric_harvest(test_group): + # FIXME: The test fixture assumes that matching is case insensitive when it + # is not. To avoid errors, just lowercase all rules, inputs, and expected + # values. + test_rules = _make_case_insensitive(test_group['rules']) + rules_engine = RulesEngine(test_rules) + + # Set rules engine on core application + api_application = application_instance(activate=False) + api_name = api_application.name + core_application = api_application._agent.application(api_name) + old_rules = core_application._rules_engine["metric"] # save previoius rules + core_application._rules_engine["metric"] = rules_engine + + def send_metrics(): + # Send all metrics in this test batch in one transaction, then harvest so the normalizer is run. + @background_task(name="send_metrics") + def _test(): + for test in test_group['tests']: + # lowercase each value + input_str = test['input'].lower() + record_custom_metric(input_str, {"count": 1}) + _test() + core_application.harvest() + + try: + # Create a map of all result metrics to validate after harvest + test_metrics = [] + for test in test_group['tests']: + expected = (test['expected'] or '').lower() + if expected == '': # Ignored + test_metrics.append((expected, None)) + else: + test_metrics.append((expected, 1)) + + # Harvest and validate resulting payload + validate_metric_payload(metrics=test_metrics)(send_metrics)() + finally: + # Replace original rules engine + core_application._rules_engine["metric"] = old_rules diff --git a/tests/cross_agent/test_rum_client_config.py b/tests/cross_agent/test_rum_client_config.py deleted file mode 100644 index c2a4a465f9..0000000000 --- a/tests/cross_agent/test_rum_client_config.py +++ /dev/null @@ -1,144 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os - -import pytest -import webtest -from testing_support.fixtures import override_application_settings - -from newrelic.api.transaction import ( - add_custom_attribute, - get_browser_timing_footer, - set_transaction_name, -) -from newrelic.api.wsgi_application import wsgi_application - - -def _load_tests(): - fixture = os.path.join(os.curdir, "fixtures", "rum_client_config.json") - with open(fixture, "r") as fh: - js = fh.read() - return json.loads(js) - - -fields = [ - "testname", - "apptime_milliseconds", - "queuetime_milliseconds", - "browser_monitoring.attributes.enabled", - "transaction_name", - "license_key", - "connect_reply", - "user_attributes", - "expected", -] - -# Replace . as not a valid character in python argument names - -field_names = ",".join([f.replace(".", "_") for f in fields]) - - -def _parametrize_test(test): - return tuple([test.get(f, None) for f in fields]) - - -_rum_tests = [_parametrize_test(t) for t in _load_tests()] - - -@wsgi_application() -def target_wsgi_application(environ, start_response): - status = "200 OK" - - txn_name = environ.get("txn_name") - set_transaction_name(txn_name, group="") - - user_attrs = json.loads(environ.get("user_attrs")) - for key, value in user_attrs.items(): - add_custom_attribute(key, value) - - text = "%s

RESPONSE

" - - output = (text % get_browser_timing_footer()).encode("UTF-8") - - response_headers = [("Content-Type", "text/html; charset=utf-8"), ("Content-Length", str(len(output)))] - start_response(status, response_headers) - - return [output] - - -target_application = webtest.TestApp(target_wsgi_application) - - -@pytest.mark.parametrize(field_names, _rum_tests) -def test_browser_montioring( - testname, - apptime_milliseconds, - queuetime_milliseconds, - browser_monitoring_attributes_enabled, - transaction_name, - license_key, - connect_reply, - user_attributes, - expected, -): - - settings = { - "browser_monitoring.attributes.enabled": browser_monitoring_attributes_enabled, - "license_key": license_key, - "js_agent_loader": "", - } - settings.update(connect_reply) - - @override_application_settings(settings) - def run_browser_data_test(): - - response = target_application.get( - "/", extra_environ={"txn_name": str(transaction_name), "user_attrs": json.dumps(user_attributes)} - ) - - # We actually put the "footer" in the header, the first script is the - # agent "header", the second one is where the data lives, hence the [1]. - - footer = response.html.html.head.find_all("script")[1] - footer_data = json.loads(footer.string.split("NREUM.info=")[1]) - - # Not feasible to test the time metric values in testing - - expected.pop("queueTime") - expected.pop("applicationTime") - assert footer_data["applicationTime"] >= 0 - assert footer_data["queueTime"] >= 0 - - # Python always prepends stuff to the transaction name, so this - # doesn't match the obscured value. - - expected.pop("transactionName") - - # Check that all other values are correct - - for key, value in expected.items(): - - # If there are no attributes, the spec allows us to omit the - # 'atts' field altogether, so we do. But, the cross agent tests - # don't omit it, so we need to special case 'atts' when we compare - # to 'expected'. - - if key == "atts" and value == "": - assert key not in footer_data - else: - assert footer_data[key] == value - - run_browser_data_test() diff --git a/tests/cross_agent/test_sql_obfuscation.py b/tests/cross_agent/test_sql_obfuscation.py index 480b0a4176..362fc797df 100644 --- a/tests/cross_agent/test_sql_obfuscation.py +++ b/tests/cross_agent/test_sql_obfuscation.py @@ -53,7 +53,7 @@ def get_quoting_styles(dialects): return set([_quoting_styles.get(dialect) for dialect in dialects]) -class DummyDB(object): +class DummyDB(): def __init__(self, quoting_style): self.quoting_style = quoting_style diff --git a/tests/cross_agent/test_utilization_configs.py b/tests/cross_agent/test_utilization_configs.py index 4a4adb4859..0b372f5769 100644 --- a/tests/cross_agent/test_utilization_configs.py +++ b/tests/cross_agent/test_utilization_configs.py @@ -17,6 +17,8 @@ import sys import tempfile +from importlib import reload + import pytest # NOTE: the test_utilization_settings_from_env_vars test mocks several of the @@ -29,12 +31,6 @@ from newrelic.common.utilization import CommonUtilization from newrelic.core.agent_protocol import AgentProtocol -try: - # python 2.x - reload -except NameError: - # python 3.x - from importlib import reload INITIAL_ENV = os.environ @@ -76,7 +72,7 @@ def getips(*args, **kwargs): return getips -class UpdatedSettings(object): +class UpdatedSettings(): def __init__(self): self.initial_settings = newrelic.core.config._settings @@ -179,7 +175,6 @@ def _patch_system_info(wrapped, instance, args, kwargs): @pytest.mark.parametrize("test", _load_tests()) def test_utilization_settings(test, monkeypatch): - env = test.get("input_environment_variables", {}) if test.get("input_pcf_guid"): diff --git a/tests/cross_agent/test_w3c_trace_context.py b/tests/cross_agent/test_w3c_trace_context.py index 05f157f7b7..b10ec60818 100644 --- a/tests/cross_agent/test_w3c_trace_context.py +++ b/tests/cross_agent/test_w3c_trace_context.py @@ -14,88 +14,104 @@ import json import os + import pytest import webtest -from newrelic.packages import six - -from newrelic.api.transaction import current_transaction +from testing_support.fixtures import override_application_settings, validate_attributes +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_event_attributes import ( + validate_transaction_event_attributes, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.transaction import ( + accept_distributed_trace_headers, + current_transaction, + insert_distributed_trace_headers, +) from newrelic.api.wsgi_application import wsgi_application -from newrelic.common.object_wrapper import transient_function_wrapper -from testing_support.validators.validate_span_events import ( - validate_span_events) -from testing_support.fixtures import (override_application_settings, - validate_attributes) from newrelic.common.encoding_utils import W3CTraceState -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes +from newrelic.common.object_wrapper import transient_function_wrapper CURRENT_DIR = os.path.dirname(os.path.realpath(__file__)) -JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, 'fixtures', - 'distributed_tracing')) - -_parameters_list = ('test_name', 'trusted_account_key', 'account_id', - 'web_transaction', 'raises_exception', 'force_sampled_true', - 'span_events_enabled', 'transport_type', 'inbound_headers', - 'outbound_payloads', 'intrinsics', 'expected_metrics') - -_parameters = ','.join(_parameters_list) +JSON_DIR = os.path.normpath(os.path.join(CURRENT_DIR, "fixtures", "distributed_tracing")) + +_parameters_list = ( + "test_name", + "trusted_account_key", + "account_id", + "web_transaction", + "raises_exception", + "force_sampled_true", + "span_events_enabled", + "transport_type", + "inbound_headers", + "outbound_payloads", + "intrinsics", + "expected_metrics", +) + +_parameters = ",".join(_parameters_list) XFAIL_TESTS = [ - 'spans_disabled_root', - 'missing_traceparent', - 'missing_traceparent_and_tracestate', - 'w3c_and_newrelc_headers_present_error_parsing_traceparent' + "spans_disabled_root", + "missing_traceparent", + "missing_traceparent_and_tracestate", + "w3c_and_newrelc_headers_present_error_parsing_traceparent", ] + def load_tests(): result = [] - path = os.path.join(JSON_DIR, 'trace_context.json') - with open(path, 'r') as fh: + path = os.path.join(JSON_DIR, "trace_context.json") + with open(path, "r") as fh: tests = json.load(fh) for test in tests: values = (test.get(param, None) for param in _parameters_list) - param = pytest.param(*values, id=test.get('test_name')) + param = pytest.param(*values, id=test.get("test_name")) result.append(param) return result ATTR_MAP = { - 'traceparent.version': 0, - 'traceparent.trace_id': 1, - 'traceparent.parent_id': 2, - 'traceparent.trace_flags': 3, - 'tracestate.version': 0, - 'tracestate.parent_type': 1, - 'tracestate.parent_account_id': 2, - 'tracestate.parent_application_id': 3, - 'tracestate.span_id': 4, - 'tracestate.transaction_id': 5, - 'tracestate.sampled': 6, - 'tracestate.priority': 7, - 'tracestate.timestamp': 8, - 'tracestate.tenant_id': None, + "traceparent.version": 0, + "traceparent.trace_id": 1, + "traceparent.parent_id": 2, + "traceparent.trace_flags": 3, + "tracestate.version": 0, + "tracestate.parent_type": 1, + "tracestate.parent_account_id": 2, + "tracestate.parent_application_id": 3, + "tracestate.span_id": 4, + "tracestate.transaction_id": 5, + "tracestate.sampled": 6, + "tracestate.priority": 7, + "tracestate.timestamp": 8, + "tracestate.tenant_id": None, } def validate_outbound_payload(actual, expected, trusted_account_key): - traceparent = '' - tracestate = '' + traceparent = "" + tracestate = "" for key, value in actual: - if key == 'traceparent': - traceparent = value.split('-') - elif key == 'tracestate': + if key == "traceparent": + traceparent = value.split("-") + elif key == "tracestate": vendors = W3CTraceState.decode(value) - nr_entry = vendors.pop(trusted_account_key + '@nr', '') - tracestate = nr_entry.split('-') - exact_values = expected.get('exact', {}) - expected_attrs = expected.get('expected', []) - unexpected_attrs = expected.get('unexpected', []) - expected_vendors = expected.get('vendors', []) + nr_entry = vendors.pop(f"{trusted_account_key}@nr", "") + tracestate = nr_entry.split("-") + exact_values = expected.get("exact", {}) + expected_attrs = expected.get("expected", []) + unexpected_attrs = expected.get("unexpected", []) + expected_vendors = expected.get("vendors", []) for key, value in exact_values.items(): - header = traceparent if key.startswith('traceparent.') else tracestate + header = traceparent if key.startswith("traceparent.") else tracestate attr = ATTR_MAP[key] if attr is not None: if isinstance(value, bool): @@ -106,13 +122,13 @@ def validate_outbound_payload(actual, expected, trusted_account_key): assert header[attr] == str(value) for key in expected_attrs: - header = traceparent if key.startswith('traceparent.') else tracestate + header = traceparent if key.startswith("traceparent.") else tracestate attr = ATTR_MAP[key] if attr is not None: assert header[attr], key for key in unexpected_attrs: - header = traceparent if key.startswith('traceparent.') else tracestate + header = traceparent if key.startswith("traceparent.") else tracestate attr = ATTR_MAP[key] if attr is not None: assert not header[attr], key @@ -125,127 +141,127 @@ def validate_outbound_payload(actual, expected, trusted_account_key): def target_wsgi_application(environ, start_response): transaction = current_transaction() - if not environ['.web_transaction']: + if not environ[".web_transaction"]: transaction.background_task = True - if environ['.raises_exception']: + if environ[".raises_exception"]: try: raise ValueError("oops") except: transaction.notice_error() - if '.inbound_headers' in environ: - transaction.accept_distributed_trace_headers( - environ['.inbound_headers'], - transport_type=environ['.transport_type'], + if ".inbound_headers" in environ: + accept_distributed_trace_headers( + environ[".inbound_headers"], + transport_type=environ[".transport_type"], ) payloads = [] - for _ in range(environ['.outbound_calls']): + for _ in range(environ[".outbound_calls"]): payloads.append([]) - transaction.insert_distributed_trace_headers(payloads[-1]) + insert_distributed_trace_headers(payloads[-1]) - start_response('200 OK', [('Content-Type', 'application/json')]) - return [json.dumps(payloads).encode('utf-8')] + start_response("200 OK", [("Content-Type", "application/json")]) + return [json.dumps(payloads).encode("utf-8")] test_application = webtest.TestApp(target_wsgi_application) def override_compute_sampled(override): - @transient_function_wrapper('newrelic.core.adaptive_sampler', - 'AdaptiveSampler.compute_sampled') + @transient_function_wrapper("newrelic.core.adaptive_sampler", "AdaptiveSampler.compute_sampled") def _override_compute_sampled(wrapped, instance, args, kwargs): if override: return True return wrapped(*args, **kwargs) + return _override_compute_sampled @pytest.mark.parametrize(_parameters, load_tests()) -def test_trace_context(test_name, trusted_account_key, account_id, - web_transaction, raises_exception, force_sampled_true, - span_events_enabled, transport_type, inbound_headers, - outbound_payloads, intrinsics, expected_metrics): - +def test_trace_context( + test_name, + trusted_account_key, + account_id, + web_transaction, + raises_exception, + force_sampled_true, + span_events_enabled, + transport_type, + inbound_headers, + outbound_payloads, + intrinsics, + expected_metrics, +): if test_name in XFAIL_TESTS: pytest.xfail("Waiting on cross agent tests update.") # Prepare assertions if not intrinsics: intrinsics = {} - common = intrinsics.get('common', {}) - common_required = common.get('expected', []) - common_forgone = common.get('unexpected', []) - common_exact = common.get('exact', {}) - - txn_intrinsics = intrinsics.get('Transaction', {}) - txn_event_required = {'agent': [], 'user': [], - 'intrinsic': txn_intrinsics.get('expected', [])} - txn_event_required['intrinsic'].extend(common_required) - txn_event_forgone = {'agent': [], 'user': [], - 'intrinsic': txn_intrinsics.get('unexpected', [])} - txn_event_forgone['intrinsic'].extend(common_forgone) - txn_event_exact = {'agent': {}, 'user': {}, - 'intrinsic': txn_intrinsics.get('exact', {})} - txn_event_exact['intrinsic'].update(common_exact) + common = intrinsics.get("common", {}) + common_required = common.get("expected", []) + common_forgone = common.get("unexpected", []) + common_exact = common.get("exact", {}) + + txn_intrinsics = intrinsics.get("Transaction", {}) + txn_event_required = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("expected", [])} + txn_event_required["intrinsic"].extend(common_required) + txn_event_forgone = {"agent": [], "user": [], "intrinsic": txn_intrinsics.get("unexpected", [])} + txn_event_forgone["intrinsic"].extend(common_forgone) + txn_event_exact = {"agent": {}, "user": {}, "intrinsic": txn_intrinsics.get("exact", {})} + txn_event_exact["intrinsic"].update(common_exact) override_settings = { - 'distributed_tracing.enabled': True, - 'span_events.enabled': span_events_enabled, - 'account_id': account_id, - 'trusted_account_key': trusted_account_key, + "distributed_tracing.enabled": True, + "span_events.enabled": span_events_enabled, + "account_id": account_id, + "trusted_account_key": trusted_account_key, } extra_environ = { - '.web_transaction': web_transaction, - '.raises_exception': raises_exception, - '.transport_type': transport_type, - '.outbound_calls': outbound_payloads and len(outbound_payloads) or 0, + ".web_transaction": web_transaction, + ".raises_exception": raises_exception, + ".transport_type": transport_type, + ".outbound_calls": outbound_payloads and len(outbound_payloads) or 0, } inbound_headers = inbound_headers and inbound_headers[0] or None - if transport_type != 'HTTP': - extra_environ['.inbound_headers'] = inbound_headers + if transport_type != "HTTP": + extra_environ[".inbound_headers"] = inbound_headers inbound_headers = None - elif six.PY2 and inbound_headers: - inbound_headers = { - k.encode('utf-8'): v.encode('utf-8') - for k, v in inbound_headers.items()} - - @validate_transaction_metrics(test_name, - group="Uri", - rollup_metrics=expected_metrics, - background_task=not web_transaction) - @validate_transaction_event_attributes( - txn_event_required, txn_event_forgone, txn_event_exact) - @validate_attributes('intrinsic', common_required, common_forgone) + + @validate_transaction_metrics( + test_name, group="Uri", rollup_metrics=expected_metrics, background_task=not web_transaction + ) + @validate_transaction_event_attributes(txn_event_required, txn_event_forgone, txn_event_exact) + @validate_attributes("intrinsic", common_required, common_forgone) @override_application_settings(override_settings) @override_compute_sampled(force_sampled_true) def _test(): return test_application.get( - '/' + test_name, + f"/{test_name}", headers=inbound_headers, extra_environ=extra_environ, ) - if 'Span' in intrinsics: - span_intrinsics = intrinsics.get('Span') - span_expected = span_intrinsics.get('expected', []) + if "Span" in intrinsics: + span_intrinsics = intrinsics.get("Span") + span_expected = span_intrinsics.get("expected", []) span_expected.extend(common_required) - span_unexpected = span_intrinsics.get('unexpected', []) + span_unexpected = span_intrinsics.get("unexpected", []) span_unexpected.extend(common_forgone) - span_exact = span_intrinsics.get('exact', {}) + span_exact = span_intrinsics.get("exact", {}) span_exact.update(common_exact) - _test = validate_span_events(exact_intrinsics=span_exact, - expected_intrinsics=span_expected, - unexpected_intrinsics=span_unexpected)(_test) + _test = validate_span_events( + exact_intrinsics=span_exact, expected_intrinsics=span_expected, unexpected_intrinsics=span_unexpected + )(_test) elif not span_events_enabled: _test = validate_span_events(count=0)(_test) response = _test() - assert response.status == '200 OK' + assert response.status == "200 OK" payloads = response.json if outbound_payloads: assert len(payloads) == len(outbound_payloads) diff --git a/tests/datastore_aiomcache/conftest.py b/tests/datastore_aiomcache/conftest.py new file mode 100644 index 0000000000..e8b1724f20 --- /dev/null +++ b/tests/datastore_aiomcache/conftest.py @@ -0,0 +1,34 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (datastore_aiomcache)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_aiomcache/test_aiomcache.py b/tests/datastore_aiomcache/test_aiomcache.py new file mode 100644 index 0000000000..32d9d44d39 --- /dev/null +++ b/tests/datastore_aiomcache/test_aiomcache.py @@ -0,0 +1,111 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import aiomcache +from testing_support.db_settings import memcached_settings +from testing_support.fixture.event_loop import event_loop as loop +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.transaction import set_background_task +from newrelic.common import system_info + +DB_SETTINGS = memcached_settings()[0] + +MEMCACHED_HOST = DB_SETTINGS["host"] +MEMCACHED_PORT = DB_SETTINGS["port"] +MEMCACHED_NAMESPACE = str(os.getpid()) +INSTANCE_METRIC_HOST = system_info.gethostname() if MEMCACHED_HOST == "127.0.0.1" else MEMCACHED_HOST +INSTANCE_METRIC_NAME = f"Datastore/instance/Memcached/{INSTANCE_METRIC_HOST}/{MEMCACHED_PORT}" + +_test_bt_set_get_delete_scoped_metrics = [ + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] + +_test_bt_set_get_delete_rollup_metrics = [ + ("Datastore/all", 3), + ("Datastore/allOther", 3), + ("Datastore/Memcached/all", 3), + ("Datastore/Memcached/allOther", 3), + (INSTANCE_METRIC_NAME, 3), + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] + + +@validate_transaction_metrics( + "test_aiomcache:test_bt_set_get_delete", + scoped_metrics=_test_bt_set_get_delete_scoped_metrics, + rollup_metrics=_test_bt_set_get_delete_rollup_metrics, + background_task=True, +) +@background_task() +def test_bt_set_get_delete(loop): + set_background_task(True) + client = aiomcache.Client(host=MEMCACHED_HOST, port=MEMCACHED_PORT) + + key = f"{MEMCACHED_NAMESPACE}key".encode() + data = "value".encode() + + loop.run_until_complete(client.set(key, data)) + value = loop.run_until_complete(client.get(key)) + loop.run_until_complete(client.delete(key)) + + assert value == data + + +_test_wt_set_get_delete_scoped_metrics = [ + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] + +_test_wt_set_get_delete_rollup_metrics = [ + ("Datastore/all", 3), + ("Datastore/allWeb", 3), + ("Datastore/Memcached/all", 3), + ("Datastore/Memcached/allWeb", 3), + (INSTANCE_METRIC_NAME, 3), + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] + + +@validate_transaction_metrics( + "test_aiomcache:test_wt_set_get_delete", + scoped_metrics=_test_wt_set_get_delete_scoped_metrics, + rollup_metrics=_test_wt_set_get_delete_rollup_metrics, + background_task=False, +) +@background_task() +def test_wt_set_get_delete(loop): + set_background_task(False) + client = aiomcache.Client(host=MEMCACHED_HOST, port=MEMCACHED_PORT) + + key = f"{MEMCACHED_NAMESPACE}key".encode() + data = "value".encode() + + loop.run_until_complete(client.set(key, data)) + value = loop.run_until_complete(client.get(key)) + loop.run_until_complete(client.delete(key)) + + assert value == data diff --git a/tests/datastore_aioredis/conftest.py b/tests/datastore_aioredis/conftest.py index d501292555..895b700deb 100644 --- a/tests/datastore_aioredis/conftest.py +++ b/tests/datastore_aioredis/conftest.py @@ -12,13 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest +import os -from newrelic.common.package_version_utils import get_package_version_tuple +import pytest from testing_support.db_settings import redis_settings +from testing_support.fixture.event_loop import ( # noqa: F401; pylint: disable=W0611 + event_loop as loop, +) +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) -from testing_support.fixture.event_loop import event_loop as loop -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from newrelic.common.package_version_utils import get_package_version_tuple try: import aioredis @@ -37,6 +43,7 @@ _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, @@ -62,8 +69,15 @@ def client(request, loop): raise NotImplementedError() else: if request.param == "Redis": - return loop.run_until_complete(aioredis.create_redis("redis://%s:%d" % (DB_SETTINGS["host"], DB_SETTINGS["port"]), db=0)) + return loop.run_until_complete( + aioredis.create_redis(f"redis://{DB_SETTINGS['host']}:{DB_SETTINGS['port']}", db=0) + ) elif request.param == "StrictRedis": pytest.skip("StrictRedis not implemented.") else: raise NotImplementedError() + + +@pytest.fixture(scope="session") +def key(): + return f"AIOREDIS-TEST-{str(os.getpid())}" diff --git a/tests/datastore_aioredis/test_custom_conn_pool.py b/tests/datastore_aioredis/test_custom_conn_pool.py index b09cf0bdd3..e976f5c728 100644 --- a/tests/datastore_aioredis/test_custom_conn_pool.py +++ b/tests/datastore_aioredis/test_custom_conn_pool.py @@ -30,7 +30,7 @@ DB_SETTINGS = redis_settings()[0] -class FakeConnectionPool(object): +class FakeConnectionPool(): """Connection Pool without connection_kwargs attribute.""" def __init__(self, connection): @@ -85,7 +85,7 @@ async def execute(self, *args, **kwargs): _host = instance_hostname(DB_SETTINGS["host"]) _port = DB_SETTINGS["port"] -_instance_metric_name = "Datastore/instance/Redis/%s/%s" % (_host, _port) +_instance_metric_name = f"Datastore/instance/Redis/{_host}/{_port}" _enable_rollup_metrics.append((_instance_metric_name, 3)) diff --git a/tests/datastore_aioredis/test_execute_command.py b/tests/datastore_aioredis/test_execute_command.py index 54851a6592..b470f64b5c 100644 --- a/tests/datastore_aioredis/test_execute_command.py +++ b/tests/datastore_aioredis/test_execute_command.py @@ -13,8 +13,6 @@ # limitations under the License. import pytest - -# import aioredis from conftest import AIOREDIS_VERSION, loop # noqa # pylint: disable=E0611,W0611 from testing_support.db_settings import redis_settings from testing_support.fixtures import override_application_settings @@ -56,7 +54,7 @@ _host = instance_hostname(DB_SETTINGS["host"]) _port = DB_SETTINGS["port"] -_instance_metric_name = "Datastore/instance/Redis/%s/%s" % (_host, _port) +_instance_metric_name = f"Datastore/instance/Redis/{_host}/{_port}" _enable_rollup_metrics.append((_instance_metric_name, 1)) diff --git a/tests/datastore_aioredis/test_get_and_set.py b/tests/datastore_aioredis/test_get_and_set.py index 180f325788..3001b41800 100644 --- a/tests/datastore_aioredis/test_get_and_set.py +++ b/tests/datastore_aioredis/test_get_and_set.py @@ -57,16 +57,16 @@ _host = instance_hostname(DB_SETTINGS["host"]) _port = DB_SETTINGS["port"] -_instance_metric_name = "Datastore/instance/Redis/%s/%s" % (_host, _port) +_instance_metric_name = f"Datastore/instance/Redis/{_host}/{_port}" _enable_rollup_metrics.append((_instance_metric_name, 2)) _disable_rollup_metrics.append((_instance_metric_name, None)) -async def exercise_redis(client): - await client.set("key", "value") - await client.get("key") +async def exercise_redis(client, key): + await client.set(key, "value") + await client.get(key) @override_application_settings(_enable_instance_settings) @@ -77,8 +77,8 @@ async def exercise_redis(client): background_task=True, ) @background_task() -def test_redis_client_operation_enable_instance(client, loop): - loop.run_until_complete(exercise_redis(client)) +def test_redis_client_operation_enable_instance(client, loop, key): + loop.run_until_complete(exercise_redis(client, key)) @override_application_settings(_disable_instance_settings) @@ -89,5 +89,5 @@ def test_redis_client_operation_enable_instance(client, loop): background_task=True, ) @background_task() -def test_redis_client_operation_disable_instance(client, loop): - loop.run_until_complete(exercise_redis(client)) +def test_redis_client_operation_disable_instance(client, loop, key): + loop.run_until_complete(exercise_redis(client, key)) diff --git a/tests/datastore_aioredis/test_multiple_dbs.py b/tests/datastore_aioredis/test_multiple_dbs.py index d490c1f580..45cb067ce3 100644 --- a/tests/datastore_aioredis/test_multiple_dbs.py +++ b/tests/datastore_aioredis/test_multiple_dbs.py @@ -81,8 +81,8 @@ _host_2 = instance_hostname(redis_instance_2["host"]) _port_2 = redis_instance_2["port"] - instance_metric_name_1 = "Datastore/instance/Redis/%s/%s" % (_host_1, _port_1) - instance_metric_name_2 = "Datastore/instance/Redis/%s/%s" % (_host_2, _port_2) + instance_metric_name_1 = f"Datastore/instance/Redis/{_host_1}/{_port_1}" + instance_metric_name_2 = f"Datastore/instance/Redis/{_host_2}/{_port_2}" _enable_rollup_metrics.extend( [ @@ -125,10 +125,10 @@ def client_set(request, loop): # noqa if request.param == "Redis": return ( loop.run_until_complete( - aioredis.create_redis("redis://%s:%d" % (DB_SETTINGS[0]["host"], DB_SETTINGS[0]["port"]), db=0) + aioredis.create_redis(f"redis://{DB_SETTINGS[0]['host']}:{DB_SETTINGS[0]['port']}", db=0) ), loop.run_until_complete( - aioredis.create_redis("redis://%s:%d" % (DB_SETTINGS[1]["host"], DB_SETTINGS[1]["port"]), db=0) + aioredis.create_redis(f"redis://{DB_SETTINGS[1]['host']}:{DB_SETTINGS[1]['port']}", db=0) ), ) elif request.param == "StrictRedis": @@ -190,7 +190,7 @@ def test_concurrent_calls(client_set, loop): # noqa import asyncio async def exercise_concurrent(): - await asyncio.gather(*(client.set("key-%d" % i, i) for i, client in enumerate(client_set))) - await asyncio.gather(*(client.get("key-%d" % i) for i, client in enumerate(client_set))) + await asyncio.gather(*(client.set(f"key-{i}", i) for i, client in enumerate(client_set))) + await asyncio.gather(*(client.get(f"key-{i}") for i, client in enumerate(client_set))) loop.run_until_complete(exercise_concurrent()) diff --git a/tests/datastore_aioredis/test_span_event.py b/tests/datastore_aioredis/test_span_event.py index 1c9227e54a..7423fb9750 100644 --- a/tests/datastore_aioredis/test_span_event.py +++ b/tests/datastore_aioredis/test_span_event.py @@ -70,7 +70,7 @@ def test_span_events(client, instance_enabled, db_instance_enabled, loop): hostname = instance_hostname(DB_SETTINGS["host"]) exact_agents.update( { - "peer.address": "%s:%s" % (hostname, DB_SETTINGS["port"]), + "peer.address": f"{hostname}:{DB_SETTINGS['port']}", "peer.hostname": hostname, } ) diff --git a/tests/datastore_aioredis/test_transactions.py b/tests/datastore_aioredis/test_transactions.py index 0f84ca684e..ced9220225 100644 --- a/tests/datastore_aioredis/test_transactions.py +++ b/tests/datastore_aioredis/test_transactions.py @@ -23,42 +23,46 @@ @background_task() @pytest.mark.parametrize("in_transaction", (True, False)) -def test_pipelines_no_harm(client, in_transaction, loop): +def test_pipelines_no_harm(client, in_transaction, loop, key): async def exercise(): if AIOREDIS_VERSION >= (2,): pipe = client.pipeline(transaction=in_transaction) else: pipe = client.pipeline() # Transaction kwarg unsupported - pipe.set("TXN", 1) + pipe.set(key, 1) return await pipe.execute() status = loop.run_until_complete(exercise()) assert status == [True] -def exercise_transaction_sync(pipe): - pipe.set("TXN", 1) +def exercise_transaction_sync(key): + def _run(pipe): + pipe.set(key, 1) + return _run -async def exercise_transaction_async(pipe): - await pipe.set("TXN", 1) +def exercise_transaction_async(key): + async def _run(pipe): + await pipe.set(key, 1) + return _run @SKIPIF_AIOREDIS_V1 @pytest.mark.parametrize("exercise", (exercise_transaction_sync, exercise_transaction_async)) @background_task() -def test_transactions_no_harm(client, loop, exercise): - status = loop.run_until_complete(client.transaction(exercise)) +def test_transactions_no_harm(client, loop, key, exercise): + status = loop.run_until_complete(client.transaction(exercise(key))) assert status == [True] @SKIPIF_AIOREDIS_V2 @background_task() -def test_multi_exec_no_harm(client, loop): +def test_multi_exec_no_harm(client, loop, key): async def exercise(): pipe = client.multi_exec() - pipe.set("key", "value") + pipe.set(key, "value") status = await pipe.execute() assert status == [True] @@ -67,9 +71,7 @@ async def exercise(): @SKIPIF_AIOREDIS_V1 @background_task() -def test_pipeline_immediate_execution_no_harm(client, loop): - key = "TXN_WATCH" - +def test_pipeline_immediate_execution_no_harm(client, loop, key): async def exercise(): await client.set(key, 1) @@ -94,9 +96,7 @@ async def exercise(): @SKIPIF_AIOREDIS_V1 @background_task() -def test_transaction_immediate_execution_no_harm(client, loop): - key = "TXN_WATCH" - +def test_transaction_immediate_execution_no_harm(client, loop, key): async def exercise(): async def exercise_transaction(pipe): value = int(await pipe.get(key)) @@ -119,9 +119,7 @@ async def exercise_transaction(pipe): @SKIPIF_AIOREDIS_V1 @validate_transaction_errors([]) @background_task() -def test_transaction_watch_error_no_harm(client, loop): - key = "TXN_WATCH" - +def test_transaction_watch_error_no_harm(client, loop, key): async def exercise(): async def exercise_transaction(pipe): value = int(await pipe.get(key)) diff --git a/tests/datastore_aioredis/test_uninstrumented_methods.py b/tests/datastore_aioredis/test_uninstrumented_methods.py index 7858709c14..eeb04a996f 100644 --- a/tests/datastore_aioredis/test_uninstrumented_methods.py +++ b/tests/datastore_aioredis/test_uninstrumented_methods.py @@ -91,4 +91,4 @@ def test_uninstrumented_methods(client): is_wrapped = lambda m: hasattr(getattr(client, m), "__wrapped__") uninstrumented = {m for m in methods - IGNORED_METHODS if not is_wrapped(m)} - assert not uninstrumented, "Uninstrumented methods: %s" % sorted(uninstrumented) + assert not uninstrumented, f"Uninstrumented methods: {sorted(uninstrumented)}" diff --git a/tests/datastore_aredis/conftest.py b/tests/datastore_aredis/conftest.py index 78067e0fed..a96484ac1d 100644 --- a/tests/datastore_aredis/conftest.py +++ b/tests/datastore_aredis/conftest.py @@ -12,21 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixture.event_loop import event_loop as loop # noqa: F401 -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixture.event_loop import ( # noqa: F401; pylint: disable=W0611 + event_loop as loop, +) +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_aredis)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_aredis)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_aredis/test_custom_conn_pool.py b/tests/datastore_aredis/test_custom_conn_pool.py index 70c75de9ea..22d6b34f05 100644 --- a/tests/datastore_aredis/test_custom_conn_pool.py +++ b/tests/datastore_aredis/test_custom_conn_pool.py @@ -32,7 +32,7 @@ REDIS_PY_VERSION = aredis.VERSION -class FakeConnectionPool(object): +class FakeConnectionPool(): """Connection Pool without connection_kwargs attribute.""" def __init__(self, connection): @@ -83,7 +83,7 @@ def release(self, connection): _host = instance_hostname(DB_SETTINGS['host']) _port = DB_SETTINGS['port'] -_instance_metric_name = 'Datastore/instance/Redis/%s/%s' % (_host, _port) +_instance_metric_name = f'Datastore/instance/Redis/{_host}/{_port}' _enable_rollup_metrics.append( (_instance_metric_name, 3) diff --git a/tests/datastore_aredis/test_execute_command.py b/tests/datastore_aredis/test_execute_command.py index c5b0fc3323..e040bc57f6 100644 --- a/tests/datastore_aredis/test_execute_command.py +++ b/tests/datastore_aredis/test_execute_command.py @@ -58,7 +58,7 @@ _host = instance_hostname(DB_SETTINGS['host']) _port = DB_SETTINGS['port'] -_instance_metric_name = 'Datastore/instance/Redis/%s/%s' % (_host, _port) +_instance_metric_name = f'Datastore/instance/Redis/{_host}/{_port}' _enable_rollup_metrics.append( (_instance_metric_name, 1) diff --git a/tests/datastore_aredis/test_get_and_set.py b/tests/datastore_aredis/test_get_and_set.py index 2eeee947bc..d94777cf9c 100644 --- a/tests/datastore_aredis/test_get_and_set.py +++ b/tests/datastore_aredis/test_get_and_set.py @@ -58,7 +58,7 @@ _host = instance_hostname(DB_SETTINGS['host']) _port = DB_SETTINGS['port'] -_instance_metric_name = 'Datastore/instance/Redis/%s/%s' % (_host, _port) +_instance_metric_name = f'Datastore/instance/Redis/{_host}/{_port}' _enable_rollup_metrics.append( (_instance_metric_name, 2) diff --git a/tests/datastore_aredis/test_multiple_dbs.py b/tests/datastore_aredis/test_multiple_dbs.py index cb4cbac5b2..73d6bd8d3a 100644 --- a/tests/datastore_aredis/test_multiple_dbs.py +++ b/tests/datastore_aredis/test_multiple_dbs.py @@ -80,8 +80,8 @@ host_2 = instance_hostname(redis_2["host"]) port_2 = redis_2["port"] - instance_metric_name_1 = "Datastore/instance/Redis/%s/%s" % (host_1, port_1) - instance_metric_name_2 = "Datastore/instance/Redis/%s/%s" % (host_2, port_2) + instance_metric_name_1 = f"Datastore/instance/Redis/{host_1}/{port_1}" + instance_metric_name_2 = f"Datastore/instance/Redis/{host_2}/{port_2}" _enable_rollup_metrics.extend( [ @@ -172,7 +172,7 @@ def test_concurrent_calls(loop): clients = (client_1, client_2) async def exercise_concurrent(): - await asyncio.gather(*(client.set("key-%d" % i, i) for i, client in enumerate(clients))) - await asyncio.gather(*(client.get("key-%d" % i) for i, client in enumerate(clients))) + await asyncio.gather(*(client.set(f"key-{i}", i) for i, client in enumerate(clients))) + await asyncio.gather(*(client.get(f"key-{i}") for i, client in enumerate(clients))) loop.run_until_complete(exercise_concurrent()) diff --git a/tests/datastore_aredis/test_span_event.py b/tests/datastore_aredis/test_span_event.py index 2bd238bdae..db4a8a897b 100644 --- a/tests/datastore_aredis/test_span_event.py +++ b/tests/datastore_aredis/test_span_event.py @@ -79,7 +79,7 @@ def test_span_events(instance_enabled, db_instance_enabled, loop): settings = _enable_instance_settings.copy() hostname = instance_hostname(DB_SETTINGS['host']) exact_agents.update({ - 'peer.address': '%s:%s' % (hostname, DB_SETTINGS['port']), + 'peer.address': f"{hostname}:{DB_SETTINGS['port']}", 'peer.hostname': hostname, }) else: diff --git a/tests/datastore_aredis/test_uninstrumented_methods.py b/tests/datastore_aredis/test_uninstrumented_methods.py index 38901e5c5d..e4b9c90042 100644 --- a/tests/datastore_aredis/test_uninstrumented_methods.py +++ b/tests/datastore_aredis/test_uninstrumented_methods.py @@ -45,4 +45,4 @@ def test_uninstrumented_methods(): is_wrapped = lambda m: hasattr(getattr(strict_redis_client, m), "__wrapped__") uninstrumented = {m for m in methods - IGNORED_METHODS if not is_wrapped(m)} - assert not uninstrumented, "Uninstrumented methods: %s" % sorted(uninstrumented) + assert not uninstrumented, f"Uninstrumented methods: {sorted(uninstrumented)}" diff --git a/tests/datastore_asyncpg/conftest.py b/tests/datastore_asyncpg/conftest.py index 69bc0501a2..783e9b8462 100644 --- a/tests/datastore_asyncpg/conftest.py +++ b/tests/datastore_asyncpg/conftest.py @@ -12,11 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -from testing_support.fixture.event_loop import event_loop - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixture.event_loop import ( # noqa: F401; pylint: disable=W0611 + event_loop, +) +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, diff --git a/tests/datastore_asyncpg/test_multiple_dbs.py b/tests/datastore_asyncpg/test_multiple_dbs.py index a917a9e83d..afc6324fe2 100644 --- a/tests/datastore_asyncpg/test_multiple_dbs.py +++ b/tests/datastore_asyncpg/test_multiple_dbs.py @@ -12,20 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio - import asyncpg import pytest from testing_support.db_settings import postgresql_settings from testing_support.fixtures import override_application_settings -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple DB_MULTIPLE_SETTINGS = postgresql_settings() -ASYNCPG_VERSION = tuple(int(x) for x in getattr(asyncpg, "__version__", "0.0").split(".")[:2]) +ASYNCPG_VERSION = get_package_version_tuple("asyncpg") if ASYNCPG_VERSION < (0, 11): CONNECT_METRICS = [] @@ -84,8 +85,8 @@ _host_2 = instance_hostname(_postgresql_2["host"]) _port_2 = _postgresql_2["port"] - _instance_metric_name_1 = "Datastore/instance/Postgres/%s/%s" % (_host_1, _port_1) - _instance_metric_name_2 = "Datastore/instance/Postgres/%s/%s" % (_host_2, _port_2) + _instance_metric_name_1 = f"Datastore/instance/Postgres/{_host_1}/{_port_1}" + _instance_metric_name_2 = f"Datastore/instance/Postgres/{_host_2}/{_port_2}" _enable_rollup_metrics.extend( [ @@ -100,7 +101,6 @@ async def _exercise_db(): - postgresql1 = DB_MULTIPLE_SETTINGS[0] postgresql2 = DB_MULTIPLE_SETTINGS[1] @@ -145,6 +145,7 @@ async def _exercise_db(): ) @background_task() def test_multiple_databases_enable_instance(event_loop): + assert ASYNCPG_VERSION is not None event_loop.run_until_complete(_exercise_db()) @@ -161,4 +162,5 @@ def test_multiple_databases_enable_instance(event_loop): ) @background_task() def test_multiple_databases_disable_instance(event_loop): + assert ASYNCPG_VERSION is not None event_loop.run_until_complete(_exercise_db()) diff --git a/tests/datastore_asyncpg/test_query.py b/tests/datastore_asyncpg/test_query.py index 838ced61da..bccafbdfd5 100644 --- a/tests/datastore_asyncpg/test_query.py +++ b/tests/datastore_asyncpg/test_query.py @@ -27,17 +27,18 @@ ) from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple DB_SETTINGS = postgresql_settings()[0] PG_PREFIX = "Datastore/operation/Postgres/" -ASYNCPG_VERSION = tuple(int(x) for x in getattr(asyncpg, "__version__", "0.0").split(".")[:2]) +ASYNCPG_VERSION = get_package_version_tuple("asyncpg") if ASYNCPG_VERSION < (0, 11): CONNECT_METRICS = () else: - CONNECT_METRICS = ((PG_PREFIX + "connect", 1),) + CONNECT_METRICS = ((f"{PG_PREFIX}connect", 1),) @pytest.fixture @@ -58,13 +59,14 @@ def conn(event_loop): @validate_transaction_metrics( "test_single", background_task=True, - scoped_metrics=((PG_PREFIX + "select", 1),), + scoped_metrics=((f"{PG_PREFIX}select", 1),), rollup_metrics=(("Datastore/all", 1),), ) @validate_tt_collector_json(datastore_params={"port_path_or_id": str(DB_SETTINGS["port"])}) @background_task(name="test_single") @pytest.mark.parametrize("method", ("execute",)) def test_single(event_loop, method, conn): + assert ASYNCPG_VERSION is not None _method = getattr(conn, method) event_loop.run_until_complete(_method("""SELECT 0""")) @@ -73,14 +75,15 @@ def test_single(event_loop, method, conn): "test_prepared_single", background_task=True, scoped_metrics=( - (PG_PREFIX + "prepare", 1), - (PG_PREFIX + "select", 1), + (f"{PG_PREFIX}prepare", 1), + (f"{PG_PREFIX}select", 1), ), rollup_metrics=(("Datastore/all", 2),), ) @background_task(name="test_prepared_single") @pytest.mark.parametrize("method", ("fetch", "fetchrow", "fetchval")) def test_prepared_single(event_loop, method, conn): + assert ASYNCPG_VERSION is not None _method = getattr(conn, method) event_loop.run_until_complete(_method("""SELECT 0""")) @@ -88,19 +91,20 @@ def test_prepared_single(event_loop, method, conn): @validate_transaction_metrics( "test_prepare", background_task=True, - scoped_metrics=((PG_PREFIX + "prepare", 1),), + scoped_metrics=((f"{PG_PREFIX}prepare", 1),), rollup_metrics=(("Datastore/all", 1),), ) @background_task(name="test_prepare") def test_prepare(event_loop, conn): + assert ASYNCPG_VERSION is not None event_loop.run_until_complete(conn.prepare("""SELECT 0""")) @pytest.fixture def table(event_loop, conn): - table_name = "table_%d" % os.getpid() + table_name = f"table_{os.getpid()}" - event_loop.run_until_complete(conn.execute("""create table %s (a integer, b real, c text)""" % table_name)) + event_loop.run_until_complete(conn.execute(f"""create table {table_name} (a integer, b real, c text)""")) return table_name @@ -110,8 +114,8 @@ def table(event_loop, conn): "test_copy", background_task=True, scoped_metrics=( - (PG_PREFIX + "prepare", 1), - (PG_PREFIX + "copy", 3), + (f"{PG_PREFIX}prepare", 1), + (f"{PG_PREFIX}copy", 3), ), rollup_metrics=(("Datastore/all", 4),), ) @@ -125,6 +129,7 @@ async def amain(): # 2 statements await conn.copy_from_query("""SELECT 0""", output=BytesIO()) + assert ASYNCPG_VERSION is not None event_loop.run_until_complete(amain()) @@ -132,13 +137,14 @@ async def amain(): "test_select_many", background_task=True, scoped_metrics=( - (PG_PREFIX + "prepare", 1), - (PG_PREFIX + "select", 1), + (f"{PG_PREFIX}prepare", 1), + (f"{PG_PREFIX}select", 1), ), rollup_metrics=(("Datastore/all", 2),), ) @background_task(name="test_select_many") def test_select_many(event_loop, conn): + assert ASYNCPG_VERSION is not None event_loop.run_until_complete(conn.executemany("""SELECT $1::int""", ((1,), (2,)))) @@ -146,9 +152,9 @@ def test_select_many(event_loop, conn): "test_transaction", background_task=True, scoped_metrics=( - (PG_PREFIX + "begin", 1), - (PG_PREFIX + "select", 1), - (PG_PREFIX + "commit", 1), + (f"{PG_PREFIX}begin", 1), + (f"{PG_PREFIX}select", 1), + (f"{PG_PREFIX}commit", 1), ), rollup_metrics=(("Datastore/all", 3),), ) @@ -158,6 +164,7 @@ async def amain(): async with conn.transaction(): await conn.execute("""SELECT 0""") + assert ASYNCPG_VERSION is not None event_loop.run_until_complete(amain()) @@ -165,10 +172,10 @@ async def amain(): "test_cursor", background_task=True, scoped_metrics=( - (PG_PREFIX + "begin", 1), - (PG_PREFIX + "prepare", 2), - (PG_PREFIX + "select", 3), - (PG_PREFIX + "commit", 1), + (f"{PG_PREFIX}begin", 1), + (f"{PG_PREFIX}prepare", 2), + (f"{PG_PREFIX}select", 3), + (f"{PG_PREFIX}commit", 1), ), rollup_metrics=(("Datastore/all", 7),), ) @@ -181,6 +188,7 @@ async def amain(): await conn.cursor("SELECT 0") + assert ASYNCPG_VERSION is not None event_loop.run_until_complete(amain()) @@ -193,13 +201,14 @@ async def amain(): background_task=True, rollup_metrics=[ ( - "Datastore/instance/Postgres/" + instance_hostname("localhost") + "//.s.PGSQL.THIS_FILE_BETTER_NOT_EXIST", + f"Datastore/instance/Postgres/{instance_hostname('localhost')}//.s.PGSQL.THIS_FILE_BETTER_NOT_EXIST", 1, ) ], ) @background_task(name="test_unix_socket_connect") def test_unix_socket_connect(event_loop): + assert ASYNCPG_VERSION is not None with pytest.raises(OSError): event_loop.run_until_complete(asyncpg.connect("postgres://?host=/.s.PGSQL.THIS_FILE_BETTER_NOT_EXIST")) @@ -211,7 +220,7 @@ def test_unix_socket_connect(event_loop): @validate_transaction_metrics( "test_pool_acquire", background_task=True, - scoped_metrics=((PG_PREFIX + "connect", 2),), + scoped_metrics=((f"{PG_PREFIX}connect", 2),), ) @background_task(name="test_pool_acquire") def test_pool_acquire(event_loop): @@ -233,4 +242,5 @@ async def amain(): finally: await pool.close() + assert ASYNCPG_VERSION is not None event_loop.run_until_complete(amain()) diff --git a/tests/datastore_bmemcached/conftest.py b/tests/datastore_bmemcached/conftest.py index c970c1c347..91149e18a5 100644 --- a/tests/datastore_bmemcached/conftest.py +++ b/tests/datastore_bmemcached/conftest.py @@ -12,20 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_bmemcached)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_bmemcached)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_bmemcached/test_memcache.py b/tests/datastore_bmemcached/test_memcache.py index 68eee06333..94b8a04810 100644 --- a/tests/datastore_bmemcached/test_memcache.py +++ b/tests/datastore_bmemcached/test_memcache.py @@ -13,83 +13,97 @@ # limitations under the License. import os -from testing_support.db_settings import memcached_settings + import bmemcached +from testing_support.db_settings import memcached_settings +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.background_task import background_task from newrelic.api.transaction import set_background_task - -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.db_settings import memcached_settings - +from newrelic.common import system_info DB_SETTINGS = memcached_settings()[0] -MEMCACHED_HOST = DB_SETTINGS['host'] -MEMCACHED_PORT = DB_SETTINGS['port'] +MEMCACHED_HOST = DB_SETTINGS["host"] +MEMCACHED_PORT = DB_SETTINGS["port"] MEMCACHED_NAMESPACE = str(os.getpid()) -MEMCACHED_ADDR = '%s:%s' % (MEMCACHED_HOST, MEMCACHED_PORT) +MEMCACHED_ADDR = f"{MEMCACHED_HOST}:{MEMCACHED_PORT}" +INSTANCE_METRIC_HOST = system_info.gethostname() if MEMCACHED_HOST == "127.0.0.1" else MEMCACHED_HOST +INSTANCE_METRIC_NAME = f"Datastore/instance/Memcached/{INSTANCE_METRIC_HOST}/{MEMCACHED_PORT}" _test_bt_set_get_delete_scoped_metrics = [ - ('Datastore/operation/Memcached/set', 1), - ('Datastore/operation/Memcached/get', 1), - ('Datastore/operation/Memcached/delete', 1)] + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] _test_bt_set_get_delete_rollup_metrics = [ - ('Datastore/all', 3), - ('Datastore/allOther', 3), - ('Datastore/Memcached/all', 3), - ('Datastore/Memcached/allOther', 3), - ('Datastore/operation/Memcached/set', 1), - ('Datastore/operation/Memcached/get', 1), - ('Datastore/operation/Memcached/delete', 1)] + ("Datastore/all", 3), + ("Datastore/allOther", 3), + ("Datastore/Memcached/all", 3), + ("Datastore/Memcached/allOther", 3), + (INSTANCE_METRIC_NAME, 3), + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] + @validate_transaction_metrics( - 'test_memcache:test_bt_set_get_delete', - scoped_metrics=_test_bt_set_get_delete_scoped_metrics, - rollup_metrics=_test_bt_set_get_delete_rollup_metrics, - background_task=True) + "test_memcache:test_bt_set_get_delete", + scoped_metrics=_test_bt_set_get_delete_scoped_metrics, + rollup_metrics=_test_bt_set_get_delete_rollup_metrics, + background_task=True, +) @background_task() def test_bt_set_get_delete(): set_background_task(True) client = bmemcached.Client([MEMCACHED_ADDR]) - key = MEMCACHED_NAMESPACE + 'key' + key = f"{MEMCACHED_NAMESPACE}key" - client.set(key, 'value') + client.set(key, "value") value = client.get(key) client.delete(key) - assert value == 'value' + assert value == "value" + _test_wt_set_get_delete_scoped_metrics = [ - ('Datastore/operation/Memcached/set', 1), - ('Datastore/operation/Memcached/get', 1), - ('Datastore/operation/Memcached/delete', 1)] + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] _test_wt_set_get_delete_rollup_metrics = [ - ('Datastore/all', 3), - ('Datastore/allWeb', 3), - ('Datastore/Memcached/all', 3), - ('Datastore/Memcached/allWeb', 3), - ('Datastore/operation/Memcached/set', 1), - ('Datastore/operation/Memcached/get', 1), - ('Datastore/operation/Memcached/delete', 1)] + ("Datastore/all", 3), + ("Datastore/allWeb", 3), + ("Datastore/Memcached/all", 3), + ("Datastore/Memcached/allWeb", 3), + (INSTANCE_METRIC_NAME, 3), + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] + @validate_transaction_metrics( - 'test_memcache:test_wt_set_get_delete', - scoped_metrics=_test_wt_set_get_delete_scoped_metrics, - rollup_metrics=_test_wt_set_get_delete_rollup_metrics, - background_task=False) + "test_memcache:test_wt_set_get_delete", + scoped_metrics=_test_wt_set_get_delete_scoped_metrics, + rollup_metrics=_test_wt_set_get_delete_rollup_metrics, + background_task=False, +) @background_task() def test_wt_set_get_delete(): set_background_task(False) client = bmemcached.Client([MEMCACHED_ADDR]) - key = MEMCACHED_NAMESPACE + 'key' + key = f"{MEMCACHED_NAMESPACE}key" - client.set(key, 'value') + client.set(key, "value") value = client.get(key) client.delete(key) - assert value == 'value' + assert value == "value" diff --git a/tests/datastore_elasticsearch/conftest.py b/tests/datastore_elasticsearch/conftest.py index 53fa6fcdc3..e70dde884f 100644 --- a/tests/datastore_elasticsearch/conftest.py +++ b/tests/datastore_elasticsearch/conftest.py @@ -14,13 +14,15 @@ import pytest from testing_support.db_settings import elasticsearch_settings - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) from newrelic.common.package_version_utils import get_package_version - _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, @@ -37,7 +39,7 @@ ES_VERSION = tuple([int(n) for n in get_package_version("elasticsearch").split(".")]) ES_SETTINGS = elasticsearch_settings()[0] ES_MULTIPLE_SETTINGS = elasticsearch_settings() -ES_URL = "http://%s:%s" % (ES_SETTINGS["host"], ES_SETTINGS["port"]) +ES_URL = f"http://{ES_SETTINGS['host']}:{ES_SETTINGS['port']}" @pytest.fixture(scope="session") diff --git a/tests/datastore_elasticsearch/test_connection.py b/tests/datastore_elasticsearch/test_connection.py index 2e888af9b5..9e8f17b4c1 100644 --- a/tests/datastore_elasticsearch/test_connection.py +++ b/tests/datastore_elasticsearch/test_connection.py @@ -36,7 +36,7 @@ def test_connection_default(): else: conn = Connection(**HOST) - assert conn._nr_host_port == ("localhost", ES_SETTINGS["port"]) + assert conn._nr_host_port == (ES_SETTINGS["host"], ES_SETTINGS["port"]) @SKIP_IF_V7 diff --git a/tests/datastore_elasticsearch/test_elasticsearch.py b/tests/datastore_elasticsearch/test_elasticsearch.py index d2c892ea92..294118192a 100644 --- a/tests/datastore_elasticsearch/test_elasticsearch.py +++ b/tests/datastore_elasticsearch/test_elasticsearch.py @@ -138,7 +138,7 @@ def is_importable(module_path): _host = instance_hostname(ES_SETTINGS["host"]) _port = ES_SETTINGS["port"] -_instance_metric_name = "Datastore/instance/Elasticsearch/%s/%s" % (_host, _port) +_instance_metric_name = f"Datastore/instance/Elasticsearch/{_host}/{_port}" _enable_rollup_metrics.append((_instance_metric_name, _all_count)) diff --git a/tests/datastore_elasticsearch/test_instrumented_methods.py b/tests/datastore_elasticsearch/test_instrumented_methods.py index 4ad88c2a58..7c38bcaa8b 100644 --- a/tests/datastore_elasticsearch/test_instrumented_methods.py +++ b/tests/datastore_elasticsearch/test_instrumented_methods.py @@ -71,7 +71,7 @@ def client(client): ], ) def test_method_on_client_datastore_trace_inputs(client, sub_module, method, args, kwargs, expected_index): - expected_operation = "%s.%s" % (sub_module, method) if sub_module else method + expected_operation = f"{sub_module}.{method}" if sub_module else method @validate_datastore_trace_inputs(target=expected_index, operation=expected_operation) @background_task() @@ -93,7 +93,7 @@ def is_wrapped(m): methods = {m for m in dir(_object) if not m[0] == "_"} uninstrumented = {m for m in (methods - ignored_methods) if not is_wrapped(m)} - assert not uninstrumented, "There are uninstrumented methods: %s" % uninstrumented + assert not uninstrumented, f"There are uninstrumented methods: {uninstrumented}" @RUN_IF_V8 diff --git a/tests/datastore_elasticsearch/test_mget.py b/tests/datastore_elasticsearch/test_mget.py index f3f7c09790..5058146fe6 100644 --- a/tests/datastore_elasticsearch/test_mget.py +++ b/tests/datastore_elasticsearch/test_mget.py @@ -68,8 +68,8 @@ host_2 = instance_hostname(es_2["host"]) port_2 = es_2["port"] - instance_metric_name_1 = "Datastore/instance/Elasticsearch/%s/%s" % (host_1, port_1) - instance_metric_name_2 = "Datastore/instance/Elasticsearch/%s/%s" % (host_2, port_2) + instance_metric_name_1 = f"Datastore/instance/Elasticsearch/{host_1}/{port_1}" + instance_metric_name_2 = f"Datastore/instance/Elasticsearch/{host_2}/{port_2}" _enable_rollup_metrics.extend( [ @@ -88,7 +88,7 @@ @pytest.fixture(scope="module") def client(): - urls = ["http://%s:%s" % (db["host"], db["port"]) for db in ES_MULTIPLE_SETTINGS] + urls = [f"http://{db['host']}:{db['port']}" for db in ES_MULTIPLE_SETTINGS] # When selecting a connection from the pool, use the round robin method. # This is actually the default already. Using round robin will ensure that # doing two db calls will mean elastic search is talking to two different diff --git a/tests/datastore_elasticsearch/test_multiple_dbs.py b/tests/datastore_elasticsearch/test_multiple_dbs.py index 71c47b1685..b427c90a12 100644 --- a/tests/datastore_elasticsearch/test_multiple_dbs.py +++ b/tests/datastore_elasticsearch/test_multiple_dbs.py @@ -61,8 +61,8 @@ host_2 = instance_hostname(es_2["host"]) port_2 = es_2["port"] - instance_metric_name_1 = "Datastore/instance/Elasticsearch/%s/%s" % (host_1, port_1) - instance_metric_name_2 = "Datastore/instance/Elasticsearch/%s/%s" % (host_2, port_2) + instance_metric_name_1 = f"Datastore/instance/Elasticsearch/{host_1}/{port_1}" + instance_metric_name_2 = f"Datastore/instance/Elasticsearch/{host_2}/{port_2}" _enable_rollup_metrics.extend( [ @@ -104,7 +104,7 @@ def _exercise_es(es): @background_task() def test_multiple_dbs_enabled(): for db in ES_MULTIPLE_SETTINGS: - es_url = "http://%s:%s" % (db["host"], db["port"]) + es_url = f"http://{db['host']}:{db['port']}" client = Elasticsearch(es_url) _exercise_es(client) @@ -120,6 +120,6 @@ def test_multiple_dbs_enabled(): @background_task() def test_multiple_dbs_disabled(): for db in ES_MULTIPLE_SETTINGS: - es_url = "http://%s:%s" % (db["host"], db["port"]) + es_url = f"http://{db['host']}:{db['port']}" client = Elasticsearch(es_url) _exercise_es(client) diff --git a/tests/datastore_firestore/conftest.py b/tests/datastore_firestore/conftest.py new file mode 100644 index 0000000000..6fd1550753 --- /dev/null +++ b/tests/datastore_firestore/conftest.py @@ -0,0 +1,128 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import uuid + +import pytest +from google.cloud.firestore import AsyncClient, Client +from testing_support.db_settings import firestore_settings +from testing_support.fixture.event_loop import ( # noqa: F401; pylint: disable=W0611 + event_loop as loop, +) +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) + +from newrelic.api.datastore_trace import DatastoreTrace +from newrelic.api.time_trace import current_trace +from newrelic.common.system_info import LOCALHOST_EQUIVALENTS, gethostname + +DB_SETTINGS = firestore_settings()[0] +FIRESTORE_HOST = DB_SETTINGS["host"] +FIRESTORE_PORT = DB_SETTINGS["port"] + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_explain_plan_queries": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (datastore_firestore)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) + + +@pytest.fixture() +def instance_info(): + host = gethostname() if FIRESTORE_HOST in LOCALHOST_EQUIVALENTS else FIRESTORE_HOST + return { + "host": host, + "port_path_or_id": str(FIRESTORE_PORT), + "db.instance": "projects/google-cloud-firestore-emulator/databases/(default)", + } + + +@pytest.fixture(scope="session") +def client(): + os.environ["FIRESTORE_EMULATOR_HOST"] = f"{FIRESTORE_HOST}:{FIRESTORE_PORT}" + client = Client() + # Ensure connection is available + client.collection("healthcheck").document("healthcheck").set({}, retry=None, timeout=5) + return client + + +@pytest.fixture(scope="function") +def collection(client): + collection_ = client.collection(f"firestore_collection_{str(uuid.uuid4())}") + yield collection_ + client.recursive_delete(collection_) + + +@pytest.fixture(scope="session") +def async_client(loop): + os.environ["FIRESTORE_EMULATOR_HOST"] = f"{FIRESTORE_HOST}:{FIRESTORE_PORT}" + client = AsyncClient() + loop.run_until_complete( + client.collection("healthcheck").document("healthcheck").set({}, retry=None, timeout=5) + ) # Ensure connection is available + return client + + +@pytest.fixture(scope="function") +def async_collection(async_client, collection): + # Use the same collection name as the collection fixture + yield async_client.collection(collection.id) + + +@pytest.fixture(scope="session") +def assert_trace_for_generator(): + def _assert_trace_for_generator(generator_func, *args, **kwargs): + txn = current_trace() + assert not isinstance(txn, DatastoreTrace) + + # Check for generator trace on collections + _trace_check = [] + for _ in generator_func(*args, **kwargs): + _trace_check.append(isinstance(current_trace(), DatastoreTrace)) + assert _trace_check and all(_trace_check) # All checks are True, and at least 1 is present. + assert current_trace() is txn # Generator trace has exited. + + return _assert_trace_for_generator + + +@pytest.fixture(scope="session") +def assert_trace_for_async_generator(loop): + def _assert_trace_for_async_generator(generator_func, *args, **kwargs): + _trace_check = [] + txn = current_trace() + assert not isinstance(txn, DatastoreTrace) + + async def coro(): + # Check for generator trace on collections + async for _ in generator_func(*args, **kwargs): + _trace_check.append(isinstance(current_trace(), DatastoreTrace)) + + loop.run_until_complete(coro()) + + assert _trace_check and all(_trace_check) # All checks are True, and at least 1 is present. + assert current_trace() is txn # Generator trace has exited. + + return _assert_trace_for_async_generator diff --git a/tests/datastore_firestore/test_async_batching.py b/tests/datastore_firestore/test_async_batching.py new file mode 100644 index 0000000000..5e6fbd3c7d --- /dev/null +++ b/tests/datastore_firestore/test_async_batching.py @@ -0,0 +1,73 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_database_duration import ( + validate_database_duration, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_collector_json import ( + validate_tt_collector_json, +) + +from newrelic.api.background_task import background_task + + +@pytest.fixture() +def exercise_async_write_batch(async_client, async_collection): + async def _exercise_async_write_batch(): + docs = [async_collection.document(str(x)) for x in range(1, 4)] + async_batch = async_client.batch() + for doc in docs: + async_batch.set(doc, {}) + + await async_batch.commit() + + return _exercise_async_write_batch + + +def test_firestore_async_write_batch(loop, exercise_async_write_batch, instance_info): + _test_scoped_metrics = [ + ("Datastore/operation/Firestore/commit", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/all", 1), + ("Datastore/allOther", 1), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 1), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_async_write_batch", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_async_write_batch") + def _test(): + loop.run_until_complete(exercise_async_write_batch()) + + _test() + + +def test_firestore_async_write_batch_trace_node_datastore_params(loop, exercise_async_write_batch, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + loop.run_until_complete(exercise_async_write_batch()) + + _test() \ No newline at end of file diff --git a/tests/datastore_firestore/test_async_client.py b/tests/datastore_firestore/test_async_client.py new file mode 100644 index 0000000000..236d9c2161 --- /dev/null +++ b/tests/datastore_firestore/test_async_client.py @@ -0,0 +1,87 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_database_duration import ( + validate_database_duration, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_collector_json import ( + validate_tt_collector_json, +) + +from newrelic.api.background_task import background_task + + +@pytest.fixture() +def existing_document(collection): + doc = collection.document("document") + doc.set({"x": 1}) + return doc + + +@pytest.fixture() +def exercise_async_client(async_client, existing_document): + async def _exercise_async_client(): + assert len([_ async for _ in async_client.collections()]) >= 1 + doc = [_ async for _ in async_client.get_all([existing_document])][0] + assert doc.to_dict()["x"] == 1 + + return _exercise_async_client + + +def test_firestore_async_client(loop, exercise_async_client, instance_info): + _test_scoped_metrics = [ + ("Datastore/operation/Firestore/collections", 1), + ("Datastore/operation/Firestore/get_all", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/all", 2), + ("Datastore/allOther", 2), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 2), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_async_client", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_async_client") + def _test(): + loop.run_until_complete(exercise_async_client()) + + _test() + + +@background_task() +def test_firestore_async_client_generators(async_client, collection, assert_trace_for_async_generator): + doc = collection.document("test") + doc.set({}) + + assert_trace_for_async_generator(async_client.collections) + assert_trace_for_async_generator(async_client.get_all, [doc]) + + +def test_firestore_async_client_trace_node_datastore_params(loop, exercise_async_client, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + loop.run_until_complete(exercise_async_client()) + + _test() \ No newline at end of file diff --git a/tests/datastore_firestore/test_async_collections.py b/tests/datastore_firestore/test_async_collections.py new file mode 100644 index 0000000000..c1658d18b3 --- /dev/null +++ b/tests/datastore_firestore/test_async_collections.py @@ -0,0 +1,94 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_database_duration import ( + validate_database_duration, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_collector_json import ( + validate_tt_collector_json, +) + +from newrelic.api.background_task import background_task + + +@pytest.fixture() +def exercise_async_collections(async_collection): + async def _exercise_async_collections(): + async_collection.document("DoesNotExist") + await async_collection.add({"capital": "Rome", "currency": "Euro", "language": "Italian"}, "Italy") + await async_collection.add({"capital": "Mexico City", "currency": "Peso", "language": "Spanish"}, "Mexico") + + documents_get = await async_collection.get() + assert len(documents_get) == 2 + documents_stream = [_ async for _ in async_collection.stream()] + assert len(documents_stream) == 2 + documents_list = [_ async for _ in async_collection.list_documents()] + assert len(documents_list) == 2 + + return _exercise_async_collections + + +def test_firestore_async_collections(loop, exercise_async_collections, async_collection, instance_info): + _test_scoped_metrics = [ + (f"Datastore/statement/Firestore/{async_collection.id}/stream", 1), + (f"Datastore/statement/Firestore/{async_collection.id}/get", 1), + (f"Datastore/statement/Firestore/{async_collection.id}/list_documents", 1), + (f"Datastore/statement/Firestore/{async_collection.id}/add", 2), + ] + + _test_rollup_metrics = [ + ("Datastore/operation/Firestore/add", 2), + ("Datastore/operation/Firestore/get", 1), + ("Datastore/operation/Firestore/stream", 1), + ("Datastore/operation/Firestore/list_documents", 1), + ("Datastore/all", 5), + ("Datastore/allOther", 5), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 5), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_async_collections", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_async_collections") + def _test(): + loop.run_until_complete(exercise_async_collections()) + + _test() + + +@background_task() +def test_firestore_async_collections_generators(collection, async_collection, assert_trace_for_async_generator): + collection.add({}) + collection.add({}) + assert len([_ for _ in collection.list_documents()]) == 2 + + assert_trace_for_async_generator(async_collection.stream) + assert_trace_for_async_generator(async_collection.list_documents) + + +def test_firestore_async_collections_trace_node_datastore_params(loop, exercise_async_collections, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + loop.run_until_complete(exercise_async_collections()) + + _test() \ No newline at end of file diff --git a/tests/datastore_firestore/test_async_documents.py b/tests/datastore_firestore/test_async_documents.py new file mode 100644 index 0000000000..2a0d5e9b81 --- /dev/null +++ b/tests/datastore_firestore/test_async_documents.py @@ -0,0 +1,108 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_database_duration import ( + validate_database_duration, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_collector_json import ( + validate_tt_collector_json, +) + +from newrelic.api.background_task import background_task + + +@pytest.fixture() +def exercise_async_documents(async_collection): + async def _exercise_async_documents(): + italy_doc = async_collection.document("Italy") + await italy_doc.set({"capital": "Rome", "currency": "Euro", "language": "Italian"}) + await italy_doc.get() + italian_cities = italy_doc.collection("cities") + await italian_cities.add({"capital": "Rome"}) + retrieved_coll = [_ async for _ in italy_doc.collections()] + assert len(retrieved_coll) == 1 + + usa_doc = async_collection.document("USA") + await usa_doc.create({"capital": "Washington D.C.", "currency": "Dollar", "language": "English"}) + await usa_doc.update({"president": "Joe Biden"}) + + await async_collection.document("USA").delete() + + return _exercise_async_documents + + +def test_firestore_async_documents(loop, exercise_async_documents, instance_info): + _test_scoped_metrics = [ + ("Datastore/statement/Firestore/Italy/set", 1), + ("Datastore/statement/Firestore/Italy/get", 1), + ("Datastore/statement/Firestore/Italy/collections", 1), + ("Datastore/statement/Firestore/cities/add", 1), + ("Datastore/statement/Firestore/USA/create", 1), + ("Datastore/statement/Firestore/USA/update", 1), + ("Datastore/statement/Firestore/USA/delete", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/operation/Firestore/set", 1), + ("Datastore/operation/Firestore/get", 1), + ("Datastore/operation/Firestore/add", 1), + ("Datastore/operation/Firestore/collections", 1), + ("Datastore/operation/Firestore/create", 1), + ("Datastore/operation/Firestore/update", 1), + ("Datastore/operation/Firestore/delete", 1), + ("Datastore/all", 7), + ("Datastore/allOther", 7), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 7), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_async_documents", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_async_documents") + def _test(): + loop.run_until_complete(exercise_async_documents()) + + _test() + + +@background_task() +def test_firestore_async_documents_generators( + collection, async_collection, assert_trace_for_async_generator, instance_info +): + subcollection_doc = collection.document("SubCollections") + subcollection_doc.set({}) + subcollection_doc.collection("collection1").add({}) + subcollection_doc.collection("collection2").add({}) + assert len([_ for _ in subcollection_doc.collections()]) == 2 + + async_subcollection = async_collection.document(subcollection_doc.id) + + assert_trace_for_async_generator(async_subcollection.collections) + + +def test_firestore_async_documents_trace_node_datastore_params(loop, exercise_async_documents, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + loop.run_until_complete(exercise_async_documents()) + + _test() \ No newline at end of file diff --git a/tests/datastore_firestore/test_async_query.py b/tests/datastore_firestore/test_async_query.py new file mode 100644 index 0000000000..4d0267e90c --- /dev/null +++ b/tests/datastore_firestore/test_async_query.py @@ -0,0 +1,249 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_database_duration import ( + validate_database_duration, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_collector_json import ( + validate_tt_collector_json, +) + +from newrelic.api.background_task import background_task + + +@pytest.fixture(autouse=True) +def sample_data(collection): + for x in range(1, 6): + collection.add({"x": x}) + + subcollection_doc = collection.document("subcollection") + subcollection_doc.set({}) + subcollection_doc.collection("subcollection1").add({}) + + +# ===== AsyncQuery ===== + + +@pytest.fixture() +def exercise_async_query(async_collection): + async def _exercise_async_query(): + async_query = ( + async_collection.select("x").limit(10).order_by("x").where(field_path="x", op_string="<=", value=3) + ) + assert len(await async_query.get()) == 3 + assert len([_ async for _ in async_query.stream()]) == 3 + + return _exercise_async_query + + +def test_firestore_async_query(loop, exercise_async_query, async_collection, instance_info): + _test_scoped_metrics = [ + (f"Datastore/statement/Firestore/{async_collection.id}/stream", 1), + (f"Datastore/statement/Firestore/{async_collection.id}/get", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/operation/Firestore/get", 1), + ("Datastore/operation/Firestore/stream", 1), + ("Datastore/all", 2), + ("Datastore/allOther", 2), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 2), + ] + + # @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_async_query", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_async_query") + def _test(): + loop.run_until_complete(exercise_async_query()) + + _test() + + +@background_task() +def test_firestore_async_query_generators(async_collection, assert_trace_for_async_generator): + async_query = async_collection.select("x").where(field_path="x", op_string="<=", value=3) + assert_trace_for_async_generator(async_query.stream) + + +def test_firestore_async_query_trace_node_datastore_params(loop, exercise_async_query, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + loop.run_until_complete(exercise_async_query()) + + _test() + + +# ===== AsyncAggregationQuery ===== + + +@pytest.fixture() +def exercise_async_aggregation_query(async_collection): + async def _exercise_async_aggregation_query(): + async_aggregation_query = async_collection.select("x").where(field_path="x", op_string="<=", value=3).count() + assert (await async_aggregation_query.get())[0][0].value == 3 + assert [_ async for _ in async_aggregation_query.stream()][0][0].value == 3 + + return _exercise_async_aggregation_query + + +def test_firestore_async_aggregation_query(loop, exercise_async_aggregation_query, async_collection, instance_info): + _test_scoped_metrics = [ + (f"Datastore/statement/Firestore/{async_collection.id}/stream", 1), + (f"Datastore/statement/Firestore/{async_collection.id}/get", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/operation/Firestore/get", 1), + ("Datastore/operation/Firestore/stream", 1), + ("Datastore/all", 2), + ("Datastore/allOther", 2), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 2), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_async_aggregation_query", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_async_aggregation_query") + def _test(): + loop.run_until_complete(exercise_async_aggregation_query()) + + _test() + + +@background_task() +def test_firestore_async_aggregation_query_generators(async_collection, assert_trace_for_async_generator): + async_aggregation_query = async_collection.select("x").where(field_path="x", op_string="<=", value=3).count() + assert_trace_for_async_generator(async_aggregation_query.stream) + + +def test_firestore_async_aggregation_query_trace_node_datastore_params( + loop, exercise_async_aggregation_query, instance_info +): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + loop.run_until_complete(exercise_async_aggregation_query()) + + _test() + + +# ===== CollectionGroup ===== + + +@pytest.fixture() +def patch_partition_queries(monkeypatch, async_client, collection, sample_data): + """ + Partitioning is not implemented in the Firestore emulator. + + Ordinarily this method would return a coroutine that returns an async_generator of Cursor objects. + Each Cursor must point at a valid document path. To test this, we can patch the RPC to return 1 Cursor + which is pointed at any document available. The get_partitions will take that and make 2 QueryPartition + objects out of it, which should be enough to ensure we can exercise the generator's tracing. + """ + from google.cloud.firestore_v1.types.document import Value + from google.cloud.firestore_v1.types.query import Cursor + + subcollection = collection.document("subcollection").collection("subcollection1") + documents = [d for d in subcollection.list_documents()] + + async def mock_partition_query(*args, **kwargs): + async def _mock_partition_query(): + yield Cursor(before=False, values=[Value(reference_value=documents[0].path)]) + + return _mock_partition_query() + + monkeypatch.setattr(async_client._firestore_api, "partition_query", mock_partition_query) + yield + + +@pytest.fixture() +def exercise_async_collection_group(async_client, async_collection): + async def _exercise_async_collection_group(): + async_collection_group = async_client.collection_group(async_collection.id) + assert len(await async_collection_group.get()) + assert len([d async for d in async_collection_group.stream()]) + + partitions = [p async for p in async_collection_group.get_partitions(1)] + assert len(partitions) == 2 + documents = [] + while partitions: + documents.extend(await partitions.pop().query().get()) + assert len(documents) == 6 + + return _exercise_async_collection_group + + +def test_firestore_async_collection_group( + loop, exercise_async_collection_group, async_collection, patch_partition_queries, instance_info +): + _test_scoped_metrics = [ + (f"Datastore/statement/Firestore/{async_collection.id}/get", 3), + (f"Datastore/statement/Firestore/{async_collection.id}/stream", 1), + (f"Datastore/statement/Firestore/{async_collection.id}/get_partitions", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/operation/Firestore/get", 3), + ("Datastore/operation/Firestore/stream", 1), + ("Datastore/operation/Firestore/get_partitions", 1), + ("Datastore/all", 5), + ("Datastore/allOther", 5), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 5), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_async_collection_group", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_async_collection_group") + def _test(): + loop.run_until_complete(exercise_async_collection_group()) + + _test() + + +@background_task() +def test_firestore_async_collection_group_generators( + async_client, async_collection, assert_trace_for_async_generator, patch_partition_queries +): + async_collection_group = async_client.collection_group(async_collection.id) + assert_trace_for_async_generator(async_collection_group.get_partitions, 1) + + +def test_firestore_async_collection_group_trace_node_datastore_params( + loop, exercise_async_collection_group, instance_info, patch_partition_queries +): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + loop.run_until_complete(exercise_async_collection_group()) + + _test() \ No newline at end of file diff --git a/tests/datastore_firestore/test_async_transaction.py b/tests/datastore_firestore/test_async_transaction.py new file mode 100644 index 0000000000..37a5cc76bd --- /dev/null +++ b/tests/datastore_firestore/test_async_transaction.py @@ -0,0 +1,169 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_database_duration import ( + validate_database_duration, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_collector_json import ( + validate_tt_collector_json, +) + +from newrelic.api.background_task import background_task + + +@pytest.fixture(autouse=True) +def sample_data(collection): + for x in range(1, 4): + collection.add({"x": x}, f"doc{x}") + + +@pytest.fixture() +def exercise_async_transaction_commit(async_client, async_collection): + async def _exercise_async_transaction_commit(): + from google.cloud.firestore import async_transactional + + @async_transactional + async def _exercise(async_transaction): + # get a DocumentReference + with pytest.raises( + TypeError + ): # get is currently broken. It attempts to await an async_generator instead of consuming it. + [_ async for _ in async_transaction.get(async_collection.document("doc1"))] + + # get a Query + with pytest.raises( + TypeError + ): # get is currently broken. It attempts to await an async_generator instead of consuming it. + async_query = async_collection.select("x").where(field_path="x", op_string=">", value=2) + assert len([_ async for _ in async_transaction.get(async_query)]) == 1 + + # get_all on a list of DocumentReferences + with pytest.raises( + TypeError + ): # get_all is currently broken. It attempts to await an async_generator instead of consuming it. + all_docs = async_transaction.get_all([async_collection.document(f"doc{x}") for x in range(1, 4)]) + assert len([_ async for _ in all_docs]) == 3 + + # set and delete methods + async_transaction.set(async_collection.document("doc2"), {"x": 0}) + async_transaction.delete(async_collection.document("doc3")) + + await _exercise(async_client.transaction()) + assert len([_ async for _ in async_collection.list_documents()]) == 2 + + return _exercise_async_transaction_commit + + +@pytest.fixture() +def exercise_async_transaction_rollback(async_client, async_collection): + async def _exercise_async_transaction_rollback(): + from google.cloud.firestore import async_transactional + + @async_transactional + async def _exercise(async_transaction): + # set and delete methods + async_transaction.set(async_collection.document("doc2"), {"x": 99}) + async_transaction.delete(async_collection.document("doc1")) + raise RuntimeError() + + with pytest.raises(RuntimeError): + await _exercise(async_client.transaction()) + assert len([_ async for _ in async_collection.list_documents()]) == 3 + + return _exercise_async_transaction_rollback + + +def test_firestore_async_transaction_commit(loop, exercise_async_transaction_commit, async_collection, instance_info): + _test_scoped_metrics = [ + ("Datastore/operation/Firestore/commit", 1), + # ("Datastore/operation/Firestore/get_all", 2), + # (f"Datastore/statement/Firestore/{async_collection.id}/stream", 1), + (f"Datastore/statement/Firestore/{async_collection.id}/list_documents", 1), + ] + + _test_rollup_metrics = [ + # ("Datastore/operation/Firestore/stream", 1), + ("Datastore/operation/Firestore/list_documents", 1), + ("Datastore/all", 2), # Should be 5 if not for broken APIs + ("Datastore/allOther", 2), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 2), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_async_transaction", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_async_transaction") + def _test(): + loop.run_until_complete(exercise_async_transaction_commit()) + + _test() + + +def test_firestore_async_transaction_rollback( + loop, exercise_async_transaction_rollback, async_collection, instance_info +): + _test_scoped_metrics = [ + ("Datastore/operation/Firestore/rollback", 1), + (f"Datastore/statement/Firestore/{async_collection.id}/list_documents", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/operation/Firestore/list_documents", 1), + ("Datastore/all", 2), + ("Datastore/allOther", 2), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 2), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_async_transaction", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_async_transaction") + def _test(): + loop.run_until_complete(exercise_async_transaction_rollback()) + + _test() + + +def test_firestore_async_transaction_commit_trace_node_datastore_params( + loop, exercise_async_transaction_commit, instance_info +): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + loop.run_until_complete(exercise_async_transaction_commit()) + + _test() + + +def test_firestore_async_transaction_rollback_trace_node_datastore_params( + loop, exercise_async_transaction_rollback, instance_info +): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + loop.run_until_complete(exercise_async_transaction_rollback()) + + _test() diff --git a/tests/datastore_firestore/test_batching.py b/tests/datastore_firestore/test_batching.py new file mode 100644 index 0000000000..67b1b28a0b --- /dev/null +++ b/tests/datastore_firestore/test_batching.py @@ -0,0 +1,127 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_database_duration import ( + validate_database_duration, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_collector_json import ( + validate_tt_collector_json, +) + +from newrelic.api.background_task import background_task + +# ===== WriteBatch ===== + + +@pytest.fixture() +def exercise_write_batch(client, collection): + def _exercise_write_batch(): + docs = [collection.document(str(x)) for x in range(1, 4)] + batch = client.batch() + for doc in docs: + batch.set(doc, {}) + + batch.commit() + + return _exercise_write_batch + + +def test_firestore_write_batch(exercise_write_batch, instance_info): + _test_scoped_metrics = [ + ("Datastore/operation/Firestore/commit", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/all", 1), + ("Datastore/allOther", 1), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 1), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_write_batch", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_write_batch") + def _test(): + exercise_write_batch() + + _test() + + +def test_firestore_write_batch_trace_node_datastore_params(exercise_write_batch, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + exercise_write_batch() + + _test() + + +# ===== BulkWriteBatch ===== + + +@pytest.fixture() +def exercise_bulk_write_batch(client, collection): + def _exercise_bulk_write_batch(): + from google.cloud.firestore_v1.bulk_batch import BulkWriteBatch + + docs = [collection.document(str(x)) for x in range(1, 4)] + batch = BulkWriteBatch(client) + for doc in docs: + batch.set(doc, {}) + + batch.commit() + + return _exercise_bulk_write_batch + + +def test_firestore_bulk_write_batch(exercise_bulk_write_batch, instance_info): + _test_scoped_metrics = [ + ("Datastore/operation/Firestore/commit", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/all", 1), + ("Datastore/allOther", 1), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 1), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_bulk_write_batch", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_bulk_write_batch") + def _test(): + exercise_bulk_write_batch() + + _test() + + +def test_firestore_bulk_write_batch_trace_node_datastore_params(exercise_bulk_write_batch, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + exercise_bulk_write_batch() + + _test() diff --git a/tests/datastore_firestore/test_client.py b/tests/datastore_firestore/test_client.py new file mode 100644 index 0000000000..3e00d4d335 --- /dev/null +++ b/tests/datastore_firestore/test_client.py @@ -0,0 +1,83 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +from testing_support.validators.validate_database_duration import ( + validate_database_duration, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_collector_json import ( + validate_tt_collector_json, +) + +from newrelic.api.background_task import background_task + + +@pytest.fixture() +def sample_data(collection): + doc = collection.document("document") + doc.set({"x": 1}) + return doc + + +@pytest.fixture() +def exercise_client(client, sample_data): + def _exercise_client(): + assert len([_ for _ in client.collections()]) + doc = [_ for _ in client.get_all([sample_data])][0] + assert doc.to_dict()["x"] == 1 + + return _exercise_client + + +def test_firestore_client(exercise_client, instance_info): + _test_scoped_metrics = [ + ("Datastore/operation/Firestore/collections", 1), + ("Datastore/operation/Firestore/get_all", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/all", 2), + ("Datastore/allOther", 2), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 2), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_client", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_client") + def _test(): + exercise_client() + + _test() + + +@background_task() +def test_firestore_client_generators(client, sample_data, assert_trace_for_generator): + assert_trace_for_generator(client.collections) + assert_trace_for_generator(client.get_all, [sample_data]) + + +def test_firestore_client_trace_node_datastore_params(exercise_client, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + exercise_client() + + _test() \ No newline at end of file diff --git a/tests/datastore_firestore/test_collections.py b/tests/datastore_firestore/test_collections.py new file mode 100644 index 0000000000..8597cdd5e9 --- /dev/null +++ b/tests/datastore_firestore/test_collections.py @@ -0,0 +1,94 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_database_duration import ( + validate_database_duration, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_collector_json import ( + validate_tt_collector_json, +) + +from newrelic.api.background_task import background_task + + +@pytest.fixture() +def exercise_collections(collection): + def _exercise_collections(): + collection.document("DoesNotExist") + collection.add({"capital": "Rome", "currency": "Euro", "language": "Italian"}, "Italy") + collection.add({"capital": "Mexico City", "currency": "Peso", "language": "Spanish"}, "Mexico") + + documents_get = collection.get() + assert len(documents_get) == 2 + documents_stream = [_ for _ in collection.stream()] + assert len(documents_stream) == 2 + documents_list = [_ for _ in collection.list_documents()] + assert len(documents_list) == 2 + + return _exercise_collections + + +def test_firestore_collections(exercise_collections, collection, instance_info): + _test_scoped_metrics = [ + (f"Datastore/statement/Firestore/{collection.id}/stream", 1), + (f"Datastore/statement/Firestore/{collection.id}/get", 1), + (f"Datastore/statement/Firestore/{collection.id}/list_documents", 1), + (f"Datastore/statement/Firestore/{collection.id}/add", 2), + ] + + _test_rollup_metrics = [ + ("Datastore/operation/Firestore/add", 2), + ("Datastore/operation/Firestore/get", 1), + ("Datastore/operation/Firestore/stream", 1), + ("Datastore/operation/Firestore/list_documents", 1), + ("Datastore/all", 5), + ("Datastore/allOther", 5), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 5), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_collections", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_collections") + def _test(): + exercise_collections() + + _test() + + +@background_task() +def test_firestore_collections_generators(collection, assert_trace_for_generator): + collection.add({}) + collection.add({}) + assert len([_ for _ in collection.list_documents()]) == 2 + + assert_trace_for_generator(collection.stream) + assert_trace_for_generator(collection.list_documents) + + +def test_firestore_collections_trace_node_datastore_params(exercise_collections, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + exercise_collections() + + _test() diff --git a/tests/datastore_firestore/test_documents.py b/tests/datastore_firestore/test_documents.py new file mode 100644 index 0000000000..11a737cbc8 --- /dev/null +++ b/tests/datastore_firestore/test_documents.py @@ -0,0 +1,104 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_database_duration import ( + validate_database_duration, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_collector_json import ( + validate_tt_collector_json, +) + +from newrelic.api.background_task import background_task + + +@pytest.fixture() +def exercise_documents(collection): + def _exercise_documents(): + italy_doc = collection.document("Italy") + italy_doc.set({"capital": "Rome", "currency": "Euro", "language": "Italian"}) + italy_doc.get() + italian_cities = italy_doc.collection("cities") + italian_cities.add({"capital": "Rome"}) + retrieved_coll = [_ for _ in italy_doc.collections()] + assert len(retrieved_coll) == 1 + + usa_doc = collection.document("USA") + usa_doc.create({"capital": "Washington D.C.", "currency": "Dollar", "language": "English"}) + usa_doc.update({"president": "Joe Biden"}) + + collection.document("USA").delete() + + return _exercise_documents + + +def test_firestore_documents(exercise_documents, instance_info): + _test_scoped_metrics = [ + ("Datastore/statement/Firestore/Italy/set", 1), + ("Datastore/statement/Firestore/Italy/get", 1), + ("Datastore/statement/Firestore/Italy/collections", 1), + ("Datastore/statement/Firestore/cities/add", 1), + ("Datastore/statement/Firestore/USA/create", 1), + ("Datastore/statement/Firestore/USA/update", 1), + ("Datastore/statement/Firestore/USA/delete", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/operation/Firestore/set", 1), + ("Datastore/operation/Firestore/get", 1), + ("Datastore/operation/Firestore/add", 1), + ("Datastore/operation/Firestore/collections", 1), + ("Datastore/operation/Firestore/create", 1), + ("Datastore/operation/Firestore/update", 1), + ("Datastore/operation/Firestore/delete", 1), + ("Datastore/all", 7), + ("Datastore/allOther", 7), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 7), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_documents", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_documents") + def _test(): + exercise_documents() + + _test() + + +@background_task() +def test_firestore_documents_generators(collection, assert_trace_for_generator): + subcollection_doc = collection.document("SubCollections") + subcollection_doc.set({}) + subcollection_doc.collection("collection1").add({}) + subcollection_doc.collection("collection2").add({}) + assert len([_ for _ in subcollection_doc.collections()]) == 2 + + assert_trace_for_generator(subcollection_doc.collections) + + +def test_firestore_documents_trace_node_datastore_params(exercise_documents, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + exercise_documents() + + _test() diff --git a/tests/datastore_firestore/test_query.py b/tests/datastore_firestore/test_query.py new file mode 100644 index 0000000000..6ec576e4a5 --- /dev/null +++ b/tests/datastore_firestore/test_query.py @@ -0,0 +1,236 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.validators.validate_database_duration import ( + validate_database_duration, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_collector_json import ( + validate_tt_collector_json, +) + +from newrelic.api.background_task import background_task + + +@pytest.fixture(autouse=True) +def sample_data(collection): + for x in range(1, 6): + collection.add({"x": x}) + + subcollection_doc = collection.document("subcollection") + subcollection_doc.set({}) + subcollection_doc.collection("subcollection1").add({}) + + +# ===== Query ===== + + +@pytest.fixture() +def exercise_query(collection): + def _exercise_query(): + query = collection.select("x").limit(10).order_by("x").where(field_path="x", op_string="<=", value=3) + assert len(query.get()) == 3 + assert len([_ for _ in query.stream()]) == 3 + + return _exercise_query + + +def test_firestore_query(exercise_query, collection, instance_info): + _test_scoped_metrics = [ + (f"Datastore/statement/Firestore/{collection.id}/stream", 1), + (f"Datastore/statement/Firestore/{collection.id}/get", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/operation/Firestore/get", 1), + ("Datastore/operation/Firestore/stream", 1), + ("Datastore/all", 2), + ("Datastore/allOther", 2), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 2), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_query", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_query") + def _test(): + exercise_query() + + _test() + + +@background_task() +def test_firestore_query_generators(collection, assert_trace_for_generator): + query = collection.select("x").where(field_path="x", op_string="<=", value=3) + assert_trace_for_generator(query.stream) + + +def test_firestore_query_trace_node_datastore_params(exercise_query, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + exercise_query() + + _test() + + +# ===== AggregationQuery ===== + + +@pytest.fixture() +def exercise_aggregation_query(collection): + def _exercise_aggregation_query(): + aggregation_query = collection.select("x").where(field_path="x", op_string="<=", value=3).count() + assert aggregation_query.get()[0][0].value == 3 + assert [_ for _ in aggregation_query.stream()][0][0].value == 3 + + return _exercise_aggregation_query + + +def test_firestore_aggregation_query(exercise_aggregation_query, collection, instance_info): + _test_scoped_metrics = [ + (f"Datastore/statement/Firestore/{collection.id}/stream", 1), + (f"Datastore/statement/Firestore/{collection.id}/get", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/operation/Firestore/get", 1), + ("Datastore/operation/Firestore/stream", 1), + ("Datastore/all", 2), + ("Datastore/allOther", 2), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 2), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_aggregation_query", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_aggregation_query") + def _test(): + exercise_aggregation_query() + + _test() + + +@background_task() +def test_firestore_aggregation_query_generators(collection, assert_trace_for_generator): + aggregation_query = collection.select("x").where(field_path="x", op_string="<=", value=3).count() + assert_trace_for_generator(aggregation_query.stream) + + +def test_firestore_aggregation_query_trace_node_datastore_params(exercise_aggregation_query, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + exercise_aggregation_query() + + _test() + + +# ===== CollectionGroup ===== + + +@pytest.fixture() +def patch_partition_queries(monkeypatch, client, collection, sample_data): + """ + Partitioning is not implemented in the Firestore emulator. + + Ordinarily this method would return a generator of Cursor objects. Each Cursor must point at a valid document path. + To test this, we can patch the RPC to return 1 Cursor which is pointed at any document available. + The get_partitions will take that and make 2 QueryPartition objects out of it, which should be enough to ensure + we can exercise the generator's tracing. + """ + from google.cloud.firestore_v1.types.document import Value + from google.cloud.firestore_v1.types.query import Cursor + + subcollection = collection.document("subcollection").collection("subcollection1") + documents = [d for d in subcollection.list_documents()] + + def mock_partition_query(*args, **kwargs): + yield Cursor(before=False, values=[Value(reference_value=documents[0].path)]) + + monkeypatch.setattr(client._firestore_api, "partition_query", mock_partition_query) + yield + + +@pytest.fixture() +def exercise_collection_group(client, collection, patch_partition_queries): + def _exercise_collection_group(): + collection_group = client.collection_group(collection.id) + assert len(collection_group.get()) + assert len([d for d in collection_group.stream()]) + + partitions = [p for p in collection_group.get_partitions(1)] + assert len(partitions) == 2 + documents = [] + while partitions: + documents.extend(partitions.pop().query().get()) + assert len(documents) == 6 + + return _exercise_collection_group + + +def test_firestore_collection_group(exercise_collection_group, client, collection, instance_info): + _test_scoped_metrics = [ + (f"Datastore/statement/Firestore/{collection.id}/get", 3), + (f"Datastore/statement/Firestore/{collection.id}/stream", 1), + (f"Datastore/statement/Firestore/{collection.id}/get_partitions", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/operation/Firestore/get", 3), + ("Datastore/operation/Firestore/stream", 1), + ("Datastore/operation/Firestore/get_partitions", 1), + ("Datastore/all", 5), + ("Datastore/allOther", 5), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 5), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_collection_group", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_collection_group") + def _test(): + exercise_collection_group() + + _test() + + +@background_task() +def test_firestore_collection_group_generators(client, collection, assert_trace_for_generator, patch_partition_queries): + collection_group = client.collection_group(collection.id) + assert_trace_for_generator(collection_group.get_partitions, 1) + + +def test_firestore_collection_group_trace_node_datastore_params(exercise_collection_group, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + exercise_collection_group() + + _test() diff --git a/tests/datastore_firestore/test_transaction.py b/tests/datastore_firestore/test_transaction.py new file mode 100644 index 0000000000..495c61e767 --- /dev/null +++ b/tests/datastore_firestore/test_transaction.py @@ -0,0 +1,153 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import pytest +from testing_support.validators.validate_database_duration import ( + validate_database_duration, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_collector_json import ( + validate_tt_collector_json, +) + +from newrelic.api.background_task import background_task + + +@pytest.fixture(autouse=True) +def sample_data(collection): + for x in range(1, 4): + collection.add({"x": x}, f"doc{x}") + + +@pytest.fixture() +def exercise_transaction_commit(client, collection): + def _exercise_transaction_commit(): + from google.cloud.firestore_v1.transaction import transactional + + @transactional + def _exercise(transaction): + # get a DocumentReference + [_ for _ in transaction.get(collection.document("doc1"))] + + # get a Query + query = collection.select("x").where(field_path="x", op_string=">", value=2) + assert len([_ for _ in transaction.get(query)]) == 1 + + # get_all on a list of DocumentReferences + all_docs = transaction.get_all([collection.document(f"doc{x}") for x in range(1, 4)]) + assert len([_ for _ in all_docs]) == 3 + + # set and delete methods + transaction.set(collection.document("doc2"), {"x": 0}) + transaction.delete(collection.document("doc3")) + + _exercise(client.transaction()) + assert len([_ for _ in collection.list_documents()]) == 2 + + return _exercise_transaction_commit + + +@pytest.fixture() +def exercise_transaction_rollback(client, collection): + def _exercise_transaction_rollback(): + from google.cloud.firestore_v1.transaction import transactional + + @transactional + def _exercise(transaction): + # set and delete methods + transaction.set(collection.document("doc2"), {"x": 99}) + transaction.delete(collection.document("doc1")) + raise RuntimeError() + + with pytest.raises(RuntimeError): + _exercise(client.transaction()) + assert len([_ for _ in collection.list_documents()]) == 3 + + return _exercise_transaction_rollback + + +def test_firestore_transaction_commit(exercise_transaction_commit, collection, instance_info): + _test_scoped_metrics = [ + ("Datastore/operation/Firestore/commit", 1), + ("Datastore/operation/Firestore/get_all", 2), + (f"Datastore/statement/Firestore/{collection.id}/stream", 1), + (f"Datastore/statement/Firestore/{collection.id}/list_documents", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/operation/Firestore/stream", 1), + ("Datastore/operation/Firestore/list_documents", 1), + ("Datastore/all", 5), + ("Datastore/allOther", 5), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 5), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_transaction", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_transaction") + def _test(): + exercise_transaction_commit() + + _test() + + +def test_firestore_transaction_rollback(exercise_transaction_rollback, collection, instance_info): + _test_scoped_metrics = [ + ("Datastore/operation/Firestore/rollback", 1), + (f"Datastore/statement/Firestore/{collection.id}/list_documents", 1), + ] + + _test_rollup_metrics = [ + ("Datastore/operation/Firestore/list_documents", 1), + ("Datastore/all", 2), + ("Datastore/allOther", 2), + (f"Datastore/instance/Firestore/{instance_info['host']}/{instance_info['port_path_or_id']}", 2), + ] + + @validate_database_duration() + @validate_transaction_metrics( + "test_firestore_transaction", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, + ) + @background_task(name="test_firestore_transaction") + def _test(): + exercise_transaction_rollback() + + _test() + + +def test_firestore_transaction_commit_trace_node_datastore_params(exercise_transaction_commit, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + exercise_transaction_commit() + + _test() + + +def test_firestore_transaction_rollback_trace_node_datastore_params(exercise_transaction_rollback, instance_info): + @validate_tt_collector_json(datastore_params=instance_info) + @background_task() + def _test(): + exercise_transaction_rollback() + + _test() diff --git a/tests/datastore_memcache/conftest.py b/tests/datastore_memcache/conftest.py index 835e895bd8..e63f9c0274 100644 --- a/tests/datastore_memcache/conftest.py +++ b/tests/datastore_memcache/conftest.py @@ -14,32 +14,36 @@ import random import string -import pytest -import memcache - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +import memcache +import pytest from testing_support.db_settings import memcached_settings - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_memcache)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_memcache)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) + -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def memcached_multi(): """Generate keys that will go onto different servers""" DB_SETTINGS = memcached_settings() - db_servers = ['%s:%s' % (s['host'], s['port']) for s in DB_SETTINGS] + db_servers = [f"{s['host']}:{s['port']}" for s in DB_SETTINGS] clients = [memcache.Client([s]) for s in db_servers] client_all = memcache.Client(db_servers) @@ -48,9 +52,8 @@ def memcached_multi(): for try_num in range(10 * num_servers): multi_dict = {} for i in range(num_servers): - random_chars = (random.choice(string.ascii_uppercase) - for _ in range(10)) - key_candidate = ''.join(random_chars) + random_chars = (random.choice(string.ascii_uppercase) for _ in range(10)) + key_candidate = "".join(random_chars) multi_dict[key_candidate] = key_candidate client_all.set_multi(multi_dict) diff --git a/tests/datastore_memcache/test_memcache.py b/tests/datastore_memcache/test_memcache.py index a66c114eef..dba37d6071 100644 --- a/tests/datastore_memcache/test_memcache.py +++ b/tests/datastore_memcache/test_memcache.py @@ -23,7 +23,7 @@ from newrelic.common.object_wrapper import wrap_function_wrapper DB_SETTINGS = memcached_settings()[0] -MEMCACHED_ADDR = '%s:%s' % (DB_SETTINGS['host'], DB_SETTINGS['port']) +MEMCACHED_ADDR = f"{DB_SETTINGS['host']}:{DB_SETTINGS['port']}" # Settings @@ -59,7 +59,7 @@ _host = instance_hostname(DB_SETTINGS['host']) _port = DB_SETTINGS['port'] -_instance_metric_name = 'Datastore/instance/Memcached/%s/%s' % (_host, _port) +_instance_metric_name = f'Datastore/instance/Memcached/{_host}/{_port}' _enable_rollup_metrics.append( (_instance_metric_name, 3) @@ -72,7 +72,7 @@ # Query def _exercise_db(client): - key = DB_SETTINGS['namespace'] + 'key' + key = f"{DB_SETTINGS['namespace']}key" client.set(key, 'value') value = client.get(key) client.delete(key) diff --git a/tests/datastore_memcache/test_multiple_dbs.py b/tests/datastore_memcache/test_multiple_dbs.py index dbc3ea2b3b..c2b656775a 100644 --- a/tests/datastore_memcache/test_multiple_dbs.py +++ b/tests/datastore_memcache/test_multiple_dbs.py @@ -65,10 +65,8 @@ host_2 = instance_hostname(memcached_2['host']) port_2 = memcached_2['port'] - instance_metric_name_1 = 'Datastore/instance/Memcached/%s/%s' % (host_1, - port_1) - instance_metric_name_2 = 'Datastore/instance/Memcached/%s/%s' % (host_2, - port_2) + instance_metric_name_1 = f'Datastore/instance/Memcached/{host_1}/{port_1}' + instance_metric_name_2 = f'Datastore/instance/Memcached/{host_2}/{port_2}' _enable_rollup_metrics.extend([ (instance_metric_name_1, None), @@ -89,7 +87,7 @@ def exercise_memcached(client, multi_dict): @pytest.mark.skipif(len(DB_MULTIPLE_SETTINGS) < 2, reason='Test environment not configured with multiple databases.') @override_application_settings(_enable_instance_settings) -@validate_transaction_metrics(transaction_metric_prefix+'_enabled', +@validate_transaction_metrics(f"{transaction_metric_prefix}_enabled", scoped_metrics=_enable_scoped_metrics, rollup_metrics=_enable_rollup_metrics, background_task=True) @@ -98,7 +96,7 @@ def test_multiple_datastores_enabled(memcached_multi): memcached1 = DB_MULTIPLE_SETTINGS[0] memcached2 = DB_MULTIPLE_SETTINGS[1] settings = [memcached1, memcached2] - servers = ["%s:%s" % (x['host'], x['port']) for x in settings] + servers = [f"{x['host']}:{x['port']}" for x in settings] client = memcache.Client(servers=servers) @@ -107,7 +105,7 @@ def test_multiple_datastores_enabled(memcached_multi): @pytest.mark.skipif(len(DB_MULTIPLE_SETTINGS) < 2, reason='Test environment not configured with multiple databases.') @override_application_settings(_disable_instance_settings) -@validate_transaction_metrics(transaction_metric_prefix+'_disabled', +@validate_transaction_metrics(f"{transaction_metric_prefix}_disabled", scoped_metrics=_disable_scoped_metrics, rollup_metrics=_disable_rollup_metrics, background_task=True) @@ -116,7 +114,7 @@ def test_multiple_datastores_disabled(memcached_multi): memcached1 = DB_MULTIPLE_SETTINGS[0] memcached2 = DB_MULTIPLE_SETTINGS[1] settings = [memcached1, memcached2] - servers = ["%s:%s" % (x['host'], x['port']) for x in settings] + servers = [f"{x['host']}:{x['port']}" for x in settings] client = memcache.Client(servers=servers) diff --git a/tests/datastore_memcache/test_span_event.py b/tests/datastore_memcache/test_span_event.py index a8da4d0e56..cea8a06e14 100644 --- a/tests/datastore_memcache/test_span_event.py +++ b/tests/datastore_memcache/test_span_event.py @@ -25,7 +25,7 @@ from newrelic.api.background_task import background_task DB_SETTINGS = memcached_settings()[0] -MEMCACHED_ADDR = '%s:%s' % (DB_SETTINGS['host'], DB_SETTINGS['port']) +MEMCACHED_ADDR = f"{DB_SETTINGS['host']}:{DB_SETTINGS['port']}" # Settings @@ -44,7 +44,7 @@ # Query def _exercise_db(client): - key = DB_SETTINGS['namespace'] + 'key' + key = f"{DB_SETTINGS['namespace']}key" client.set(key, 'value') value = client.get(key) client.delete(key) @@ -73,7 +73,7 @@ def test_span_events(instance_enabled): settings = _enable_instance_settings hostname = instance_hostname(DB_SETTINGS['host']) exact_agents.update({ - 'peer.address': '%s:%s' % (hostname, DB_SETTINGS['port']), + 'peer.address': f"{hostname}:{DB_SETTINGS['port']}", 'peer.hostname': hostname, }) else: diff --git a/tests/datastore_mysql/conftest.py b/tests/datastore_mysql/conftest.py index a2f74c398f..fa2b0df727 100644 --- a/tests/datastore_mysql/conftest.py +++ b/tests/datastore_mysql/conftest.py @@ -12,26 +12,31 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest import os -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +import pytest +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, - 'debug.log_explain_plan_queries': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_explain_plan_queries": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_mysql)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_mysql)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) + @pytest.fixture(scope="session") def table_name(): - return str("datastore_mysql_%d" % os.getpid()) + return str(f"datastore_mysql_{os.getpid()}") diff --git a/tests/datastore_mysql/test_database.py b/tests/datastore_mysql/test_database.py index 2fc8ca129b..d0cbfbdaaa 100644 --- a/tests/datastore_mysql/test_database.py +++ b/tests/datastore_mysql/test_database.py @@ -13,173 +13,214 @@ # limitations under the License. import mysql.connector - -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_database_trace_inputs import validate_database_trace_inputs - from testing_support.db_settings import mysql_settings +from testing_support.util import instance_hostname +from testing_support.validators.validate_database_trace_inputs import ( + validate_database_trace_inputs, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple DB_SETTINGS = mysql_settings() DB_SETTINGS = DB_SETTINGS[0] DB_NAMESPACE = DB_SETTINGS["namespace"] -DB_PROCEDURE = "hello_" + DB_NAMESPACE +DB_PROCEDURE = f"hello_{DB_NAMESPACE}" + +mysql_version = get_package_version_tuple("mysql.connector") -mysql_version = tuple(int(x) for x in mysql.connector.__version__.split(".")[:3]) if mysql_version >= (8, 0, 30): - _connector_metric_name = 'Function/mysql.connector.pooling:connect' + _connector_metric_name = "Function/mysql.connector.pooling:connect" else: - _connector_metric_name = 'Function/mysql.connector:connect' + _connector_metric_name = "Function/mysql.connector:connect" _test_execute_via_cursor_scoped_metrics = [ - (_connector_metric_name, 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/select' % DB_NAMESPACE, 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/insert' % DB_NAMESPACE, 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/update' % DB_NAMESPACE, 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/delete' % DB_NAMESPACE, 1), - ('Datastore/operation/MySQL/drop', 2), - ('Datastore/operation/MySQL/create', 2), - ('Datastore/statement/MySQL/%s/call' % DB_PROCEDURE, 1), - ('Datastore/operation/MySQL/commit', 2), - ('Datastore/operation/MySQL/rollback', 1)] + (_connector_metric_name, 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/select", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/insert", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/update", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/delete", 1), + ("Datastore/operation/MySQL/drop", 2), + ("Datastore/operation/MySQL/create", 2), + (f"Datastore/statement/MySQL/{DB_PROCEDURE}/call", 1), + ("Datastore/operation/MySQL/commit", 2), + ("Datastore/operation/MySQL/rollback", 1), +] _test_execute_via_cursor_rollup_metrics = [ - ('Datastore/all', 13), - ('Datastore/allOther', 13), - ('Datastore/MySQL/all', 13), - ('Datastore/MySQL/allOther', 13), - ('Datastore/operation/MySQL/select', 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/select' % DB_NAMESPACE, 1), - ('Datastore/operation/MySQL/insert', 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/insert' % DB_NAMESPACE, 1), - ('Datastore/operation/MySQL/update', 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/update' % DB_NAMESPACE, 1), - ('Datastore/operation/MySQL/delete', 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/delete' % DB_NAMESPACE, 1), - ('Datastore/statement/MySQL/%s/call' % DB_PROCEDURE, 1), - ('Datastore/operation/MySQL/call', 1), - ('Datastore/operation/MySQL/drop', 2), - ('Datastore/operation/MySQL/create', 2), - ('Datastore/operation/MySQL/commit', 2), - ('Datastore/operation/MySQL/rollback', 1)] - -@validate_transaction_metrics('test_database:test_execute_via_cursor', - scoped_metrics=_test_execute_via_cursor_scoped_metrics, - rollup_metrics=_test_execute_via_cursor_rollup_metrics, - background_task=True) + ("Datastore/all", 13), + ("Datastore/allOther", 13), + ("Datastore/MySQL/all", 13), + ("Datastore/MySQL/allOther", 13), + ("Datastore/operation/MySQL/select", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/select", 1), + ("Datastore/operation/MySQL/insert", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/insert", 1), + ("Datastore/operation/MySQL/update", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/update", 1), + ("Datastore/operation/MySQL/delete", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/delete", 1), + (f"Datastore/statement/MySQL/{DB_PROCEDURE}/call", 1), + ("Datastore/operation/MySQL/call", 1), + ("Datastore/operation/MySQL/drop", 2), + ("Datastore/operation/MySQL/create", 2), + ("Datastore/operation/MySQL/commit", 2), + ("Datastore/operation/MySQL/rollback", 1), + (f"Datastore/instance/MySQL/{instance_hostname(DB_SETTINGS['host'])}/{DB_SETTINGS['port']}", 12), +] + + +@validate_transaction_metrics( + "test_database:test_execute_via_cursor", + scoped_metrics=_test_execute_via_cursor_scoped_metrics, + rollup_metrics=_test_execute_via_cursor_rollup_metrics, + background_task=True, +) +@validate_transaction_metrics( + "test_database:test_execute_via_cursor", + scoped_metrics=_test_execute_via_cursor_scoped_metrics, + rollup_metrics=_test_execute_via_cursor_rollup_metrics, + background_task=True, +) @validate_database_trace_inputs(sql_parameters_type=dict) @background_task() def test_execute_via_cursor(table_name): - - connection = mysql.connector.connect(db=DB_SETTINGS['name'], - user=DB_SETTINGS['user'], passwd=DB_SETTINGS['password'], - host=DB_SETTINGS['host'], port=DB_SETTINGS['port']) + assert mysql_version is not None + connection = mysql.connector.connect( + db=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + passwd=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ) cursor = connection.cursor() - cursor.execute("""drop table if exists `%s`""" % table_name) + cursor.execute(f"""drop table if exists `{table_name}`""") - cursor.execute("""create table %s """ - """(a integer, b real, c text)""" % table_name) + cursor.execute(f"""create table {table_name} (a integer, b real, c text)""") - cursor.executemany("""insert into `%s` """ % table_name + - """values (%(a)s, %(b)s, %(c)s)""", [dict(a=1, b=1.0, c='1.0'), - dict(a=2, b=2.2, c='2.2'), dict(a=3, b=3.3, c='3.3')]) + cursor.executemany( + f"insert into `{table_name}` values (%(a)s, %(b)s, %(c)s)", + [{"a": 1, "b": 1.0, "c": "1.0"}, {"a": 2, "b": 2.2, "c": "2.2"}, {"a": 3, "b": 3.3, "c": "3.3"}], + ) - cursor.execute("""select * from %s""" % table_name) + cursor.execute(f"""select * from {table_name}""") - for row in cursor: pass + for row in cursor: + pass - cursor.execute("""update `%s` """ % table_name + - """set a=%(a)s, b=%(b)s, c=%(c)s where a=%(old_a)s""", - dict(a=4, b=4.0, c='4.0', old_a=1)) + cursor.execute( + f"update `{table_name}` set a=%(a)s, b=%(b)s, c=%(c)s where a=%(old_a)s", + {"a": 4, "b": 4.0, "c": "4.0", "old_a": 1}, + ) - cursor.execute("""delete from `%s` where a=2""" % table_name) + cursor.execute(f"""delete from `{table_name}` where a=2""") - cursor.execute("""drop procedure if exists %s""" % DB_PROCEDURE) - cursor.execute("""CREATE PROCEDURE %s() + cursor.execute(f"""drop procedure if exists {DB_PROCEDURE}""") + cursor.execute( + f"""CREATE PROCEDURE {DB_PROCEDURE}() BEGIN SELECT 'Hello World!'; - END""" % DB_PROCEDURE) + END""" + ) - cursor.callproc("%s" % DB_PROCEDURE) + cursor.callproc(f"{DB_PROCEDURE}") connection.commit() connection.rollback() connection.commit() + _test_connect_using_alias_scoped_metrics = [ - (_connector_metric_name, 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/select' % DB_NAMESPACE, 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/insert' % DB_NAMESPACE, 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/update' % DB_NAMESPACE, 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/delete' % DB_NAMESPACE, 1), - ('Datastore/operation/MySQL/drop', 2), - ('Datastore/operation/MySQL/create', 2), - ('Datastore/statement/MySQL/%s/call' % DB_PROCEDURE, 1), - ('Datastore/operation/MySQL/commit', 2), - ('Datastore/operation/MySQL/rollback', 1)] + (_connector_metric_name, 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/select", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/insert", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/update", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/delete", 1), + ("Datastore/operation/MySQL/drop", 2), + ("Datastore/operation/MySQL/create", 2), + (f"Datastore/statement/MySQL/{DB_PROCEDURE}/call", 1), + ("Datastore/operation/MySQL/commit", 2), + ("Datastore/operation/MySQL/rollback", 1), +] _test_connect_using_alias_rollup_metrics = [ - ('Datastore/all', 13), - ('Datastore/allOther', 13), - ('Datastore/MySQL/all', 13), - ('Datastore/MySQL/allOther', 13), - ('Datastore/operation/MySQL/select', 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/select' % DB_NAMESPACE, 1), - ('Datastore/operation/MySQL/insert', 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/insert' % DB_NAMESPACE, 1), - ('Datastore/operation/MySQL/update', 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/update' % DB_NAMESPACE, 1), - ('Datastore/operation/MySQL/delete', 1), - ('Datastore/statement/MySQL/datastore_mysql_%s/delete' % DB_NAMESPACE, 1), - ('Datastore/statement/MySQL/%s/call' % DB_PROCEDURE, 1), - ('Datastore/operation/MySQL/call', 1), - ('Datastore/operation/MySQL/drop', 2), - ('Datastore/operation/MySQL/create', 2), - ('Datastore/operation/MySQL/commit', 2), - ('Datastore/operation/MySQL/rollback', 1)] - -@validate_transaction_metrics('test_database:test_connect_using_alias', - scoped_metrics=_test_connect_using_alias_scoped_metrics, - rollup_metrics=_test_connect_using_alias_rollup_metrics, - background_task=True) + ("Datastore/all", 13), + ("Datastore/allOther", 13), + ("Datastore/MySQL/all", 13), + ("Datastore/MySQL/allOther", 13), + ("Datastore/operation/MySQL/select", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/select", 1), + ("Datastore/operation/MySQL/insert", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/insert", 1), + ("Datastore/operation/MySQL/update", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/update", 1), + ("Datastore/operation/MySQL/delete", 1), + (f"Datastore/statement/MySQL/datastore_mysql_{DB_NAMESPACE}/delete", 1), + (f"Datastore/statement/MySQL/{DB_PROCEDURE}/call", 1), + ("Datastore/operation/MySQL/call", 1), + ("Datastore/operation/MySQL/drop", 2), + ("Datastore/operation/MySQL/create", 2), + ("Datastore/operation/MySQL/commit", 2), + ("Datastore/operation/MySQL/rollback", 1), + (f"Datastore/instance/MySQL/{instance_hostname(DB_SETTINGS['host'])}/{DB_SETTINGS['port']}", 12), +] + + +@validate_transaction_metrics( + "test_database:test_connect_using_alias", + scoped_metrics=_test_connect_using_alias_scoped_metrics, + rollup_metrics=_test_connect_using_alias_rollup_metrics, + background_task=True, +) @validate_database_trace_inputs(sql_parameters_type=dict) @background_task() def test_connect_using_alias(table_name): - - connection = mysql.connector.connect(db=DB_SETTINGS['name'], - user=DB_SETTINGS['user'], passwd=DB_SETTINGS['password'], - host=DB_SETTINGS['host'], port=DB_SETTINGS['port']) + assert mysql_version is not None + connection = mysql.connector.connect( + db=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + passwd=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ) cursor = connection.cursor() - cursor.execute("""drop table if exists `%s`""" % table_name) + cursor.execute(f"""drop table if exists `{table_name}`""") - cursor.execute("""create table %s """ - """(a integer, b real, c text)""" % table_name) + cursor.execute(f"""create table {table_name} (a integer, b real, c text)""") - cursor.executemany("""insert into `%s` """ % table_name + - """values (%(a)s, %(b)s, %(c)s)""", [dict(a=1, b=1.0, c='1.0'), - dict(a=2, b=2.2, c='2.2'), dict(a=3, b=3.3, c='3.3')]) + cursor.executemany( + f"insert into `{table_name}` values (%(a)s, %(b)s, %(c)s)", + [{"a": 1, "b": 1.0, "c": "1.0"}, {"a": 2, "b": 2.2, "c": "2.2"}, {"a": 3, "b": 3.3, "c": "3.3"}], + ) - cursor.execute("""select * from %s""" % table_name) + cursor.execute(f"""select * from {table_name}""") - for row in cursor: pass + for row in cursor: + pass - cursor.execute("""update `%s` """ % table_name + - """set a=%(a)s, b=%(b)s, c=%(c)s where a=%(old_a)s""", - dict(a=4, b=4.0, c='4.0', old_a=1)) + cursor.execute( + f"update `{table_name}` set a=%(a)s, b=%(b)s, c=%(c)s where a=%(old_a)s", + {"a": 4, "b": 4.0, "c": "4.0", "old_a": 1}, + ) - cursor.execute("""delete from `%s` where a=2""" % table_name) + cursor.execute(f"""delete from `{table_name}` where a=2""") - cursor.execute("""drop procedure if exists %s""" % DB_PROCEDURE) - cursor.execute("""CREATE PROCEDURE %s() + cursor.execute(f"""drop procedure if exists {DB_PROCEDURE}""") + cursor.execute( + f"""CREATE PROCEDURE {DB_PROCEDURE}() BEGIN SELECT 'Hello World!'; - END""" % DB_PROCEDURE) + END""" + ) - cursor.callproc("%s" % DB_PROCEDURE) + cursor.callproc(f"{DB_PROCEDURE}") connection.commit() connection.rollback() diff --git a/tests/datastore_postgresql/conftest.py b/tests/datastore_postgresql/conftest.py index 624fb47262..545de0f7d0 100644 --- a/tests/datastore_postgresql/conftest.py +++ b/tests/datastore_postgresql/conftest.py @@ -12,21 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, - 'debug.log_explain_plan_queries': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_explain_plan_queries": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_postgresql)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_postgresql)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_postgresql/test_database.py b/tests/datastore_postgresql/test_database.py index 2ea930b051..c7c9ba6dc3 100644 --- a/tests/datastore_postgresql/test_database.py +++ b/tests/datastore_postgresql/test_database.py @@ -13,15 +13,14 @@ # limitations under the License. import postgresql.driver.dbapi20 - - -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics - +from testing_support.db_settings import postgresql_settings +from testing_support.util import instance_hostname from testing_support.validators.validate_database_trace_inputs import ( validate_database_trace_inputs, ) - -from testing_support.db_settings import postgresql_settings +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.background_task import background_task @@ -31,31 +30,32 @@ ("Function/postgresql.driver.dbapi20:connect", 1), ("Function/postgresql.driver.dbapi20:Connection.__enter__", 1), ("Function/postgresql.driver.dbapi20:Connection.__exit__", 1), - ("Datastore/statement/Postgres/%s/select" % DB_SETTINGS["table_name"], 1), - ("Datastore/statement/Postgres/%s/insert" % DB_SETTINGS["table_name"], 1), - ("Datastore/statement/Postgres/%s/update" % DB_SETTINGS["table_name"], 1), - ("Datastore/statement/Postgres/%s/delete" % DB_SETTINGS["table_name"], 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/update", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/delete", 1), ("Datastore/statement/Postgres/now/call", 1), ("Datastore/statement/Postgres/pg_sleep/call", 1), ("Datastore/operation/Postgres/drop", 1), ("Datastore/operation/Postgres/create", 1), ("Datastore/operation/Postgres/commit", 3), ("Datastore/operation/Postgres/rollback", 1), + ("Datastore/operation/Postgres/other", 1), ] _test_execute_via_cursor_rollup_metrics = [ - ("Datastore/all", 13), - ("Datastore/allOther", 13), - ("Datastore/Postgres/all", 13), - ("Datastore/Postgres/allOther", 13), + ("Datastore/all", 14), + ("Datastore/allOther", 14), + ("Datastore/Postgres/all", 14), + ("Datastore/Postgres/allOther", 14), ("Datastore/operation/Postgres/select", 1), - ("Datastore/statement/Postgres/%s/select" % DB_SETTINGS["table_name"], 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), ("Datastore/operation/Postgres/insert", 1), - ("Datastore/statement/Postgres/%s/insert" % DB_SETTINGS["table_name"], 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 1), ("Datastore/operation/Postgres/update", 1), - ("Datastore/statement/Postgres/%s/update" % DB_SETTINGS["table_name"], 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/update", 1), ("Datastore/operation/Postgres/delete", 1), - ("Datastore/statement/Postgres/%s/delete" % DB_SETTINGS["table_name"], 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/delete", 1), ("Datastore/operation/Postgres/drop", 1), ("Datastore/operation/Postgres/create", 1), ("Datastore/statement/Postgres/now/call", 1), @@ -63,6 +63,11 @@ ("Datastore/operation/Postgres/call", 2), ("Datastore/operation/Postgres/commit", 3), ("Datastore/operation/Postgres/rollback", 1), + ("Datastore/operation/Postgres/other", 1), + (f"Datastore/instance/Postgres/{instance_hostname(DB_SETTINGS['host'])}/{DB_SETTINGS['port']}", 13), + ("Function/postgresql.driver.dbapi20:connect", 1), + ("Function/postgresql.driver.dbapi20:Connection.__enter__", 1), + ("Function/postgresql.driver.dbapi20:Connection.__exit__", 1), ] @@ -82,34 +87,29 @@ def test_execute_via_cursor(): host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], ) as connection: - cursor = connection.cursor() - cursor.execute("""drop table if exists %s""" % DB_SETTINGS["table_name"]) + cursor.execute(f"""drop table if exists {DB_SETTINGS['table_name']}""") - cursor.execute( - """create table %s """ % DB_SETTINGS["table_name"] - + """(a integer, b real, c text)""" - ) + cursor.execute(f"create table {DB_SETTINGS['table_name']} (a integer, b real, c text)") cursor.executemany( - """insert into %s """ % DB_SETTINGS["table_name"] - + """values (%s, %s, %s)""", + f"insert into {DB_SETTINGS['table_name']} values (%s, %s, %s)", [(1, 1.0, "1.0"), (2, 2.2, "2.2"), (3, 3.3, "3.3")], ) - cursor.execute("""select * from %s""" % DB_SETTINGS["table_name"]) + cursor.execute(f"""select * from {DB_SETTINGS['table_name']}""") - for row in cursor: - pass + cursor.execute( + f"with temporaryTable (averageValue) as (select avg(b) from {DB_SETTINGS['table_name']}) select * from {DB_SETTINGS['table_name']},temporaryTable where {DB_SETTINGS['table_name']}.b > temporaryTable.averageValue" + ) cursor.execute( - """update %s """ % DB_SETTINGS["table_name"] - + """set a=%s, b=%s, c=%s where a=%s""", + f"update {DB_SETTINGS['table_name']} set a=%s, b=%s, c=%s where a=%s", (4, 4.0, "4.0", 1), ) - cursor.execute("""delete from %s where a=2""" % DB_SETTINGS["table_name"]) + cursor.execute(f"""delete from {DB_SETTINGS['table_name']} where a=2""") connection.commit() @@ -152,7 +152,6 @@ def test_rollback_on_exception(): host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], ): - raise RuntimeError("error") except RuntimeError: diff --git a/tests/datastore_psycopg/conftest.py b/tests/datastore_psycopg/conftest.py new file mode 100644 index 0000000000..dd6a174d9e --- /dev/null +++ b/tests/datastore_psycopg/conftest.py @@ -0,0 +1,117 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from testing_support.db_settings import postgresql_settings +from testing_support.fixture.event_loop import event_loop as loop # noqa: F401 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_explain_plan_queries": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (datastore_psycopg)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) + + +DB_MULTIPLE_SETTINGS = postgresql_settings() +DB_SETTINGS = DB_MULTIPLE_SETTINGS[0] + + +@pytest.fixture(scope="session", params=["sync", "async"]) +def is_async(request): + return request.param == "async" + + +@pytest.fixture(scope="function") +def connection(loop, is_async): + import psycopg + + if not is_async: + connection = psycopg.connect( + dbname=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + password=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ) + else: + connection = loop.run_until_complete( + psycopg.AsyncConnection.connect( + dbname=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + password=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ) + ) + + yield connection + loop.run_until_complete(maybe_await(connection.close())) + + +@pytest.fixture(scope="function") +def multiple_connections(loop, is_async): + import psycopg + + if len(DB_MULTIPLE_SETTINGS) < 2: + pytest.skip(reason="Test environment not configured with multiple databases.") + + connections = [] + for DB_SETTINGS in DB_MULTIPLE_SETTINGS: + if not is_async: + connections.append( + psycopg.connect( + dbname=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + password=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ) + ) + else: + connections.append( + loop.run_until_complete( + psycopg.AsyncConnection.connect( + dbname=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + password=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ) + ) + ) + + yield connections + for connection in connections: + loop.run_until_complete(maybe_await(connection.close())) + + +async def maybe_await(value): + if hasattr(value, "__await__"): + return await value + + return value diff --git a/tests/datastore_psycopg/test_as_string.py b/tests/datastore_psycopg/test_as_string.py new file mode 100644 index 0000000000..675ce2c909 --- /dev/null +++ b/tests/datastore_psycopg/test_as_string.py @@ -0,0 +1,112 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +try: + from psycopg import sql +except ImportError: + sql = None + +from newrelic.api.background_task import background_task + + +@background_task() +def test_as_string_1(connection): + + # All of these are similar to those described in the doctests in + # psycopg/lib/sql.py + + comp = sql.Composed([sql.SQL("insert into "), sql.Identifier("table")]) + result = comp.as_string(connection) + assert result == 'insert into "table"' + + +@background_task() +def test_as_string_2(connection): + fields = sql.Identifier("foo") + sql.Identifier("bar") # a Composed + result = fields.join(", ").as_string(connection) + assert result == '"foo", "bar"' + + +@background_task() +def test_as_string_3(connection): + query = sql.SQL("select {0} from {1}").format( + sql.SQL(", ").join([sql.Identifier("foo"), sql.Identifier("bar")]), sql.Identifier("table") + ) + result = query.as_string(connection) + assert result == 'select "foo", "bar" from "table"' + + +@background_task() +def test_as_string_4(connection): + result = ( + sql.SQL("select * from {0} where {1} = %s") + .format(sql.Identifier("people"), sql.Identifier("id")) + .as_string(connection) + ) + assert result == 'select * from "people" where "id" = %s' + + +@background_task() +def test_as_string_5(connection): + result = ( + sql.SQL("select * from {tbl} where {pkey} = %s") + .format(tbl=sql.Identifier("people"), pkey=sql.Identifier("id")) + .as_string(connection) + ) + assert result == 'select * from "people" where "id" = %s' + + +@background_task() +def test_as_string_6(connection): + snip = sql.SQL(", ").join(sql.Identifier(n) for n in ["foo", "bar", "baz"]) + result = snip.as_string(connection) + assert result == '"foo", "bar", "baz"' + + +@background_task() +def test_as_string_7(connection): + t1 = sql.Identifier("foo") + t2 = sql.Identifier("ba'r") + t3 = sql.Identifier('ba"z') + result = sql.SQL(", ").join([t1, t2, t3]).as_string(connection) + assert result == '"foo", "ba\'r", "ba""z"' + + +@background_task() +def test_as_string_8(connection): + s1 = sql.Literal("foo") + s2 = sql.Literal("ba'r") + s3 = sql.Literal(42) + result = sql.SQL(", ").join([s1, s2, s3]).as_string(connection) + assert result == "'foo', 'ba''r', 42" + + +@background_task() +def test_as_string_9(connection): + names = ["foo", "bar", "baz"] + q1 = sql.SQL("insert into table ({0}) values ({1})").format( + sql.SQL(", ").join(map(sql.Identifier, names)), sql.SQL(", ").join(sql.Placeholder() * len(names)) + ) + result = q1.as_string(connection) + assert result == 'insert into table ("foo", "bar", "baz") values (%s, %s, %s)' + + +@background_task() +def test_as_string_10(connection): + names = ["foo", "bar", "baz"] + q2 = sql.SQL("insert into table ({0}) values ({1})").format( + sql.SQL(", ").join(map(sql.Identifier, names)), sql.SQL(", ").join(map(sql.Placeholder, names)) + ) + result = q2.as_string(connection) + assert result == 'insert into table ("foo", "bar", "baz") ' "values (%(foo)s, %(bar)s, %(baz)s)" diff --git a/tests/datastore_psycopg/test_connection.py b/tests/datastore_psycopg/test_connection.py new file mode 100644 index 0000000000..c554e30acf --- /dev/null +++ b/tests/datastore_psycopg/test_connection.py @@ -0,0 +1,220 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import psycopg +import pytest + +try: + from psycopg import sql +except ImportError: + sql = None + +from conftest import DB_SETTINGS, maybe_await +from testing_support.fixtures import override_application_settings +from testing_support.util import instance_hostname +from testing_support.validators.validate_database_trace_inputs import ( + validate_database_trace_inputs, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + +# Settings +_enable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": True, +} +_disable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": False, +} + + +# Metrics +_base_scoped_metrics = ( + ("Datastore/operation/Postgres/commit", 2), + ("Datastore/operation/Postgres/create", 2), + ("Datastore/operation/Postgres/drop", 1), + ("Datastore/operation/Postgres/rollback", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['procedure_name']}/call", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/delete", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 3), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/update", 1), +) + +_base_rollup_metrics = ( + ("Datastore/all", 14), + ("Datastore/allOther", 14), + ("Datastore/Postgres/all", 14), + ("Datastore/Postgres/allOther", 14), + ("Datastore/operation/Postgres/call", 1), + ("Datastore/operation/Postgres/commit", 2), + ("Datastore/operation/Postgres/create", 2), + ("Datastore/operation/Postgres/delete", 1), + ("Datastore/operation/Postgres/drop", 1), + ("Datastore/operation/Postgres/insert", 3), + ("Datastore/operation/Postgres/rollback", 1), + ("Datastore/operation/Postgres/select", 1), + ("Datastore/operation/Postgres/update", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['procedure_name']}/call", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/delete", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 3), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/update", 1), +) + +_disable_scoped_metrics = list(_base_scoped_metrics) +_disable_rollup_metrics = list(_base_rollup_metrics) + +_enable_scoped_metrics = list(_base_scoped_metrics) +_enable_rollup_metrics = list(_base_rollup_metrics) + +_host = instance_hostname(DB_SETTINGS["host"]) +_port = DB_SETTINGS["port"] + +_instance_metric_name = f"Datastore/instance/Postgres/{_host}/{_port}" + +_enable_rollup_metrics.append((_instance_metric_name, 13)) + +_disable_rollup_metrics.append((_instance_metric_name, None)) + + +# Query +async def _execute(connection, row_type, wrapper): + sql = f"drop table if exists {DB_SETTINGS['table_name']}" + await maybe_await(connection.execute(wrapper(sql))) + + sql = f"create table {DB_SETTINGS['table_name']} (a integer, b real, c text)" + await maybe_await(connection.execute(wrapper(sql))) + + for params in [(1, 1.0, "1.0"), (2, 2.2, "2.2"), (3, 3.3, "3.3")]: + sql = f"insert into {DB_SETTINGS['table_name']} values (%s, %s, %s)" + await maybe_await(connection.execute(wrapper(sql), params)) + + sql = f"select * from {DB_SETTINGS['table_name']}" + cursor = await maybe_await(connection.execute(wrapper(sql))) + + if hasattr(cursor, "__aiter__"): + async for row in cursor: + assert isinstance(row, row_type) + else: + for row in cursor: + assert isinstance(row, row_type) + + # Reuse cursor to ensure it is also wrapped + sql = f"update {DB_SETTINGS['table_name']} set a=%s, b=%s, c=%s where a=%s" + params = (4, 4.0, "4.0", 1) + await maybe_await(cursor.execute(wrapper(sql), params)) + + sql = f"delete from {DB_SETTINGS['table_name']} where a=2" + await maybe_await(connection.execute(wrapper(sql))) + + await maybe_await(connection.commit()) + + await maybe_await( + connection.execute( + f"create or replace procedure {DB_SETTINGS['procedure_name']}() \nlanguage plpgsql as $$ begin perform now(); end; $$" + ) + ) + await maybe_await(connection.execute(f"call {DB_SETTINGS['procedure_name']}()")) + + await maybe_await(connection.rollback()) + await maybe_await(connection.commit()) + + +async def _exercise_db(is_async, row_type=tuple, wrapper=str): + # Connect here instead of using the fixture to capture the FunctionTrace around connect + if not is_async: + connection = psycopg.connect( + dbname=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + password=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ) + else: + connection = await psycopg.AsyncConnection.connect( + dbname=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + password=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ) + + try: + await _execute(connection, row_type, wrapper) + finally: + await maybe_await(connection.close()) + + +_test_matrix = [ + "wrapper", + [ + str, + sql.SQL, + lambda q: sql.Composed([sql.SQL(q)]), + ], +] + + +# Tests +@pytest.mark.parametrize(*_test_matrix) +@override_application_settings(_enable_instance_settings) +def test_execute_via_connection_enable_instance(loop, is_async, wrapper): + if not is_async: + connect_metric = ("Function/psycopg:Connection.connect", 1) + else: + connect_metric = ("Function/psycopg:AsyncConnection.connect", 1) + + _scoped_metrics = list(_enable_scoped_metrics) + _scoped_metrics.append(connect_metric) + + @validate_transaction_metrics( + "test_execute_via_connection_enable_instance", + scoped_metrics=_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, + ) + @validate_database_trace_inputs(sql_parameters_type=tuple) + @background_task(name="test_execute_via_connection_enable_instance") + def test(): + loop.run_until_complete(_exercise_db(is_async, row_type=tuple, wrapper=wrapper)) + + test() + + +@pytest.mark.parametrize(*_test_matrix) +@override_application_settings(_disable_instance_settings) +def test_execute_via_connection_disable_instance(loop, is_async, wrapper): + if not is_async: + connect_metric = ("Function/psycopg:Connection.connect", 1) + else: + connect_metric = ("Function/psycopg:AsyncConnection.connect", 1) + + _scoped_metrics = list(_disable_scoped_metrics) + _scoped_metrics.append(connect_metric) + + @validate_transaction_metrics( + "test_execute_via_connection_disable_instance", + scoped_metrics=_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, + ) + @validate_database_trace_inputs(sql_parameters_type=tuple) + @background_task(name="test_execute_via_connection_disable_instance") + def test(): + loop.run_until_complete(_exercise_db(is_async, row_type=tuple, wrapper=wrapper)) + + test() diff --git a/tests/datastore_psycopg/test_cursor.py b/tests/datastore_psycopg/test_cursor.py new file mode 100644 index 0000000000..3eff3ed7ab --- /dev/null +++ b/tests/datastore_psycopg/test_cursor.py @@ -0,0 +1,240 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import psycopg +import pytest + +try: + from psycopg import sql +except ImportError: + sql = None + +from conftest import DB_SETTINGS, maybe_await +from testing_support.fixtures import override_application_settings +from testing_support.util import instance_hostname +from testing_support.validators.validate_database_trace_inputs import ( + validate_database_trace_inputs, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + +# Settings +_enable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": True, +} +_disable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": False, +} + + +# Metrics +_base_scoped_metrics = ( + ("Datastore/operation/Postgres/commit", 2), + ("Datastore/operation/Postgres/create", 2), + ("Datastore/operation/Postgres/drop", 1), + ("Datastore/operation/Postgres/rollback", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['procedure_name']}/call", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/delete", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/update", 1), +) + +_base_rollup_metrics = ( + ("Datastore/all", 11), + ("Datastore/allOther", 11), + ("Datastore/Postgres/all", 11), + ("Datastore/Postgres/allOther", 11), + ("Datastore/operation/Postgres/call", 1), + ("Datastore/operation/Postgres/commit", 2), + ("Datastore/operation/Postgres/create", 2), + ("Datastore/operation/Postgres/delete", 1), + ("Datastore/operation/Postgres/drop", 1), + ("Datastore/operation/Postgres/insert", 1), + ("Datastore/operation/Postgres/rollback", 1), + ("Datastore/operation/Postgres/select", 1), + ("Datastore/operation/Postgres/update", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['procedure_name']}/call", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/delete", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/update", 1), +) + +_disable_scoped_metrics = list(_base_scoped_metrics) +_disable_rollup_metrics = list(_base_rollup_metrics) + +_enable_scoped_metrics = list(_base_scoped_metrics) +_enable_rollup_metrics = list(_base_rollup_metrics) + +_host = instance_hostname(DB_SETTINGS["host"]) +_port = DB_SETTINGS["port"] + +_instance_metric_name = f"Datastore/instance/Postgres/{_host}/{_port}" + +_enable_rollup_metrics.append((_instance_metric_name, 11)) + +_disable_rollup_metrics.append((_instance_metric_name, None)) + + +# Query +async def _execute(connection, cursor, row_type, wrapper): + sql = f"drop table if exists {DB_SETTINGS['table_name']}" + await maybe_await(cursor.execute(wrapper(sql))) + + sql = f"create table {DB_SETTINGS['table_name']} (a integer, b real, c text)" + await maybe_await(cursor.execute(wrapper(sql))) + + sql = f"insert into {DB_SETTINGS['table_name']} values (%s, %s, %s) returning a, b, c" + params = [(1, 1.0, "1.0"), (2, 2.2, "2.2"), (3, 3.3, "3.3")] + await maybe_await(cursor.executemany(wrapper(sql), params, returning=True)) + + # Consume inserted records to check that returning param functions + records = [] + while True: + records.append(cursor.fetchone()) + if not cursor.nextset(): + break + assert len(records) == len(params) + + sql = f"select * from {DB_SETTINGS['table_name']}" + await maybe_await(cursor.execute(wrapper(sql))) + + if hasattr(cursor, "__aiter__"): + async for row in cursor: + assert isinstance(row, row_type) + else: + # Iterate on sync cursor + for row in cursor: + assert isinstance(row, row_type) + + sql = f"update {DB_SETTINGS['table_name']} set a=%s, b=%s, c=%s where a=%s" + params = (4, 4.0, "4.0", 1) + await maybe_await(cursor.execute(wrapper(sql), params)) + + sql = f"delete from {DB_SETTINGS['table_name']} where a=2" + await maybe_await(cursor.execute(wrapper(sql))) + + await maybe_await(connection.commit()) + + await maybe_await( + cursor.execute( + f"create or replace procedure {DB_SETTINGS['procedure_name']}() \nlanguage plpgsql as $$ begin perform now(); end; $$" + ) + ) + await maybe_await(cursor.execute(f"call {DB_SETTINGS['procedure_name']}()")) + + await maybe_await(connection.rollback()) + await maybe_await(connection.commit()) + + +async def _exercise_db(connection, row_factory=None, use_cur_context=False, row_type=tuple, wrapper=str): + kwargs = {"row_factory": row_factory} if row_factory else {} + + try: + cursor = connection.cursor(**kwargs) + if use_cur_context: + if hasattr(cursor, "__aenter__"): + async with cursor: + await _execute(connection, cursor, row_type, wrapper) + else: + with cursor: + await _execute(connection, cursor, row_type, wrapper) + else: + await _execute(connection, cursor, row_type, wrapper) + finally: + await maybe_await(connection.close()) + + +_test_matrix = [ + "wrapper,use_cur_context", + [ + (str, False), + (str, True), + (sql.SQL, False), + (sql.SQL, True), + (lambda q: sql.Composed([sql.SQL(q)]), False), + (lambda q: sql.Composed([sql.SQL(q)]), True), + ], +] + + +# Tests +@pytest.mark.parametrize(*_test_matrix) +@override_application_settings(_enable_instance_settings) +@validate_transaction_metrics( + "test_cursor:test_execute_via_cursor_enable_instance", + scoped_metrics=_enable_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) +@validate_database_trace_inputs(sql_parameters_type=tuple) +@background_task() +def test_execute_via_cursor_enable_instance(loop, connection, wrapper, use_cur_context): + loop.run_until_complete(_exercise_db(connection, use_cur_context=use_cur_context, row_type=tuple, wrapper=wrapper)) + + +@pytest.mark.parametrize(*_test_matrix) +@override_application_settings(_disable_instance_settings) +@validate_transaction_metrics( + "test_cursor:test_execute_via_cursor_disable_instance", + scoped_metrics=_disable_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) +@validate_database_trace_inputs(sql_parameters_type=tuple) +@background_task() +def test_execute_via_cursor_disable_instance(loop, connection, wrapper, use_cur_context): + loop.run_until_complete(_exercise_db(connection, use_cur_context=use_cur_context, row_type=tuple, wrapper=wrapper)) + + +@pytest.mark.parametrize(*_test_matrix) +@override_application_settings(_enable_instance_settings) +@validate_transaction_metrics( + "test_cursor:test_execute_via_cursor_dict_enable_instance", + scoped_metrics=_enable_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) +@validate_database_trace_inputs(sql_parameters_type=tuple) +@background_task() +def test_execute_via_cursor_dict_enable_instance(loop, connection, wrapper, use_cur_context): + dict_factory = psycopg.rows.dict_row + loop.run_until_complete( + _exercise_db( + connection, row_factory=dict_factory, use_cur_context=use_cur_context, row_type=dict, wrapper=wrapper + ) + ) + + +@pytest.mark.parametrize(*_test_matrix) +@override_application_settings(_disable_instance_settings) +@validate_transaction_metrics( + "test_cursor:test_execute_via_cursor_dict_disable_instance", + scoped_metrics=_disable_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) +@validate_database_trace_inputs(sql_parameters_type=tuple) +@background_task() +def test_execute_via_cursor_dict_disable_instance(loop, connection, wrapper, use_cur_context): + dict_factory = psycopg.rows.dict_row + loop.run_until_complete( + _exercise_db( + connection, row_factory=dict_factory, use_cur_context=use_cur_context, row_type=dict, wrapper=wrapper + ) + ) diff --git a/tests/datastore_psycopg/test_database_instance_info.py b/tests/datastore_psycopg/test_database_instance_info.py new file mode 100644 index 0000000000..1caa950471 --- /dev/null +++ b/tests/datastore_psycopg/test_database_instance_info.py @@ -0,0 +1,226 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest + +from newrelic.hooks.database_psycopg import ( + _add_defaults, + _parse_connect_params, + instance_info, +) + + +def test_kwargs(): + connect_params = ((), {"dbname": "foo", "host": "1.2.3.4", "port": 1234}) + output = _parse_connect_params(*connect_params) + assert output == ("1.2.3.4", None, "1234", "foo") + + +def test_arg_str(): + connect_params = (("host=foobar port=9876",), {}) + output = _parse_connect_params(*connect_params) + assert output == ("foobar", None, "9876", None) + + +def test_bind_conninfo(): + connect_params = ((), {"conninfo": "host=foobar port=9876"}) + output = _parse_connect_params(*connect_params) + assert output == ("foobar", None, "9876", None) + + +def test_bind_conninfo_ignore_kwargs(): + connect_params = ((), {"conninfo": "host=foobar", "port": 1234}) + output = _parse_connect_params(*connect_params) + assert output == ("foobar", None, None, None) + + +def test_kwargs_str_for_port(): + connect_params = ((), {"dbname": "foo", "host": "1.2.3.4", "port": "1234"}) + output = _parse_connect_params(*connect_params) + assert output == ("1.2.3.4", None, "1234", "foo") + + +def test_arg_str_missing_port(): + connect_params = (("host=foobar",), {}) + output = _parse_connect_params(*connect_params) + assert output == ("foobar", None, None, None) + + +def test_arg_str_multiple_host(): + connect_params = (("host=foobar host=barbaz",), {}) + output = _parse_connect_params(*connect_params) + assert output == ("barbaz", None, None, None) + + +def test_arg_str_multiple_port(): + connect_params = (("port=5555 port=7777",), {}) + output = _parse_connect_params(*connect_params) + assert output == (None, None, "7777", None) + + +def test_arg_str_missing_host(): + connect_params = (("port=5555",), {}) + output = _parse_connect_params(*connect_params) + assert output == (None, None, "5555", None) + + +def test_arg_str_missing_host_and_port(): + connect_params = (("nothing=here",), {}) + output = _parse_connect_params(*connect_params) + assert output == (None, None, None, None) + + +def test_malformed_arg_str(): + connect_params = (("this_is_malformed",), {}) + output = _parse_connect_params(*connect_params) + assert output == ("unknown", "unknown", "unknown", "unknown") + + +def test_str_in_port_arg_str(): + connect_params = (("port=foobar",), {}) + output = _parse_connect_params(*connect_params) + assert output == (None, None, "foobar", None) + + +def test_host_and_hostaddr_in_arg_str(): + connect_params = (("host=foobar hostaddr=1.2.3.4",), {}) + output = _parse_connect_params(*connect_params) + assert output == ("foobar", "1.2.3.4", None, None) + + +def test_host_and_hostaddr_in_kwarg(): + connect_params = ((), {"host": "foobar", "hostaddr": "1.2.3.4"}) + output = _parse_connect_params(*connect_params) + assert output == ("foobar", "1.2.3.4", None, None) + + +def test_only_hostaddr_in_kwarg(): + connect_params = ((), {"hostaddr": "1.2.3.4"}) + output = _parse_connect_params(*connect_params) + assert output == (None, "1.2.3.4", None, None) + + +def test_only_hostaddr_in_arg_str(): + connect_params = (("hostaddr=1.2.3.4",), {}) + output = _parse_connect_params(*connect_params) + assert output == (None, "1.2.3.4", None, None) + + +def test_env_var_default_host(monkeypatch): + monkeypatch.setenv("PGHOST", "envfoo") + output = _add_defaults(None, None, "1234", "foo") + assert output == ("envfoo", "1234", "foo") + + +def test_env_var_default_hostaddr(monkeypatch): + monkeypatch.setenv("PGHOSTADDR", "1.2.3.4") + output = _add_defaults(None, None, "1234", "foo") + assert output == ("1.2.3.4", "1234", "foo") + + +def test_env_var_default_database(monkeypatch): + monkeypatch.setenv("PGDATABASE", "dbenvfoo") + output = _add_defaults("foo", None, "1234", None) + assert output == ("foo", "1234", "dbenvfoo") + + +def test_env_var_default_port(monkeypatch): + monkeypatch.setenv("PGPORT", "9999") + output = _add_defaults("foo", None, None, "bar") + assert output == ("foo", "9999", "bar") + + +@pytest.mark.parametrize( + "connect_params,expected", + [ + ((("postgresql://",), {}), ("localhost", "default", "default")), + ((("postgresql://localhost",), {}), ("localhost", "5432", "default")), + ((("postgresql://localhost:5433",), {}), ("localhost", "5433", "default")), + ((("postgresql://localhost/mydb",), {}), ("localhost", "5432", "mydb")), + ((("postgresql://user@localhost",), {}), ("localhost", "5432", "default")), + ((("postgresql://user:secret@localhost",), {}), ("localhost", "5432", "default")), + ((("postgresql://[2001:db8::1234]/database",), {}), ("2001:db8::1234", "5432", "database")), + ((("postgresql://[2001:db8::1234]:2222/database",), {}), ("2001:db8::1234", "2222", "database")), + ( + (("postgresql:///dbname?host=/var/lib/postgresql",), {}), + ("localhost", "/var/lib/postgresql/.s.PGSQL.5432", "dbname"), + ), + ( + (("postgresql://%2Fvar%2Flib%2Fpostgresql/dbname",), {}), + ("localhost", "/var/lib/postgresql/.s.PGSQL.5432", "dbname"), + ), + ((("postgresql://other@localhost/otherdb?c=10&a=myapp",), {}), ("localhost", "5432", "otherdb")), + ((("postgresql:///",), {}), ("localhost", "default", "default")), + ((("postgresql:///dbname?host=foo",), {}), ("foo", "5432", "dbname")), + ((("postgresql:///dbname?port=1234",), {}), ("localhost", "default", "dbname")), + ((("postgresql:///dbname?host=foo&port=1234",), {}), ("foo", "1234", "dbname")), + ((("postgres:///dbname?host=foo&port=1234",), {}), ("foo", "1234", "dbname")), + ((("postgres://localhost:5444/blah?host=::1",), {}), ("::1", "5444", "blah")), + ((("postgresql:///dbname?host=foo&port=1234&host=bar",), {}), ("bar", "1234", "dbname")), + ((("postgresql://%2Ftmp:1234",), {}), ("localhost", "/tmp/.s.PGSQL.1234", "default")), + ((("postgresql:///foo?dbname=bar",), {}), ("localhost", "default", "bar")), + ((("postgresql://example.com/foo?hostaddr=1.2.3.4&host=bar",), {}), ("1.2.3.4", "5432", "foo")), + ], +) +def test_uri(connect_params, expected): + output = instance_info(*connect_params) + assert output == expected + + +@pytest.mark.parametrize( + "connect_params,expected", + [ + ((("postgresql://user:password@/?dbname=bar",), {}), ("localhost", "default", "bar")), + ((("postgresql://user:pass@host/?dbname=bar",), {}), ("host", "5432", "bar")), + ((("postgresql://user:password@@/?dbname=bar",), {}), ("localhost", "default", "bar")), + ((("postgresql://@",), {}), ("localhost", "default", "default")), + ((("postgresql://@@localhost",), {}), ("localhost", "5432", "default")), + ], +) +def test_security_sensitive_uri(connect_params, expected): + output = instance_info(*connect_params) + assert output == expected + + +def test_bad_uri(): + connect_params = (("blah:///foo",), {}) + output = instance_info(*connect_params) + assert output == ("unknown", "unknown", "unknown") + + +_test_add_defaults = [ + # TCP/IP + [("otherhost.com", None, "8888", "foobar"), ("otherhost.com", "8888", "foobar")], + [("otherhost.com", None, None, "foobar"), ("otherhost.com", "5432", "foobar")], + [("localhost", None, "8888", "foobar"), ("localhost", "8888", "foobar")], + [("localhost", None, None, "foobar"), ("localhost", "5432", "foobar")], + [("127.0.0.1", None, "8888", "foobar"), ("127.0.0.1", "8888", "foobar")], + [("127.0.0.1", None, None, "foobar"), ("127.0.0.1", "5432", "foobar")], + [("::1", None, "8888", None), ("::1", "8888", "default")], + [("::1", None, None, None), ("::1", "5432", "default")], + [("::1", None, None, ""), ("::1", "5432", "default")], + # Unix Domain Socket + [(None, None, None, None), ("localhost", "default", "default")], + [(None, None, "5432", None), ("localhost", "default", "default")], + [(None, None, "8888", None), ("localhost", "default", "default")], + [("/tmp", None, None, "cat"), ("localhost", "/tmp/.s.PGSQL.5432", "cat")], + [("/tmp", None, "5432", "dog"), ("localhost", "/tmp/.s.PGSQL.5432", "dog")], + [("/tmp", None, "8888", "db"), ("localhost", "/tmp/.s.PGSQL.8888", "db")], +] + + +@pytest.mark.parametrize("host_port,expected", _test_add_defaults) +def test_add_defaults(host_port, expected): + actual = _add_defaults(*host_port) + assert actual == expected diff --git a/tests/datastore_psycopg/test_explain_plans.py b/tests/datastore_psycopg/test_explain_plans.py new file mode 100644 index 0000000000..e02a04693f --- /dev/null +++ b/tests/datastore_psycopg/test_explain_plans.py @@ -0,0 +1,201 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import threading + +import psycopg +import pytest +from conftest import DB_SETTINGS, maybe_await +from testing_support.fixtures import override_application_settings +from testing_support.validators.validate_database_node import validate_database_node +from testing_support.validators.validate_transaction_slow_sql_count import ( + validate_transaction_slow_sql_count, +) + +from newrelic.api.background_task import background_task +from newrelic.core.database_utils import SQLConnections + + +class CustomCursor(psycopg.Cursor): + event = threading.Event() + + def execute(self, *args, **kwargs): + self.event.set() + return super().execute(*args, **kwargs) + + +class CustomAsyncCursor(psycopg.AsyncCursor): + event = threading.Event() + + async def execute(self, *args, **kwargs): + self.event.set() + return await super().execute(*args, **kwargs) + + +class CustomConnection(psycopg.Connection): + event = threading.Event() + + def cursor(self, *args, **kwargs): + self.event.set() + return super().cursor(*args, **kwargs) + + +class CustomAsyncConnection(psycopg.AsyncConnection): + event = threading.Event() + + def cursor(self, *args, **kwargs): + self.event.set() + return super().cursor(*args, **kwargs) + + +def reset_events(): + # Reset all event flags + CustomCursor.event.clear() + CustomAsyncCursor.event.clear() + CustomConnection.event.clear() + CustomAsyncConnection.event.clear() + + +async def _exercise_db(connection, cursor_kwargs=None): + cursor_kwargs = cursor_kwargs or {} + + try: + cursor = connection.cursor(**cursor_kwargs) + + await maybe_await(cursor.execute("SELECT setting from pg_settings where name=%s", ("server_version",))) + finally: + await maybe_await(connection.close()) + + +# Tests + + +def explain_plan_is_not_none(node): + with SQLConnections() as connections: + explain_plan = node.explain_plan(connections) + + assert explain_plan is not None + + +SCROLLABLE = (True, False) +WITHHOLD = (True, False) + + +@pytest.mark.parametrize("withhold", WITHHOLD) +@pytest.mark.parametrize("scrollable", SCROLLABLE) +@override_application_settings( + { + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.record_sql": "raw", + } +) +@validate_database_node(explain_plan_is_not_none) +@validate_transaction_slow_sql_count(1) +@background_task(name="test_explain_plan_unnamed_cursors") +def test_explain_plan_unnamed_cursors(loop, connection, withhold, scrollable): + cursor_kwargs = {} + + if withhold: + cursor_kwargs["withhold"] = withhold + + if scrollable: + cursor_kwargs["scrollable"] = scrollable + + loop.run_until_complete(_exercise_db(connection, cursor_kwargs=cursor_kwargs)) + + +@pytest.mark.parametrize("withhold", WITHHOLD) +@pytest.mark.parametrize("scrollable", SCROLLABLE) +@override_application_settings( + { + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.record_sql": "raw", + } +) +@validate_database_node(explain_plan_is_not_none) +@validate_transaction_slow_sql_count(1) +@background_task(name="test_explain_plan_named_cursors") +def test_explain_plan_named_cursors(loop, connection, withhold, scrollable): + cursor_kwargs = { + "name": "test_explain_plan_named_cursors", + } + + if withhold: + cursor_kwargs["withhold"] = withhold + + if scrollable: + cursor_kwargs["scrollable"] = scrollable + + loop.run_until_complete(_exercise_db(connection, cursor_kwargs=cursor_kwargs)) + + +# This test validates that any combination of sync or async, and default or custom connection and cursor classes will work with +# the explain plan feature. The agent should always use psycopg.connect to open a new explain plan connection and only +# use custom cursors from synchronous connections, as async cursors will not be compatible. +@pytest.mark.parametrize( + "connection_cls,cursor_cls", + [ + (psycopg.Connection, psycopg.Cursor), + (psycopg.Connection, CustomCursor), + (CustomConnection, psycopg.Cursor), + (CustomConnection, CustomCursor), + (psycopg.AsyncConnection, psycopg.AsyncCursor), + (psycopg.AsyncConnection, CustomAsyncCursor), + (CustomAsyncConnection, psycopg.AsyncCursor), + (CustomAsyncConnection, CustomAsyncCursor), + ], +) +@override_application_settings( + { + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.record_sql": "raw", + } +) +def test_explain_plan_on_custom_classes(loop, connection_cls, cursor_cls): + @validate_database_node(explain_plan_is_not_none) + @validate_transaction_slow_sql_count(1) + @background_task(name="test_explain_plan_on_custom_connect_class") + def test(): + async def coro(): + # Connect using custom Connection classes, so connect here without the fixture. + connection = await maybe_await( + connection_cls.connect( + dbname=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + password=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + cursor_factory=cursor_cls, + ) + ) + await _exercise_db(connection) + reset_events() + + loop.run_until_complete(coro()) + + test() + + # Check that the correct classes were used AFTER the explain plan validator has run + if hasattr(connection_cls, "event"): + assert not connection_cls.event.is_set(), "Custom connection class should not be used." + if hasattr(cursor_cls, "event"): + if cursor_cls is not CustomAsyncCursor: + assert cursor_cls.event.is_set(), "Custom cursor class was not used." + else: + assert not cursor_cls.event.is_set(), "Custom async cursor class should not be used." + + +# This test will verify that arguments are preserved for an explain +# plan by forcing a failure to be generated when explain plans are created and +# arguments are preserved diff --git a/tests/datastore_psycopg/test_forward_compat.py b/tests/datastore_psycopg/test_forward_compat.py new file mode 100644 index 0000000000..520dd35df3 --- /dev/null +++ b/tests/datastore_psycopg/test_forward_compat.py @@ -0,0 +1,41 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import psycopg + +from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.hooks.database_psycopg import wrapper_psycopg_as_string + + +class TestCompatability(): + def as_string(self, giraffe, lion, tiger=None): + assert type(giraffe) in (psycopg.Cursor, psycopg.AsyncCursor) + return "PASS" + + +wrap_function_wrapper(__name__, "TestCompatability.as_string", wrapper_psycopg_as_string) + + +def test_forward_compat_args(connection): + cursor = connection.cursor() + query = TestCompatability() + result = query.as_string(cursor, "giraffe-nomming-leaves") + assert result == "PASS" + + +def test_forward_compat_kwargs(connection): + cursor = connection.cursor() + query = TestCompatability() + result = query.as_string(cursor, lion="eats tiger", tiger="eats giraffe") + assert result == "PASS" diff --git a/tests/datastore_psycopg/test_multiple_dbs.py b/tests/datastore_psycopg/test_multiple_dbs.py new file mode 100644 index 0000000000..ae720c6611 --- /dev/null +++ b/tests/datastore_psycopg/test_multiple_dbs.py @@ -0,0 +1,138 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from conftest import DB_MULTIPLE_SETTINGS, DB_SETTINGS, maybe_await +from testing_support.fixtures import override_application_settings +from testing_support.util import instance_hostname +from testing_support.validators.validate_database_trace_inputs import ( + validate_database_trace_inputs, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + +# Settings + +_enable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": True, +} +_disable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": False, +} + + +# Metrics + +_base_scoped_metrics = [ + ("Datastore/statement/Postgres/pg_settings/select", 1), + ("Datastore/operation/Postgres/drop", 1), + ("Datastore/operation/Postgres/create", 1), + ("Datastore/operation/Postgres/commit", 2), +] + +_base_rollup_metrics = [ + ("Datastore/all", 5), + ("Datastore/allOther", 5), + ("Datastore/Postgres/all", 5), + ("Datastore/Postgres/allOther", 5), + ("Datastore/statement/Postgres/pg_settings/select", 1), + ("Datastore/operation/Postgres/drop", 1), + ("Datastore/operation/Postgres/create", 1), + ("Datastore/operation/Postgres/commit", 2), +] + +_enable_scoped_metrics = list(_base_scoped_metrics) +_enable_rollup_metrics = list(_base_rollup_metrics) + +_disable_scoped_metrics = list(_base_scoped_metrics) +_disable_rollup_metrics = list(_base_rollup_metrics) + +_postgresql_1 = DB_MULTIPLE_SETTINGS[0] +_host_1 = instance_hostname(_postgresql_1["host"]) +_port_1 = _postgresql_1["port"] + +_postgresql_2 = DB_MULTIPLE_SETTINGS[1] +_host_2 = instance_hostname(_postgresql_2["host"]) +_port_2 = _postgresql_2["port"] + +_instance_metric_name_1 = f"Datastore/instance/Postgres/{_host_1}/{_port_1}" +_instance_metric_name_2 = f"Datastore/instance/Postgres/{_host_2}/{_port_2}" + +_enable_rollup_metrics.extend( + [ + (_instance_metric_name_1, 2), + (_instance_metric_name_2, 3), + ] +) +_disable_rollup_metrics.extend( + [ + (_instance_metric_name_1, None), + (_instance_metric_name_2, None), + ] +) + + +# Query + + +async def _exercise_db(multiple_connections): + connection = multiple_connections[0] + try: + cursor = connection.cursor() + await maybe_await(cursor.execute("SELECT setting from pg_settings where name=%s", ("server_version",))) + await maybe_await(connection.commit()) + finally: + await maybe_await(connection.close()) + + connection = multiple_connections[1] + try: + cursor = connection.cursor() + await maybe_await(cursor.execute(f"drop table if exists {DB_SETTINGS['table_name']}")) + await maybe_await( + cursor.execute(f"create table {DB_SETTINGS['table_name']} (a integer, b real, c text)") + ) + await maybe_await(connection.commit()) + finally: + await maybe_await(connection.close()) + + +# Tests + + +@override_application_settings(_enable_instance_settings) +@validate_transaction_metrics( + "test_multiple_dbs:test_multiple_databases_enable_instance", + scoped_metrics=_enable_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) +@validate_database_trace_inputs(sql_parameters_type=tuple) +@background_task() +def test_multiple_databases_enable_instance(loop, multiple_connections): + loop.run_until_complete(_exercise_db(multiple_connections)) + + +@override_application_settings(_disable_instance_settings) +@validate_transaction_metrics( + "test_multiple_dbs:test_multiple_databases_disable_instance", + scoped_metrics=_disable_scoped_metrics, + rollup_metrics=_disable_scoped_metrics, + background_task=True, +) +@validate_database_trace_inputs(sql_parameters_type=tuple) +@background_task() +def test_multiple_databases_disable_instance(loop, multiple_connections): + loop.run_until_complete(_exercise_db(multiple_connections)) diff --git a/tests/datastore_psycopg/test_obfuscation.py b/tests/datastore_psycopg/test_obfuscation.py new file mode 100644 index 0000000000..1a656967a9 --- /dev/null +++ b/tests/datastore_psycopg/test_obfuscation.py @@ -0,0 +1,139 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from conftest import DB_SETTINGS, maybe_await +from testing_support.validators.validate_database_node import validate_database_node +from testing_support.validators.validate_sql_obfuscation import validate_sql_obfuscation + +from newrelic.api.background_task import background_task +from newrelic.core.database_utils import SQLConnections + + +@pytest.fixture() +def cursor(loop, connection): + try: + cursor = connection.cursor() + + loop.run_until_complete(maybe_await(cursor.execute(f"drop table if exists {DB_SETTINGS['table_name']}"))) + loop.run_until_complete( + maybe_await(cursor.execute(f"create table {DB_SETTINGS['table_name']} (b text, c text)")) + ) + + yield cursor + + finally: + loop.run_until_complete(maybe_await(connection.close())) + + +_quoting_style_tests = [ + ( + f"SELECT * FROM {DB_SETTINGS['table_name']} WHERE b='2'", + f"SELECT * FROM {DB_SETTINGS['table_name']} WHERE b=?", + ), + ( + f"SELECT * FROM {DB_SETTINGS['table_name']} WHERE b=$func$2$func$", + f"SELECT * FROM {DB_SETTINGS['table_name']} WHERE b=?", + ), + ( + f"SELECT * FROM {DB_SETTINGS['table_name']} WHERE b=U&'2'", + f"SELECT * FROM {DB_SETTINGS['table_name']} WHERE b=U&?", + ), +] + + +@pytest.mark.parametrize("sql,obfuscated", _quoting_style_tests) +def test_obfuscation_quoting_styles(loop, cursor, sql, obfuscated): + @validate_sql_obfuscation([obfuscated]) + @background_task() + def test(): + loop.run_until_complete(maybe_await(cursor.execute(sql))) + + test() + + +_parameter_tests = [ + ( + f"SELECT * FROM {DB_SETTINGS['table_name']} where b=%s", + f"SELECT * FROM {DB_SETTINGS['table_name']} where b=%s", + ), +] + + +@pytest.mark.parametrize("sql,obfuscated", _parameter_tests) +def test_obfuscation_parameters(loop, cursor, sql, obfuscated): + @validate_sql_obfuscation([obfuscated]) + @background_task() + def test(): + loop.run_until_complete(maybe_await(cursor.execute(sql, ("hello",)))) + + test() + + +def no_explain_plan(node): + sql_connections = SQLConnections() + explain_plan = node.explain_plan(sql_connections) + assert explain_plan is None + + +def any_length_explain_plan(node): + if node.statement.operation != "select": + return + + sql_connections = SQLConnections() + explain_plan = node.explain_plan(sql_connections) + assert explain_plan and len(explain_plan) > 0 + + +_test_explain_plans = [ + ( + f"SELECT (b, c) FROM {DB_SETTINGS['table_name']} ; SELECT (b, c) FROM {DB_SETTINGS['table_name']}", + no_explain_plan, + ), + ( + f"SELECT (b, c) FROM {DB_SETTINGS['table_name']} ; SELECT (b, c) FROM {DB_SETTINGS['table_name']};", + no_explain_plan, + ), + (f"SELECT (b, c) FROM {DB_SETTINGS['table_name']} WHERE b=';'", no_explain_plan), + (f";SELECT (b, c) FROM {DB_SETTINGS['table_name']}", no_explain_plan), + (f"SELECT (b, c) FROM {DB_SETTINGS['table_name']}", any_length_explain_plan), + (f"SELECT (b, c) FROM {DB_SETTINGS['table_name']};", any_length_explain_plan), + ( + f"SELECT (b, c) FROM {DB_SETTINGS['table_name']};;;;;;", + any_length_explain_plan, + ), + ( + f"SELECT (b, c) FROM {DB_SETTINGS['table_name']};\n\n", + any_length_explain_plan, + ), +] + + +@pytest.mark.parametrize("sql,validator", _test_explain_plans) +def test_obfuscation_explain_plans(loop, connection, sql, validator): + @validate_database_node(validator) + @background_task() + async def test(): + try: + cursor = connection.cursor() + await maybe_await(cursor.execute(f"drop table if exists {DB_SETTINGS['table_name']}")) + await maybe_await(cursor.execute(f"create table {DB_SETTINGS['table_name']} (b text, c text)")) + + await maybe_await(cursor.execute(sql)) + + finally: + await maybe_await(connection.commit()) + await maybe_await(connection.close()) + + loop.run_until_complete(test()) diff --git a/tests/datastore_psycopg/test_register.py b/tests/datastore_psycopg/test_register.py new file mode 100644 index 0000000000..4a5113baea --- /dev/null +++ b/tests/datastore_psycopg/test_register.py @@ -0,0 +1,85 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import psycopg +from conftest import maybe_await +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + + +@validate_transaction_metrics("test_register:test_register_json", background_task=True) +@validate_transaction_errors(errors=[]) +@background_task() +def test_register_json(loop, connection): + def test(): + cursor = connection.cursor() + + psycopg.types.json.set_json_loads(loads=lambda x: x, context=connection) + psycopg.types.json.set_json_loads(loads=lambda x: x, context=cursor) + + if hasattr(connection, "__aenter__"): + + async def coro(): + async with connection: + test() + + loop.run_until_complete(coro()) + else: + with connection: + test() + + +@validate_transaction_metrics("test_register:test_register_range", background_task=True) +@validate_transaction_errors(errors=[]) +@background_task() +def test_register_range(loop, connection): + async def test(): + type_name = f"floatrange_{str(os.getpid())}" + + create_sql = f"CREATE TYPE {type_name} AS RANGE (subtype = float8,subtype_diff = float8mi)" + + cursor = connection.cursor() + + await maybe_await(cursor.execute(f"DROP TYPE if exists {type_name}")) + await maybe_await(cursor.execute(create_sql)) + + range_type_info = await maybe_await(psycopg.types.range.RangeInfo.fetch(connection, type_name)) + range_type_info.register(connection) + + await maybe_await(cursor.execute(f"DROP TYPE if exists {type_name}")) + await maybe_await(cursor.execute(create_sql)) + + range_type_info = await maybe_await(psycopg.types.range.RangeInfo.fetch(connection, type_name)) + range_type_info.register(cursor) + + await maybe_await(cursor.execute(f"DROP TYPE if exists {type_name}")) + + if hasattr(connection, "__aenter__"): + + async def coro(): + async with connection: + await test() + + loop.run_until_complete(coro()) + else: + with connection: + loop.run_until_complete(test()) diff --git a/tests/datastore_psycopg/test_rollback.py b/tests/datastore_psycopg/test_rollback.py new file mode 100644 index 0000000000..57f01ac3f6 --- /dev/null +++ b/tests/datastore_psycopg/test_rollback.py @@ -0,0 +1,104 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from conftest import DB_SETTINGS +from testing_support.fixtures import override_application_settings +from testing_support.util import instance_hostname +from testing_support.validators.validate_database_trace_inputs import ( + validate_database_trace_inputs, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + +# Settings + +_enable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": True, +} +_disable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": False, +} + +# Metrics + +_base_scoped_metrics = (("Datastore/operation/Postgres/rollback", 1),) + +_base_rollup_metrics = ( + ("Datastore/all", 1), + ("Datastore/allOther", 1), + ("Datastore/Postgres/all", 1), + ("Datastore/Postgres/allOther", 1), + ("Datastore/operation/Postgres/rollback", 1), +) + +_enable_scoped_metrics = list(_base_scoped_metrics) +_enable_rollup_metrics = list(_base_rollup_metrics) + +_disable_scoped_metrics = list(_base_scoped_metrics) +_disable_rollup_metrics = list(_base_rollup_metrics) + +_host = instance_hostname(DB_SETTINGS["host"]) +_port = DB_SETTINGS["port"] + +_instance_metric_name = f"Datastore/instance/Postgres/{_host}/{_port}" + +_enable_rollup_metrics.append((_instance_metric_name, 1)) + +_disable_rollup_metrics.append((_instance_metric_name, None)) + +# Query + + +async def _exercise_db(connection): + try: + if hasattr(connection, "__aenter__"): + async with connection: + raise RuntimeError("error") + else: + with connection: + raise RuntimeError("error") + except RuntimeError: + pass + + +# Tests + + +@override_application_settings(_enable_instance_settings) +@validate_transaction_metrics( + "test_rollback:test_rollback_on_exception_enable_instance", + scoped_metrics=_enable_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) +@validate_database_trace_inputs(sql_parameters_type=tuple) +@background_task() +def test_rollback_on_exception_enable_instance(loop, connection): + loop.run_until_complete(_exercise_db(connection)) + + +@override_application_settings(_disable_instance_settings) +@validate_transaction_metrics( + "test_rollback:test_rollback_on_exception_disable_instance", + scoped_metrics=_disable_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) +@validate_database_trace_inputs(sql_parameters_type=tuple) +@background_task() +def test_rollback_on_exception_disable_instance(loop, connection): + loop.run_until_complete(_exercise_db(connection)) diff --git a/tests/datastore_psycopg/test_slow_sql.py b/tests/datastore_psycopg/test_slow_sql.py new file mode 100644 index 0000000000..abd2c31cc1 --- /dev/null +++ b/tests/datastore_psycopg/test_slow_sql.py @@ -0,0 +1,135 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from conftest import maybe_await +from testing_support.fixtures import override_application_settings +from testing_support.validators.validate_slow_sql_collector_json import ( + validate_slow_sql_collector_json, +) + +from newrelic.api.background_task import background_task +from newrelic.api.transaction import current_transaction + +# Settings + +_enable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": True, + "datastore_tracer.database_name_reporting.enabled": True, +} +_disable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": False, + "datastore_tracer.database_name_reporting.enabled": False, +} + +# Expected parameters + +_enabled_required = set(["host", "port_path_or_id", "database_name"]) +_enabled_forgone = set() + +_disabled_required = set() +_disabled_forgone = set(["host", "port_path_or_id", "database_name"]) + +# Guid is always required, regardless of DT status. +# It should be excluded from the forgone params set. +_distributed_tracing_required_params = set(["guid", "traceId", "priority", "sampled"]) +_distributed_tracing_forgone_params = set(["traceId", "priority", "sampled"]) +_distributed_tracing_payload_received_params = set( + ["parent.type", "parent.app", "parent.account", "parent.transportType", "parent.transportDuration"] +) + +_transaction_guid = "1234567890" +_distributed_tracing_exact_params = {"guid": _transaction_guid} + + +# Query + + +async def _exercise_db(connection): + try: + cursor = connection.cursor() + await maybe_await(cursor.execute("SELECT setting from pg_settings where name=%s", ("server_version",))) + finally: + await maybe_await(connection.close()) + + +# Tests + + +@pytest.mark.parametrize("instance_enabled", (True, False)) +@pytest.mark.parametrize( + "distributed_tracing_enabled,payload_received", + [ + (True, True), + (True, False), + (False, False), + ], +) +def test_slow_sql_json(loop, connection, instance_enabled, distributed_tracing_enabled, payload_received): + + exact_params = None + + if instance_enabled: + settings = _enable_instance_settings.copy() + required_params = set(_enabled_required) + forgone_params = set(_enabled_forgone) + else: + settings = _disable_instance_settings.copy() + required_params = set(_disabled_required) + forgone_params = set(_disabled_forgone) + + if distributed_tracing_enabled: + required_params.update(_distributed_tracing_required_params) + exact_params = _distributed_tracing_exact_params + settings["distributed_tracing.enabled"] = True + if payload_received: + required_params.update(_distributed_tracing_payload_received_params) + else: + forgone_params.update(_distributed_tracing_payload_received_params) + else: + forgone_params.update(_distributed_tracing_forgone_params) + forgone_params.update(_distributed_tracing_payload_received_params) + settings["distributed_tracing.enabled"] = False + + @override_application_settings(settings) + @validate_slow_sql_collector_json( + required_params=required_params, forgone_params=forgone_params, exact_params=exact_params + ) + @background_task() + def _test(): + transaction = current_transaction() + transaction.guid = _transaction_guid + + loop.run_until_complete(_exercise_db(connection)) + + if payload_received: + + payload = { + "v": [0, 1], + "d": { + "ty": "Mobile", + "ac": transaction.settings.account_id, + "tk": transaction.settings.trusted_account_key, + "ap": "2827902", + "pa": "5e5733a911cfbc73", + "id": "7d3efb1b173fecfa", + "tr": "d6b4ba0c3a712ca", + "ti": 1518469636035, + "tx": "8703ff3d88eefe9d", + }, + } + + transaction.accept_distributed_trace_payload(payload) + + _test() diff --git a/tests/datastore_psycopg/test_span_event.py b/tests/datastore_psycopg/test_span_event.py new file mode 100644 index 0000000000..df68f79662 --- /dev/null +++ b/tests/datastore_psycopg/test_span_event.py @@ -0,0 +1,136 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from conftest import DB_SETTINGS, maybe_await +from testing_support.fixtures import override_application_settings +from testing_support.util import instance_hostname +from testing_support.validators.validate_span_events import validate_span_events + +from newrelic.api.background_task import background_task +from newrelic.api.transaction import current_transaction + +# Settings + +_enable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": True, + "datastore_tracer.database_name_reporting.enabled": True, + "distributed_tracing.enabled": True, + "span_events.enabled": True, +} +_disable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": False, + "datastore_tracer.database_name_reporting.enabled": False, + "distributed_tracing.enabled": True, + "span_events.enabled": True, +} + + +async def _exercise_db(connection): + try: + cursor = connection.cursor() + await maybe_await(cursor.execute("SELECT setting from pg_settings where name=%s", ("server_version",))) + + # No target + await maybe_await(cursor.execute("SELECT 1")) + finally: + await maybe_await(connection.close()) + + +# Tests + + +@pytest.mark.parametrize("db_instance_enabled", (True, False)) +@pytest.mark.parametrize("instance_enabled", (True, False)) +def test_span_events(loop, connection, instance_enabled, db_instance_enabled): + guid = "dbb533c53b749e0b" + priority = 0.5 + + common_intrinsics = { + "type": "Span", + "transactionId": guid, + "priority": priority, + "sampled": True, + "category": "datastore", + "component": "Postgres", + "span.kind": "client", + } + + exact_agents = {} + + if instance_enabled: + settings = _enable_instance_settings.copy() + hostname = instance_hostname(DB_SETTINGS["host"]) + exact_agents.update( + { + "peer.address": f"{hostname}:{DB_SETTINGS['port']}", + "peer.hostname": hostname, + } + ) + else: + settings = _disable_instance_settings.copy() + exact_agents.update( + { + "peer.address": "Unknown:Unknown", + "peer.hostname": "Unknown", + } + ) + + if db_instance_enabled and instance_enabled: + exact_agents.update( + { + "db.instance": DB_SETTINGS["name"], + } + ) + unexpected_agents = () + else: + settings["attributes.exclude"] = ["db.instance"] + unexpected_agents = ("db.instance",) + + query_1_intrinsics = common_intrinsics.copy() + query_1_intrinsics["name"] = "Datastore/statement/Postgres/pg_settings/select" + + query_1_agents = exact_agents.copy() + query_1_agents["db.statement"] = "SELECT setting from pg_settings where name=%s" + + query_2_intrinsics = common_intrinsics.copy() + query_2_intrinsics["name"] = "Datastore/operation/Postgres/select" + + query_2_agents = exact_agents.copy() + query_2_agents["db.statement"] = "SELECT ?" + + @validate_span_events( + count=1, + exact_intrinsics=query_1_intrinsics, + unexpected_intrinsics=("db.instance", "db.statement"), + exact_agents=query_1_agents, + unexpected_agents=unexpected_agents, + ) + @validate_span_events( + count=1, + exact_intrinsics=query_2_intrinsics, + unexpected_intrinsics=("db.instance", "db.statement"), + exact_agents=query_2_agents, + unexpected_agents=unexpected_agents, + ) + @override_application_settings(settings) + @background_task(name="span_events") + def _test(): + txn = current_transaction() + txn.guid = guid + txn._priority = priority + txn._sampled = True + loop.run_until_complete(_exercise_db(connection)) + + _test() diff --git a/tests/datastore_psycopg/test_trace_node.py b/tests/datastore_psycopg/test_trace_node.py new file mode 100644 index 0000000000..584e27bbfe --- /dev/null +++ b/tests/datastore_psycopg/test_trace_node.py @@ -0,0 +1,104 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import psycopg +from conftest import DB_SETTINGS, maybe_await +from testing_support.fixtures import ( + override_application_settings, + validate_tt_parenting, +) +from testing_support.util import instance_hostname +from testing_support.validators.validate_tt_collector_json import ( + validate_tt_collector_json, +) + +from newrelic.api.background_task import background_task + +# Settings + +_enable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": True, + "datastore_tracer.database_name_reporting.enabled": True, +} +_disable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": False, + "datastore_tracer.database_name_reporting.enabled": False, +} + +# Expected parameters + +_enabled_required = { + "host": instance_hostname(DB_SETTINGS["host"]), + "port_path_or_id": str(DB_SETTINGS["port"]), + "db.instance": DB_SETTINGS["name"], +} +_enabled_forgone = {} + +_disabled_required = {} +_disabled_forgone = { + "host": "VALUE NOT USED", + "port_path_or_id": "VALUE NOT USED", + "db.instance": "VALUE NOT USED", +} + +_tt_parenting = ( + "TransactionNode", + [ + ("FunctionNode", []), + ("DatabaseNode", []), + ], +) + + +# Query + + +async def _exercise_db(async_=False): + # Connect here without using fixture to assert on the parenting of the FunctionTrace for Connection.connect() + # This is only possible when the connection is done inside a transaction, so connect after the test starts. + connect = psycopg.Connection.connect if async_ else psycopg.AsyncConnection.connect + connection = await maybe_await( + connect( + dbname=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + password=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ) + ) + + try: + cursor = connection.cursor() + await maybe_await(cursor.execute("SELECT setting from pg_settings where name=%s", ("server_version",))) + finally: + await maybe_await(connection.close()) + + +# Tests + + +@override_application_settings(_enable_instance_settings) +@validate_tt_collector_json(datastore_params=_enabled_required, datastore_forgone_params=_enabled_forgone) +@validate_tt_parenting(_tt_parenting) +@background_task() +def test_trace_node_datastore_params_enable_instance(loop, is_async): + loop.run_until_complete(_exercise_db(is_async)) + + +@override_application_settings(_disable_instance_settings) +@validate_tt_collector_json(datastore_params=_disabled_required, datastore_forgone_params=_disabled_forgone) +@validate_tt_parenting(_tt_parenting) +@background_task() +def test_trace_node_datastore_params_disable_instance(loop, is_async): + loop.run_until_complete(_exercise_db(is_async)) diff --git a/tests/datastore_psycopg2/conftest.py b/tests/datastore_psycopg2/conftest.py index dd271909d7..7fe6cb7e89 100644 --- a/tests/datastore_psycopg2/conftest.py +++ b/tests/datastore_psycopg2/conftest.py @@ -12,21 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, - 'debug.log_explain_plan_queries': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_explain_plan_queries": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_psycopg2)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_psycopg2)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_psycopg2/test_async.py b/tests/datastore_psycopg2/test_async.py index 7af9adc6a5..ca281084e5 100644 --- a/tests/datastore_psycopg2/test_async.py +++ b/tests/datastore_psycopg2/test_async.py @@ -44,8 +44,8 @@ # Metrics _base_scoped_metrics = ( - ('Datastore/statement/Postgres/%s/select' % DB_SETTINGS['table_name'], 1), - ('Datastore/statement/Postgres/%s/insert' % DB_SETTINGS['table_name'], 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 1), ('Datastore/operation/Postgres/drop', 1), ('Datastore/operation/Postgres/create', 1) ) @@ -56,9 +56,9 @@ ('Datastore/Postgres/all', 5), ('Datastore/Postgres/allOther', 5), ('Datastore/operation/Postgres/select', 1), - ('Datastore/statement/Postgres/%s/select' % DB_SETTINGS['table_name'], 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), ('Datastore/operation/Postgres/insert', 1), - ('Datastore/statement/Postgres/%s/insert' % DB_SETTINGS['table_name'], 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 1), ('Datastore/operation/Postgres/drop', 1), ('Datastore/operation/Postgres/create', 1) ) @@ -75,7 +75,7 @@ _host = instance_hostname(DB_SETTINGS['host']) _port = DB_SETTINGS['port'] -_instance_metric_name = 'Datastore/instance/Postgres/%s/%s' % (_host, _port) +_instance_metric_name = f'Datastore/instance/Postgres/{_host}/{_port}' _enable_rollup_metrics.append( (_instance_metric_name, 4) @@ -102,18 +102,16 @@ def _exercise_db(async_keyword): wait(async_conn) async_cur = async_conn.cursor() - async_cur.execute("""drop table if exists %s""" % DB_SETTINGS['table_name']) + async_cur.execute(f"""drop table if exists {DB_SETTINGS['table_name']}""") wait(async_cur.connection) - async_cur.execute("""create table %s """ % DB_SETTINGS['table_name'] + - """(a integer, b real, c text)""") + async_cur.execute(f"create table {DB_SETTINGS['table_name']} (a integer, b real, c text)") wait(async_cur.connection) - async_cur.execute("""insert into %s """ % DB_SETTINGS['table_name'] + - """values (%s, %s, %s)""", (1, 1.0, '1.0')) + async_cur.execute(f"insert into {DB_SETTINGS['table_name']} values (%s, %s, %s)", (1, 1.0, '1.0')) wait(async_cur.connection) - async_cur.execute("""select * from %s""" % DB_SETTINGS['table_name']) + async_cur.execute(f"""select * from {DB_SETTINGS['table_name']}""") wait(async_cur.connection) for row in async_cur: diff --git a/tests/datastore_psycopg2/test_cursor.py b/tests/datastore_psycopg2/test_cursor.py index d66d73ff84..a534e8e643 100644 --- a/tests/datastore_psycopg2/test_cursor.py +++ b/tests/datastore_psycopg2/test_cursor.py @@ -42,10 +42,10 @@ # Metrics _base_scoped_metrics = ( ('Function/psycopg2:connect', 1), - ('Datastore/statement/Postgres/%s/select' % DB_SETTINGS['table_name'], 1), - ('Datastore/statement/Postgres/%s/insert' % DB_SETTINGS['table_name'], 1), - ('Datastore/statement/Postgres/%s/update' % DB_SETTINGS['table_name'], 1), - ('Datastore/statement/Postgres/%s/delete' % DB_SETTINGS['table_name'], 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/update", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/delete", 1), ('Datastore/statement/Postgres/now/call', 1), ('Datastore/statement/Postgres/pg_sleep/call', 1), ('Datastore/operation/Postgres/drop', 1), @@ -59,10 +59,10 @@ ('Datastore/allOther', 12), ('Datastore/Postgres/all', 12), ('Datastore/Postgres/allOther', 12), - ('Datastore/statement/Postgres/%s/select' % DB_SETTINGS['table_name'], 1), - ('Datastore/statement/Postgres/%s/insert' % DB_SETTINGS['table_name'], 1), - ('Datastore/statement/Postgres/%s/update' % DB_SETTINGS['table_name'], 1), - ('Datastore/statement/Postgres/%s/delete' % DB_SETTINGS['table_name'], 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/update", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/delete", 1), ('Datastore/operation/Postgres/select', 1), ('Datastore/operation/Postgres/insert', 1), ('Datastore/operation/Postgres/update', 1), @@ -85,7 +85,7 @@ _host = instance_hostname(DB_SETTINGS['host']) _port = DB_SETTINGS['port'] -_instance_metric_name = 'Datastore/instance/Postgres/%s/%s' % (_host, _port) +_instance_metric_name = f'Datastore/instance/Postgres/{_host}/{_port}' _enable_rollup_metrics.append( (_instance_metric_name, 11) @@ -103,27 +103,27 @@ def _execute(connection, cursor, row_type, wrapper): psycopg2.extensions.register_type(unicode_type, connection) psycopg2.extensions.register_type(unicode_type, cursor) - sql = """drop table if exists %s""" % DB_SETTINGS["table_name"] + sql = f"""drop table if exists {DB_SETTINGS['table_name']}""" cursor.execute(wrapper(sql)) - sql = """create table %s (a integer, b real, c text)""" % DB_SETTINGS["table_name"] + sql = f"""create table {DB_SETTINGS['table_name']} (a integer, b real, c text)""" cursor.execute(wrapper(sql)) - sql = """insert into %s """ % DB_SETTINGS["table_name"] + """values (%s, %s, %s)""" + sql = f"insert into {DB_SETTINGS['table_name']} values (%s, %s, %s)" params = [(1, 1.0, '1.0'), (2, 2.2, '2.2'), (3, 3.3, '3.3')] cursor.executemany(wrapper(sql), params) - sql = """select * from %s""" % DB_SETTINGS["table_name"] + sql = f"""select * from {DB_SETTINGS['table_name']}""" cursor.execute(wrapper(sql)) for row in cursor: assert isinstance(row, row_type) - sql = """update %s""" % DB_SETTINGS["table_name"] + """ set a=%s, b=%s, c=%s where a=%s""" + sql = f"update {DB_SETTINGS['table_name']} set a=%s, b=%s, c=%s where a=%s" params = (4, 4.0, '4.0', 1) cursor.execute(wrapper(sql), params) - sql = """delete from %s where a=2""" % DB_SETTINGS["table_name"] + sql = f"""delete from {DB_SETTINGS['table_name']} where a=2""" cursor.execute(wrapper(sql)) connection.commit() @@ -162,9 +162,7 @@ def _exercise_db(cursor_factory=None, use_cur_context=False, row_type=tuple, _test_matrix[1].append((str, True)) # Composable SQL is expected to be available in versions 2.7 and up -assert sql, ( - "Composable sql (from psycopg2 import sql) is expected to load" - "but is not loading") +assert sql, "Composable sql (from psycopg2 import sql) is expected to load but is not loading" # exercise with regular SQL wrapper _test_matrix[1].append((sql.SQL, True)) diff --git a/tests/datastore_psycopg2/test_forward_compat.py b/tests/datastore_psycopg2/test_forward_compat.py index d150943288..35611f0e00 100644 --- a/tests/datastore_psycopg2/test_forward_compat.py +++ b/tests/datastore_psycopg2/test_forward_compat.py @@ -20,7 +20,7 @@ from newrelic.hooks.database_psycopg2 import wrapper_psycopg2_as_string -class TestCompatability(object): +class TestCompatability(): def as_string(self, giraffe, lion, tiger=None): assert isinstance(giraffe, ext.cursor) return "PASS" diff --git a/tests/datastore_psycopg2/test_multiple_dbs.py b/tests/datastore_psycopg2/test_multiple_dbs.py index afbdd66f2d..f69288cb85 100644 --- a/tests/datastore_psycopg2/test_multiple_dbs.py +++ b/tests/datastore_psycopg2/test_multiple_dbs.py @@ -71,10 +71,8 @@ _host_2 = instance_hostname(_postgresql_2['host']) _port_2 = _postgresql_2['port'] - _instance_metric_name_1 = 'Datastore/instance/Postgres/%s/%s' % ( - _host_1, _port_1) - _instance_metric_name_2 = 'Datastore/instance/Postgres/%s/%s' % ( - _host_2, _port_2) + _instance_metric_name_1 = f'Datastore/instance/Postgres/{_host_1}/{_port_1}' + _instance_metric_name_2 = f'Datastore/instance/Postgres/{_host_2}/{_port_2}' _enable_rollup_metrics.extend([ (_instance_metric_name_1, 2), @@ -111,9 +109,8 @@ def _exercise_db(): port=postgresql2['port']) try: cursor = connection.cursor() - cursor.execute("""drop table if exists %s""" % postgresql2["table_name"]) - cursor.execute("""create table %s """ % postgresql2["table_name"] + - """(a integer, b real, c text)""") + cursor.execute(f"""drop table if exists {postgresql2['table_name']}""") + cursor.execute(f"create table {postgresql2['table_name']} (a integer, b real, c text)") connection.commit() finally: connection.close() diff --git a/tests/datastore_psycopg2/test_obfuscation.py b/tests/datastore_psycopg2/test_obfuscation.py index 90f15d375b..69d2e00142 100644 --- a/tests/datastore_psycopg2/test_obfuscation.py +++ b/tests/datastore_psycopg2/test_obfuscation.py @@ -37,8 +37,8 @@ def psycopg2_cursor(): try: cursor = connection.cursor() - cursor.execute("drop table if exists %s" % DB_SETTINGS["table_name"]) - cursor.execute("create table %s (b text, c text)" % DB_SETTINGS["table_name"]) + cursor.execute(f"drop table if exists {DB_SETTINGS['table_name']}") + cursor.execute(f"create table {DB_SETTINGS['table_name']} (b text, c text)") yield cursor @@ -48,16 +48,16 @@ def psycopg2_cursor(): _quoting_style_tests = [ ( - "SELECT * FROM %s WHERE b='2'" % DB_SETTINGS["table_name"], - "SELECT * FROM %s WHERE b=?" % DB_SETTINGS["table_name"], + f"SELECT * FROM {DB_SETTINGS['table_name']} WHERE b='2'", + f"SELECT * FROM {DB_SETTINGS['table_name']} WHERE b=?", ), ( - "SELECT * FROM %s WHERE b=$func$2$func$" % DB_SETTINGS["table_name"], - "SELECT * FROM %s WHERE b=?" % DB_SETTINGS["table_name"], + f"SELECT * FROM {DB_SETTINGS['table_name']} WHERE b=$func$2$func$", + f"SELECT * FROM {DB_SETTINGS['table_name']} WHERE b=?", ), ( - "SELECT * FROM %s WHERE b=U&'2'" % DB_SETTINGS["table_name"], - "SELECT * FROM %s WHERE b=U&?" % DB_SETTINGS["table_name"], + f"SELECT * FROM {DB_SETTINGS['table_name']} WHERE b=U&'2'", + f"SELECT * FROM {DB_SETTINGS['table_name']} WHERE b=U&?", ), ] @@ -74,8 +74,8 @@ def test(): _parameter_tests = [ ( - "SELECT * FROM " + DB_SETTINGS["table_name"] + " where b=%s", - "SELECT * FROM " + DB_SETTINGS["table_name"] + " where b=%s", + f"SELECT * FROM {DB_SETTINGS['table_name']} where b=%s", + f"SELECT * FROM {DB_SETTINGS['table_name']} where b=%s", ), ] @@ -107,25 +107,23 @@ def any_length_explain_plan(node): _test_explain_plans = [ ( - "SELECT (b, c) FROM %s ; SELECT (b, c) FROM %s" - % (DB_SETTINGS["table_name"], DB_SETTINGS["table_name"]), + f"SELECT (b, c) FROM {DB_SETTINGS['table_name']} ; SELECT (b, c) FROM {DB_SETTINGS['table_name']}", no_explain_plan, ), ( - "SELECT (b, c) FROM %s ; SELECT (b, c) FROM %s;" - % (DB_SETTINGS["table_name"], DB_SETTINGS["table_name"]), + f"SELECT (b, c) FROM {DB_SETTINGS['table_name']} ; SELECT (b, c) FROM {DB_SETTINGS['table_name']};", no_explain_plan, ), - ("SELECT (b, c) FROM %s WHERE b=';'" % DB_SETTINGS["table_name"], no_explain_plan), - (";SELECT (b, c) FROM %s" % DB_SETTINGS["table_name"], no_explain_plan), - ("SELECT (b, c) FROM %s" % DB_SETTINGS["table_name"], any_length_explain_plan), - ("SELECT (b, c) FROM %s;" % DB_SETTINGS["table_name"], any_length_explain_plan), + (f"SELECT (b, c) FROM {DB_SETTINGS['table_name']} WHERE b=';'", no_explain_plan), + (f";SELECT (b, c) FROM {DB_SETTINGS['table_name']}", no_explain_plan), + (f"SELECT (b, c) FROM {DB_SETTINGS['table_name']}", any_length_explain_plan), + (f"SELECT (b, c) FROM {DB_SETTINGS['table_name']};", any_length_explain_plan), ( - "SELECT (b, c) FROM %s;;;;;;" % DB_SETTINGS["table_name"], + f"SELECT (b, c) FROM {DB_SETTINGS['table_name']};;;;;;", any_length_explain_plan, ), ( - "SELECT (b, c) FROM %s;\n\n" % DB_SETTINGS["table_name"], + f"SELECT (b, c) FROM {DB_SETTINGS['table_name']};\n\n", any_length_explain_plan, ), ] @@ -148,9 +146,9 @@ def test(): try: cursor = connection.cursor() - cursor.execute("drop table if exists %s" % DB_SETTINGS["table_name"]) + cursor.execute(f"drop table if exists {DB_SETTINGS['table_name']}") cursor.execute( - "create table %s (b text, c text)" % DB_SETTINGS["table_name"] + f"create table {DB_SETTINGS['table_name']} (b text, c text)" ) cursor.execute(sql) diff --git a/tests/datastore_psycopg2/test_register.py b/tests/datastore_psycopg2/test_register.py index b5450c3588..61cba82758 100644 --- a/tests/datastore_psycopg2/test_register.py +++ b/tests/datastore_psycopg2/test_register.py @@ -48,24 +48,22 @@ def test_register_range(): password=DB_SETTINGS['password'], host=DB_SETTINGS['host'], port=DB_SETTINGS['port']) as connection: - type_name = "floatrange_" + str(os.getpid()) + type_name = f"floatrange_{str(os.getpid())}" - create_sql = ('CREATE TYPE %s AS RANGE (' % type_name + - 'subtype = float8,' - 'subtype_diff = float8mi)') + create_sql = f"CREATE TYPE {type_name} AS RANGE (subtype = float8,subtype_diff = float8mi)" cursor = connection.cursor() - cursor.execute("DROP TYPE if exists %s" % type_name) + cursor.execute(f"DROP TYPE if exists {type_name}") cursor.execute(create_sql) psycopg2.extras.register_range(type_name, psycopg2.extras.NumericRange, connection) - cursor.execute("DROP TYPE if exists %s" % type_name) + cursor.execute(f"DROP TYPE if exists {type_name}") cursor.execute(create_sql) psycopg2.extras.register_range(type_name, psycopg2.extras.NumericRange, cursor) - cursor.execute("DROP TYPE if exists %s" % type_name) + cursor.execute(f"DROP TYPE if exists {type_name}") diff --git a/tests/datastore_psycopg2/test_rollback.py b/tests/datastore_psycopg2/test_rollback.py index 0a23b1005e..248edee43b 100644 --- a/tests/datastore_psycopg2/test_rollback.py +++ b/tests/datastore_psycopg2/test_rollback.py @@ -57,7 +57,7 @@ _host = instance_hostname(DB_SETTINGS['host']) _port = DB_SETTINGS['port'] -_instance_metric_name = 'Datastore/instance/Postgres/%s/%s' % (_host, _port) +_instance_metric_name = f'Datastore/instance/Postgres/{_host}/{_port}' _enable_rollup_metrics.append( (_instance_metric_name, 1) diff --git a/tests/datastore_psycopg2/test_slow_sql.py b/tests/datastore_psycopg2/test_slow_sql.py index aea45dd183..8feeef4d2d 100644 --- a/tests/datastore_psycopg2/test_slow_sql.py +++ b/tests/datastore_psycopg2/test_slow_sql.py @@ -14,71 +14,78 @@ import psycopg2 import pytest - from testing_support.fixtures import override_application_settings -from testing_support.validators.validate_slow_sql_collector_json import validate_slow_sql_collector_json - +from testing_support.validators.validate_slow_sql_collector_json import ( + validate_slow_sql_collector_json, +) from utils import DB_SETTINGS from newrelic.api.background_task import background_task from newrelic.api.transaction import current_transaction - # Settings _enable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': True, - 'datastore_tracer.database_name_reporting.enabled': True, + "datastore_tracer.instance_reporting.enabled": True, + "datastore_tracer.database_name_reporting.enabled": True, } _disable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': False, - 'datastore_tracer.database_name_reporting.enabled': False, + "datastore_tracer.instance_reporting.enabled": False, + "datastore_tracer.database_name_reporting.enabled": False, } # Expected parameters -_enabled_required = set(['host', 'port_path_or_id', 'database_name']) +_enabled_required = set(["host", "port_path_or_id", "database_name"]) _enabled_forgone = set() _disabled_required = set() -_disabled_forgone = set(['host', 'port_path_or_id', 'database_name']) +_disabled_forgone = set(["host", "port_path_or_id", "database_name"]) -_distributed_tracing_always_params = set(['guid', 'traceId', 'priority', - 'sampled']) -_distributed_tracing_payload_received_params = set(['parent.type', - 'parent.app', 'parent.account', 'parent.transportType', - 'parent.transportDuration']) +# Guid is always required, regardless of DT status. +# It should be excluded from the forgone params set. +_distributed_tracing_required_params = set(["guid", "traceId", "priority", "sampled"]) +_distributed_tracing_forgone_params = set(["traceId", "priority", "sampled"]) +_distributed_tracing_payload_received_params = set( + ["parent.type", "parent.app", "parent.account", "parent.transportType", "parent.transportDuration"] +) -_transaction_guid = '1234567890' -_distributed_tracing_exact_params = {'guid': _transaction_guid} +_transaction_guid = "1234567890" +_distributed_tracing_exact_params = {"guid": _transaction_guid} # Query + def _exercise_db(): connection = psycopg2.connect( - database=DB_SETTINGS['name'], user=DB_SETTINGS['user'], - password=DB_SETTINGS['password'], host=DB_SETTINGS['host'], - port=DB_SETTINGS['port']) + database=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + password=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ) try: cursor = connection.cursor() - cursor.execute("""SELECT setting from pg_settings where name=%s""", - ('server_version',)) + cursor.execute("""SELECT setting from pg_settings where name=%s""", ("server_version",)) finally: connection.close() # Tests -@pytest.mark.parametrize('instance_enabled', (True, False)) -@pytest.mark.parametrize('distributed_tracing_enabled,payload_received', [ + +@pytest.mark.parametrize("instance_enabled", (True, False)) +@pytest.mark.parametrize( + "distributed_tracing_enabled,payload_received", + [ (True, True), (True, False), (False, False), -]) -def test_slow_sql_json(instance_enabled, distributed_tracing_enabled, - payload_received): + ], +) +def test_slow_sql_json(instance_enabled, distributed_tracing_enabled, payload_received): exact_params = None @@ -92,25 +99,22 @@ def test_slow_sql_json(instance_enabled, distributed_tracing_enabled, forgone_params = set(_disabled_forgone) if distributed_tracing_enabled: - required_params.update(_distributed_tracing_always_params) + required_params.update(_distributed_tracing_required_params) exact_params = _distributed_tracing_exact_params - settings['distributed_tracing.enabled'] = True + settings["distributed_tracing.enabled"] = True if payload_received: - required_params.update( - _distributed_tracing_payload_received_params) + required_params.update(_distributed_tracing_payload_received_params) else: - forgone_params.update( - _distributed_tracing_payload_received_params) + forgone_params.update(_distributed_tracing_payload_received_params) else: - forgone_params.update(_distributed_tracing_always_params) + forgone_params.update(_distributed_tracing_forgone_params) forgone_params.update(_distributed_tracing_payload_received_params) - settings['distributed_tracing.enabled'] = False + settings["distributed_tracing.enabled"] = False @override_application_settings(settings) @validate_slow_sql_collector_json( - required_params=required_params, - forgone_params=forgone_params, - exact_params=exact_params) + required_params=required_params, forgone_params=forgone_params, exact_params=exact_params + ) @background_task() def _test(): transaction = current_transaction() @@ -121,18 +125,18 @@ def _test(): if payload_received: payload = { - 'v': [0, 1], - 'd': { - 'ty': 'Mobile', - 'ac': transaction.settings.account_id, - 'tk': transaction.settings.trusted_account_key, - 'ap': '2827902', - 'pa': '5e5733a911cfbc73', - 'id': '7d3efb1b173fecfa', - 'tr': 'd6b4ba0c3a712ca', - 'ti': 1518469636035, - 'tx': '8703ff3d88eefe9d', - } + "v": [0, 1], + "d": { + "ty": "Mobile", + "ac": transaction.settings.account_id, + "tk": transaction.settings.trusted_account_key, + "ap": "2827902", + "pa": "5e5733a911cfbc73", + "id": "7d3efb1b173fecfa", + "tr": "d6b4ba0c3a712ca", + "ti": 1518469636035, + "tx": "8703ff3d88eefe9d", + }, } transaction.accept_distributed_trace_payload(payload) diff --git a/tests/datastore_psycopg2/test_span_event.py b/tests/datastore_psycopg2/test_span_event.py index 0834061c70..019194ce01 100644 --- a/tests/datastore_psycopg2/test_span_event.py +++ b/tests/datastore_psycopg2/test_span_event.py @@ -82,7 +82,7 @@ def test_span_events(instance_enabled, db_instance_enabled): settings = _enable_instance_settings.copy() hostname = instance_hostname(DB_SETTINGS['host']) exact_agents.update({ - 'peer.address': '%s:%s' % (hostname, DB_SETTINGS['port']), + 'peer.address': f"{hostname}:{DB_SETTINGS['port']}", 'peer.hostname': hostname, }) else: diff --git a/tests/datastore_psycopg2cffi/conftest.py b/tests/datastore_psycopg2cffi/conftest.py index c9df1369bb..28ef3ca984 100644 --- a/tests/datastore_psycopg2cffi/conftest.py +++ b/tests/datastore_psycopg2cffi/conftest.py @@ -12,21 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, - 'debug.log_explain_plan_queries': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_explain_plan_queries": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_psycopg2cffi)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_psycopg2cffi)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_psycopg2cffi/test_database.py b/tests/datastore_psycopg2cffi/test_database.py index 54ff6ad09d..9b6bb64d29 100644 --- a/tests/datastore_psycopg2cffi/test_database.py +++ b/tests/datastore_psycopg2cffi/test_database.py @@ -15,199 +15,223 @@ import psycopg2cffi import psycopg2cffi.extensions import psycopg2cffi.extras - -from testing_support.fixtures import validate_stats_engine_explain_plan_output_is_none -from testing_support.validators.validate_transaction_errors import validate_transaction_errors -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_transaction_slow_sql_count import \ - validate_transaction_slow_sql_count -from testing_support.validators.validate_database_trace_inputs import validate_database_trace_inputs - from testing_support.db_settings import postgresql_settings +from testing_support.fixtures import validate_stats_engine_explain_plan_output_is_none +from testing_support.util import instance_hostname +from testing_support.validators.validate_database_trace_inputs import ( + validate_database_trace_inputs, +) +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_transaction_slow_sql_count import ( + validate_transaction_slow_sql_count, +) from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple DB_SETTINGS = postgresql_settings()[0] _test_execute_via_cursor_scoped_metrics = [ - ('Function/psycopg2cffi:connect', 1), - ('Function/psycopg2cffi._impl.connection:Connection.__enter__', 1), - ('Function/psycopg2cffi._impl.connection:Connection.__exit__', 1), - ('Datastore/statement/Postgres/%s/select' % DB_SETTINGS["table_name"], 1), - ('Datastore/statement/Postgres/%s/insert' % DB_SETTINGS["table_name"], 1), - ('Datastore/statement/Postgres/%s/update' % DB_SETTINGS["table_name"], 1), - ('Datastore/statement/Postgres/%s/delete' % DB_SETTINGS["table_name"], 1), - ('Datastore/statement/Postgres/now/call', 1), - ('Datastore/statement/Postgres/pg_sleep/call', 1), - ('Datastore/operation/Postgres/drop', 1), - ('Datastore/operation/Postgres/create', 1), - ('Datastore/operation/Postgres/commit', 3), - ('Datastore/operation/Postgres/rollback', 1)] + ("Function/psycopg2cffi:connect", 1), + ("Function/psycopg2cffi._impl.connection:Connection.__enter__", 1), + ("Function/psycopg2cffi._impl.connection:Connection.__exit__", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/update", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/delete", 1), + ("Datastore/statement/Postgres/now/call", 1), + ("Datastore/statement/Postgres/pg_sleep/call", 1), + ("Datastore/operation/Postgres/drop", 1), + ("Datastore/operation/Postgres/create", 1), + ("Datastore/operation/Postgres/commit", 3), + ("Datastore/operation/Postgres/rollback", 1), +] _test_execute_via_cursor_rollup_metrics = [ - ('Datastore/all', 13), - ('Datastore/allOther', 13), - ('Datastore/Postgres/all', 13), - ('Datastore/Postgres/allOther', 13), - ('Datastore/operation/Postgres/select', 1), - ('Datastore/statement/Postgres/%s/select' % DB_SETTINGS["table_name"], 1), - ('Datastore/operation/Postgres/insert', 1), - ('Datastore/statement/Postgres/%s/insert' % DB_SETTINGS["table_name"], 1), - ('Datastore/operation/Postgres/update', 1), - ('Datastore/statement/Postgres/%s/update' % DB_SETTINGS["table_name"], 1), - ('Datastore/operation/Postgres/delete', 1), - ('Datastore/statement/Postgres/%s/delete' % DB_SETTINGS["table_name"], 1), - ('Datastore/operation/Postgres/drop', 1), - ('Datastore/operation/Postgres/create', 1), - ('Datastore/statement/Postgres/now/call', 1), - ('Datastore/statement/Postgres/pg_sleep/call', 1), - ('Datastore/operation/Postgres/call', 2), - ('Datastore/operation/Postgres/commit', 3), - ('Datastore/operation/Postgres/rollback', 1)] - - -@validate_transaction_metrics('test_database:test_execute_via_cursor', - scoped_metrics=_test_execute_via_cursor_scoped_metrics, - rollup_metrics=_test_execute_via_cursor_rollup_metrics, - background_task=True) + ("Datastore/all", 13), + ("Datastore/allOther", 13), + ("Datastore/Postgres/all", 13), + ("Datastore/Postgres/allOther", 13), + ("Datastore/operation/Postgres/select", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), + ("Datastore/operation/Postgres/insert", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 1), + ("Datastore/operation/Postgres/update", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/update", 1), + ("Datastore/operation/Postgres/delete", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/delete", 1), + ("Datastore/operation/Postgres/drop", 1), + ("Datastore/operation/Postgres/create", 1), + ("Datastore/statement/Postgres/now/call", 1), + ("Datastore/statement/Postgres/pg_sleep/call", 1), + ("Datastore/operation/Postgres/call", 2), + ("Datastore/operation/Postgres/commit", 3), + ("Datastore/operation/Postgres/rollback", 1), + (f"Datastore/instance/Postgres/{instance_hostname(DB_SETTINGS['host'])}/{DB_SETTINGS['port']}", 12), +] + + +@validate_transaction_metrics( + "test_database:test_execute_via_cursor", + scoped_metrics=_test_execute_via_cursor_scoped_metrics, + rollup_metrics=_test_execute_via_cursor_rollup_metrics, + background_task=True, +) @validate_database_trace_inputs(sql_parameters_type=tuple) @background_task() def test_execute_via_cursor(): with psycopg2cffi.connect( - database=DB_SETTINGS['name'], user=DB_SETTINGS['user'], - password=DB_SETTINGS['password'], host=DB_SETTINGS['host'], - port=DB_SETTINGS['port']) as connection: - + database=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + password=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ) as connection: cursor = connection.cursor() psycopg2cffi.extensions.register_type(psycopg2cffi.extensions.UNICODE) - psycopg2cffi.extensions.register_type( - psycopg2cffi.extensions.UNICODE, - connection) - psycopg2cffi.extensions.register_type( - psycopg2cffi.extensions.UNICODE, - cursor) + psycopg2cffi.extensions.register_type(psycopg2cffi.extensions.UNICODE, connection) + psycopg2cffi.extensions.register_type(psycopg2cffi.extensions.UNICODE, cursor) - cursor.execute("""drop table if exists %s""" % DB_SETTINGS["table_name"]) + cursor.execute(f"""drop table if exists {DB_SETTINGS['table_name']}""") - cursor.execute("""create table %s """ % DB_SETTINGS["table_name"] + - """(a integer, b real, c text)""") + cursor.execute(f"create table {DB_SETTINGS['table_name']} (a integer, b real, c text)") - cursor.executemany("""insert into %s """ % DB_SETTINGS["table_name"] + - """values (%s, %s, %s)""", [(1, 1.0, '1.0'), - (2, 2.2, '2.2'), (3, 3.3, '3.3')]) + cursor.executemany( + f"insert into {DB_SETTINGS['table_name']} values (%s, %s, %s)", + [(1, 1.0, "1.0"), (2, 2.2, "2.2"), (3, 3.3, "3.3")], + ) - cursor.execute("""select * from %s""" % DB_SETTINGS["table_name"]) + cursor.execute(f"""select * from {DB_SETTINGS['table_name']}""") for row in cursor: pass - cursor.execute("""update %s""" % DB_SETTINGS["table_name"] + """ set a=%s, b=%s, """ - """c=%s where a=%s""", (4, 4.0, '4.0', 1)) + cursor.execute( + f"update {DB_SETTINGS['table_name']} set a=%s, b=%s, c=%s where a=%s", + (4, 4.0, "4.0", 1), + ) - cursor.execute("""delete from %s where a=2""" % DB_SETTINGS["table_name"]) + cursor.execute(f"""delete from {DB_SETTINGS['table_name']} where a=2""") connection.commit() - cursor.callproc('now') - cursor.callproc('pg_sleep', (0,)) + cursor.callproc("now") + cursor.callproc("pg_sleep", (0,)) connection.rollback() connection.commit() _test_rollback_on_exception_scoped_metrics = [ - ('Function/psycopg2cffi:connect', 1), - ('Function/psycopg2cffi._impl.connection:Connection.__enter__', 1), - ('Function/psycopg2cffi._impl.connection:Connection.__exit__', 1), - ('Datastore/operation/Postgres/rollback', 1)] + ("Function/psycopg2cffi:connect", 1), + ("Function/psycopg2cffi._impl.connection:Connection.__enter__", 1), + ("Function/psycopg2cffi._impl.connection:Connection.__exit__", 1), + ("Datastore/operation/Postgres/rollback", 1), +] _test_rollback_on_exception_rollup_metrics = [ - ('Datastore/all', 2), - ('Datastore/allOther', 2), - ('Datastore/Postgres/all', 2), - ('Datastore/Postgres/allOther', 2)] - - -@validate_transaction_metrics('test_database:test_rollback_on_exception', - scoped_metrics=_test_rollback_on_exception_scoped_metrics, - rollup_metrics=_test_rollback_on_exception_rollup_metrics, - background_task=True) + ("Datastore/all", 2), + ("Datastore/allOther", 2), + ("Datastore/Postgres/all", 2), + ("Datastore/Postgres/allOther", 2), +] + + +@validate_transaction_metrics( + "test_database:test_rollback_on_exception", + scoped_metrics=_test_rollback_on_exception_scoped_metrics, + rollup_metrics=_test_rollback_on_exception_rollup_metrics, + background_task=True, +) @validate_database_trace_inputs(sql_parameters_type=tuple) @background_task() def test_rollback_on_exception(): try: with psycopg2cffi.connect( - database=DB_SETTINGS['name'], user=DB_SETTINGS['user'], - password=DB_SETTINGS['password'], host=DB_SETTINGS['host'], - port=DB_SETTINGS['port']): - - raise RuntimeError('error') + database=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + password=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ): + raise RuntimeError("error") except RuntimeError: pass _test_async_mode_scoped_metrics = [ - ('Function/psycopg2cffi:connect', 1), - ('Datastore/statement/Postgres/%s/select' % DB_SETTINGS["table_name"], 1), - ('Datastore/statement/Postgres/%s/insert' % DB_SETTINGS["table_name"], 1), - ('Datastore/operation/Postgres/drop', 1), - ('Datastore/operation/Postgres/create', 1)] + ("Function/psycopg2cffi:connect", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 1), + ("Datastore/operation/Postgres/drop", 1), + ("Datastore/operation/Postgres/create", 1), +] _test_async_mode_rollup_metrics = [ - ('Datastore/all', 5), - ('Datastore/allOther', 5), - ('Datastore/Postgres/all', 5), - ('Datastore/Postgres/allOther', 5), - ('Datastore/operation/Postgres/select', 1), - ('Datastore/statement/Postgres/%s/select' % DB_SETTINGS["table_name"], 1), - ('Datastore/operation/Postgres/insert', 1), - ('Datastore/statement/Postgres/%s/insert' % DB_SETTINGS["table_name"], 1), - ('Datastore/operation/Postgres/drop', 1), - ('Datastore/operation/Postgres/create', 1)] + ("Datastore/all", 5), + ("Datastore/allOther", 5), + ("Datastore/Postgres/all", 5), + ("Datastore/Postgres/allOther", 5), + ("Datastore/operation/Postgres/select", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/select", 1), + ("Datastore/operation/Postgres/insert", 1), + (f"Datastore/statement/Postgres/{DB_SETTINGS['table_name']}/insert", 1), + ("Datastore/operation/Postgres/drop", 1), + ("Datastore/operation/Postgres/create", 1), + (f"Datastore/instance/Postgres/{instance_hostname(DB_SETTINGS['host'])}/{DB_SETTINGS['port']}", 4), +] @validate_stats_engine_explain_plan_output_is_none() @validate_transaction_slow_sql_count(num_slow_sql=4) @validate_database_trace_inputs(sql_parameters_type=tuple) -@validate_transaction_metrics('test_database:test_async_mode', - scoped_metrics=_test_async_mode_scoped_metrics, - rollup_metrics=_test_async_mode_rollup_metrics, - background_task=True) +@validate_transaction_metrics( + "test_database:test_async_mode", + scoped_metrics=_test_async_mode_scoped_metrics, + rollup_metrics=_test_async_mode_rollup_metrics, + background_task=True, +) @validate_transaction_errors(errors=[]) @background_task() def test_async_mode(): - wait = psycopg2cffi.extras.wait_select kwargs = {} - version = tuple(int(_) for _ in psycopg2cffi.__version__.split('.')) + version = get_package_version_tuple("psycopg2cffi") + assert version is not None if version >= (2, 8): - kwargs['async_'] = 1 + kwargs["async_"] = 1 else: - kwargs['async'] = 1 + kwargs["async"] = 1 async_conn = psycopg2cffi.connect( - database=DB_SETTINGS['name'], user=DB_SETTINGS['user'], - password=DB_SETTINGS['password'], host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], **kwargs + database=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + password=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + **kwargs ) wait(async_conn) async_cur = async_conn.cursor() - async_cur.execute("""drop table if exists %s""" % DB_SETTINGS["table_name"]) + async_cur.execute(f"""drop table if exists {DB_SETTINGS['table_name']}""") wait(async_cur.connection) - async_cur.execute("""create table %s """ % DB_SETTINGS["table_name"] + - """(a integer, b real, c text)""") + async_cur.execute(f"create table {DB_SETTINGS['table_name']} (a integer, b real, c text)") wait(async_cur.connection) - async_cur.execute("""insert into %s """ % DB_SETTINGS["table_name"] + - """values (%s, %s, %s)""", (1, 1.0, '1.0')) + async_cur.execute(f"insert into {DB_SETTINGS['table_name']} values (%s, %s, %s)", (1, 1.0, "1.0")) wait(async_cur.connection) - async_cur.execute("""select * from %s""" % DB_SETTINGS["table_name"]) + async_cur.execute(f"""select * from {DB_SETTINGS['table_name']}""") wait(async_cur.connection) for row in async_cur: diff --git a/tests/datastore_pylibmc/conftest.py b/tests/datastore_pylibmc/conftest.py index 40970bdcae..093a522cb8 100644 --- a/tests/datastore_pylibmc/conftest.py +++ b/tests/datastore_pylibmc/conftest.py @@ -12,20 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_pylibmc)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_pylibmc)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_pylibmc/test_memcache.py b/tests/datastore_pylibmc/test_memcache.py index 769f3b483c..dd4cb9413e 100644 --- a/tests/datastore_pylibmc/test_memcache.py +++ b/tests/datastore_pylibmc/test_memcache.py @@ -12,85 +12,92 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - import pylibmc - from testing_support.db_settings import memcached_settings -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.background_task import background_task from newrelic.api.transaction import set_background_task - DB_SETTINGS = memcached_settings()[0] MEMCACHED_HOST = DB_SETTINGS["host"] MEMCACHED_PORT = DB_SETTINGS["port"] MEMCACHED_NAMESPACE = DB_SETTINGS["namespace"] -MEMCACHED_ADDR = '%s:%s' % (MEMCACHED_HOST, MEMCACHED_PORT) +MEMCACHED_ADDR = f"{MEMCACHED_HOST}:{MEMCACHED_PORT}" _test_bt_set_get_delete_scoped_metrics = [ - ('Datastore/operation/Memcached/set', 1), - ('Datastore/operation/Memcached/get', 1), - ('Datastore/operation/Memcached/delete', 1)] + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] _test_bt_set_get_delete_rollup_metrics = [ - ('Datastore/all', 3), - ('Datastore/allOther', 3), - ('Datastore/Memcached/all', 3), - ('Datastore/Memcached/allOther', 3), - ('Datastore/operation/Memcached/set', 1), - ('Datastore/operation/Memcached/get', 1), - ('Datastore/operation/Memcached/delete', 1)] + ("Datastore/all", 3), + ("Datastore/allOther", 3), + ("Datastore/Memcached/all", 3), + ("Datastore/Memcached/allOther", 3), + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] + @validate_transaction_metrics( - 'test_memcache:test_bt_set_get_delete', - scoped_metrics=_test_bt_set_get_delete_scoped_metrics, - rollup_metrics=_test_bt_set_get_delete_rollup_metrics, - background_task=True) + "test_memcache:test_bt_set_get_delete", + scoped_metrics=_test_bt_set_get_delete_scoped_metrics, + rollup_metrics=_test_bt_set_get_delete_rollup_metrics, + background_task=True, +) @background_task() def test_bt_set_get_delete(): set_background_task(True) client = pylibmc.Client([MEMCACHED_ADDR]) - key = MEMCACHED_NAMESPACE + 'key' + key = f"{MEMCACHED_NAMESPACE}key" - client.set(key, 'value') + client.set(key, "value") value = client.get(key) client.delete(key) - assert value == 'value' + assert value == "value" + _test_wt_set_get_delete_scoped_metrics = [ - ('Datastore/operation/Memcached/set', 1), - ('Datastore/operation/Memcached/get', 1), - ('Datastore/operation/Memcached/delete', 1)] + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] _test_wt_set_get_delete_rollup_metrics = [ - ('Datastore/all', 3), - ('Datastore/allWeb', 3), - ('Datastore/Memcached/all', 3), - ('Datastore/Memcached/allWeb', 3), - ('Datastore/operation/Memcached/set', 1), - ('Datastore/operation/Memcached/get', 1), - ('Datastore/operation/Memcached/delete', 1)] + ("Datastore/all", 3), + ("Datastore/allWeb", 3), + ("Datastore/Memcached/all", 3), + ("Datastore/Memcached/allWeb", 3), + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] + @validate_transaction_metrics( - 'test_memcache:test_wt_set_get_delete', - scoped_metrics=_test_wt_set_get_delete_scoped_metrics, - rollup_metrics=_test_wt_set_get_delete_rollup_metrics, - background_task=False) + "test_memcache:test_wt_set_get_delete", + scoped_metrics=_test_wt_set_get_delete_scoped_metrics, + rollup_metrics=_test_wt_set_get_delete_rollup_metrics, + background_task=False, +) @background_task() def test_wt_set_get_delete(): set_background_task(False) client = pylibmc.Client([MEMCACHED_ADDR]) - key = MEMCACHED_NAMESPACE + 'key' + key = f"{MEMCACHED_NAMESPACE}key" - client.set(key, 'value') + client.set(key, "value") value = client.get(key) client.delete(key) - assert value == 'value' + assert value == "value" diff --git a/tests/datastore_pymemcache/conftest.py b/tests/datastore_pymemcache/conftest.py index 3d4e1ce766..22252eb9ab 100644 --- a/tests/datastore_pymemcache/conftest.py +++ b/tests/datastore_pymemcache/conftest.py @@ -12,20 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_pymemcache)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_pymemcache)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_pymemcache/test_memcache.py b/tests/datastore_pymemcache/test_memcache.py index 9aeea4d54d..66033af445 100644 --- a/tests/datastore_pymemcache/test_memcache.py +++ b/tests/datastore_pymemcache/test_memcache.py @@ -12,84 +12,98 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - import pymemcache.client - -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from testing_support.db_settings import memcached_settings +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.background_task import background_task from newrelic.api.transaction import set_background_task +from newrelic.common import system_info + DB_SETTINGS = memcached_settings()[0] MEMCACHED_HOST = DB_SETTINGS["host"] MEMCACHED_PORT = DB_SETTINGS["port"] MEMCACHED_NAMESPACE = DB_SETTINGS["namespace"] - MEMCACHED_ADDR = (MEMCACHED_HOST, int(MEMCACHED_PORT)) +INSTANCE_METRIC_HOST = system_info.gethostname() if MEMCACHED_HOST == "127.0.0.1" else MEMCACHED_HOST +INSTANCE_METRIC_NAME = f"Datastore/instance/Memcached/{INSTANCE_METRIC_HOST}/{MEMCACHED_PORT}" + _test_bt_set_get_delete_scoped_metrics = [ - ('Datastore/operation/Memcached/set', 1), - ('Datastore/operation/Memcached/get', 1), - ('Datastore/operation/Memcached/delete', 1)] + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] _test_bt_set_get_delete_rollup_metrics = [ - ('Datastore/all', 3), - ('Datastore/allOther', 3), - ('Datastore/Memcached/all', 3), - ('Datastore/Memcached/allOther', 3), - ('Datastore/operation/Memcached/set', 1), - ('Datastore/operation/Memcached/get', 1), - ('Datastore/operation/Memcached/delete', 1)] + ("Datastore/all", 3), + ("Datastore/allOther", 3), + ("Datastore/Memcached/all", 3), + ("Datastore/Memcached/allOther", 3), + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), + (INSTANCE_METRIC_NAME, 3), +] + @validate_transaction_metrics( - 'test_memcache:test_bt_set_get_delete', - scoped_metrics=_test_bt_set_get_delete_scoped_metrics, - rollup_metrics=_test_bt_set_get_delete_rollup_metrics, - background_task=True) + "test_memcache:test_bt_set_get_delete", + scoped_metrics=_test_bt_set_get_delete_scoped_metrics, + rollup_metrics=_test_bt_set_get_delete_rollup_metrics, + background_task=True, +) @background_task() def test_bt_set_get_delete(): set_background_task(True) client = pymemcache.client.Client(MEMCACHED_ADDR) - key = MEMCACHED_NAMESPACE + 'key' + key = f"{MEMCACHED_NAMESPACE}key" - client.set(key, b'value') + client.set(key, b"value") value = client.get(key) client.delete(key) - assert value == b'value' + assert value == b"value" + _test_wt_set_get_delete_scoped_metrics = [ - ('Datastore/operation/Memcached/set', 1), - ('Datastore/operation/Memcached/get', 1), - ('Datastore/operation/Memcached/delete', 1)] + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), +] _test_wt_set_get_delete_rollup_metrics = [ - ('Datastore/all', 3), - ('Datastore/allWeb', 3), - ('Datastore/Memcached/all', 3), - ('Datastore/Memcached/allWeb', 3), - ('Datastore/operation/Memcached/set', 1), - ('Datastore/operation/Memcached/get', 1), - ('Datastore/operation/Memcached/delete', 1)] + ("Datastore/all", 3), + ("Datastore/allWeb", 3), + ("Datastore/Memcached/all", 3), + ("Datastore/Memcached/allWeb", 3), + ("Datastore/operation/Memcached/set", 1), + ("Datastore/operation/Memcached/get", 1), + ("Datastore/operation/Memcached/delete", 1), + (INSTANCE_METRIC_NAME, 3), +] + @validate_transaction_metrics( - 'test_memcache:test_wt_set_get_delete', - scoped_metrics=_test_wt_set_get_delete_scoped_metrics, - rollup_metrics=_test_wt_set_get_delete_rollup_metrics, - background_task=False) + "test_memcache:test_wt_set_get_delete", + scoped_metrics=_test_wt_set_get_delete_scoped_metrics, + rollup_metrics=_test_wt_set_get_delete_rollup_metrics, + background_task=False, +) @background_task() def test_wt_set_get_delete(): set_background_task(False) client = pymemcache.client.Client(MEMCACHED_ADDR) - key = MEMCACHED_NAMESPACE + 'key' + key = f"{MEMCACHED_NAMESPACE}key" - client.set(key, b'value') + client.set(key, b"value") value = client.get(key) client.delete(key) - assert value == b'value' + assert value == b"value" diff --git a/tests/datastore_pymongo/conftest.py b/tests/datastore_pymongo/conftest.py index d269182b03..0579578328 100644 --- a/tests/datastore_pymongo/conftest.py +++ b/tests/datastore_pymongo/conftest.py @@ -12,10 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, diff --git a/tests/datastore_pymongo/test_pymongo.py b/tests/datastore_pymongo/test_pymongo.py index 4649062cee..6a0dfe5ef4 100644 --- a/tests/datastore_pymongo/test_pymongo.py +++ b/tests/datastore_pymongo/test_pymongo.py @@ -22,7 +22,6 @@ from testing_support.validators.validate_transaction_errors import validate_transaction_errors from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from newrelic.api.background_task import background_task -from newrelic.packages import six DB_SETTINGS = mongodb_settings()[0] MONGODB_HOST = DB_SETTINGS["host"] @@ -30,6 +29,15 @@ MONGODB_COLLECTION = DB_SETTINGS["collection"] +# Find correct metric name based on import availability. +try: + from pymongo.synchronous.mongo_client import MongoClient # noqa + INIT_FUNCTION_METRIC = "Function/pymongo.synchronous.mongo_client:MongoClient.__init__" +except ImportError: + from pymongo.mongo_client import MongoClient # noqa + INIT_FUNCTION_METRIC = "Function/pymongo.mongo_client:MongoClient.__init__" + + def _exercise_mongo_v3(db): db[MONGODB_COLLECTION].save({"x": 10}) db[MONGODB_COLLECTION].save({"x": 8}) @@ -115,56 +123,56 @@ def _exercise_mongo(db): _test_pymongo_scoped_metrics_v3 = [ - ("Function/pymongo.mongo_client:MongoClient.__init__", 1), - ("Datastore/statement/MongoDB/%s/create_index" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find" % MONGODB_COLLECTION, 3), - ("Datastore/statement/MongoDB/%s/find_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/save" % MONGODB_COLLECTION, 3), - ("Datastore/statement/MongoDB/%s" % MONGODB_COLLECTION + "/initialize_unordered_bulk_op", 1), - ("Datastore/statement/MongoDB/%s" % MONGODB_COLLECTION + "/initialize_ordered_bulk_op", 1), - ("Datastore/statement/MongoDB/%s/parallel_scan" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/insert_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/bulk_write" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/insert_many" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/replace_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/update_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/delete_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/delete_many" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_raw_batches" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/create_indexes" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/list_indexes" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/aggregate" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/aggregate_raw_batches" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_one_and_delete" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_one_and_replace" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_one_and_update" % MONGODB_COLLECTION, 1), + (INIT_FUNCTION_METRIC, 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/create_index", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find", 3), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/save", 3), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/initialize_unordered_bulk_op", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/initialize_ordered_bulk_op", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/parallel_scan", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/insert_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/bulk_write", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/insert_many", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/replace_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/update_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/delete_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/delete_many", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_raw_batches", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/create_indexes", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/list_indexes", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/aggregate", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/aggregate_raw_batches", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one_and_delete", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one_and_replace", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one_and_update", 1), ] _test_pymongo_scoped_metrics_v4 = [ - ("Function/pymongo.mongo_client:MongoClient.__init__", 1), - ("Datastore/statement/MongoDB/%s/create_index" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find" % MONGODB_COLLECTION, 3), - ("Datastore/statement/MongoDB/%s/find_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/insert_one" % MONGODB_COLLECTION, 4), - ("Datastore/statement/MongoDB/%s/bulk_write" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/insert_many" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/replace_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/update_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/delete_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/delete_many" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_raw_batches" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/create_indexes" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/list_indexes" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/aggregate" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/aggregate_raw_batches" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_one_and_delete" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_one_and_replace" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_one_and_update" % MONGODB_COLLECTION, 1), + (INIT_FUNCTION_METRIC, 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/create_index", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find", 3), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/insert_one", 4), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/bulk_write", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/insert_many", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/replace_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/update_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/delete_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/delete_many", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_raw_batches", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/create_indexes", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/list_indexes", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/aggregate", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/aggregate_raw_batches", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one_and_delete", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one_and_replace", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one_and_update", 1), ] _test_pymongo_rollup_metrics_v3 = [ - ("Function/pymongo.mongo_client:MongoClient.__init__", 1), + (INIT_FUNCTION_METRIC, 1), ("Datastore/all", 28), ("Datastore/allOther", 28), ("Datastore/MongoDB/all", 28), @@ -172,17 +180,17 @@ def _exercise_mongo(db): ("Datastore/operation/MongoDB/create_index", 1), ("Datastore/operation/MongoDB/find", 3), ("Datastore/operation/MongoDB/find_one", 1), - ("Datastore/statement/MongoDB/%s/create_index" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find" % MONGODB_COLLECTION, 3), - ("Datastore/statement/MongoDB/%s/find_one" % MONGODB_COLLECTION, 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/create_index", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find", 3), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one", 1), ("Datastore/operation/MongoDB/save", 3), ("Datastore/operation/MongoDB/initialize_unordered_bulk_op", 1), ("Datastore/operation/MongoDB/initialize_ordered_bulk_op", 1), ("Datastore/operation/MongoDB/parallel_scan", 1), - ("Datastore/statement/MongoDB/%s/save" % MONGODB_COLLECTION, 3), - (("Datastore/statement/MongoDB/%s" % MONGODB_COLLECTION + "/initialize_unordered_bulk_op"), 1), - (("Datastore/statement/MongoDB/%s" % MONGODB_COLLECTION + "/initialize_ordered_bulk_op"), 1), - ("Datastore/statement/MongoDB/%s/parallel_scan" % MONGODB_COLLECTION, 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/save", 3), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/initialize_unordered_bulk_op", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/initialize_ordered_bulk_op", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/parallel_scan", 1), ("Datastore/operation/MongoDB/bulk_write", 1), ("Datastore/operation/MongoDB/insert_one", 1), ("Datastore/operation/MongoDB/insert_many", 1), @@ -198,25 +206,25 @@ def _exercise_mongo(db): ("Datastore/operation/MongoDB/find_one_and_delete", 1), ("Datastore/operation/MongoDB/find_one_and_replace", 1), ("Datastore/operation/MongoDB/find_one_and_update", 1), - ("Datastore/statement/MongoDB/%s/bulk_write" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/insert_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/insert_many" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/replace_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/update_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/delete_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/delete_many" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_raw_batches" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/create_indexes" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/list_indexes" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/aggregate" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/aggregate_raw_batches" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_one_and_delete" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_one_and_replace" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_one_and_update" % MONGODB_COLLECTION, 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/bulk_write", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/insert_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/insert_many", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/replace_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/update_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/delete_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/delete_many", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_raw_batches", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/create_indexes", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/list_indexes", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/aggregate", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/aggregate_raw_batches", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one_and_delete", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one_and_replace", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one_and_update", 1), ] _test_pymongo_rollup_metrics_v4 = [ - ("Function/pymongo.mongo_client:MongoClient.__init__", 1), + (INIT_FUNCTION_METRIC, 1), ("Datastore/all", 25), ("Datastore/allOther", 25), ("Datastore/MongoDB/all", 25), @@ -224,9 +232,9 @@ def _exercise_mongo(db): ("Datastore/operation/MongoDB/create_index", 1), ("Datastore/operation/MongoDB/find", 3), ("Datastore/operation/MongoDB/find_one", 1), - ("Datastore/statement/MongoDB/%s/create_index" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find" % MONGODB_COLLECTION, 3), - ("Datastore/statement/MongoDB/%s/find_one" % MONGODB_COLLECTION, 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/create_index", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find", 3), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one", 1), ("Datastore/operation/MongoDB/bulk_write", 1), ("Datastore/operation/MongoDB/insert_one", 4), ("Datastore/operation/MongoDB/insert_many", 1), @@ -242,21 +250,21 @@ def _exercise_mongo(db): ("Datastore/operation/MongoDB/find_one_and_delete", 1), ("Datastore/operation/MongoDB/find_one_and_replace", 1), ("Datastore/operation/MongoDB/find_one_and_update", 1), - ("Datastore/statement/MongoDB/%s/bulk_write" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/insert_one" % MONGODB_COLLECTION, 4), - ("Datastore/statement/MongoDB/%s/insert_many" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/replace_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/update_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/delete_one" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/delete_many" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_raw_batches" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/create_indexes" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/list_indexes" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/aggregate" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/aggregate_raw_batches" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_one_and_delete" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_one_and_replace" % MONGODB_COLLECTION, 1), - ("Datastore/statement/MongoDB/%s/find_one_and_update" % MONGODB_COLLECTION, 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/bulk_write", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/insert_one", 4), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/insert_many", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/replace_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/update_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/delete_one", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/delete_many", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_raw_batches", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/create_indexes", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/list_indexes", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/aggregate", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/aggregate_raw_batches", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one_and_delete", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one_and_replace", 1), + (f"Datastore/statement/MongoDB/{MONGODB_COLLECTION}/find_one_and_update", 1), ] @@ -268,18 +276,16 @@ def test_mongodb_client_operation(): _test_pymongo_client_scoped_metrics = _test_pymongo_scoped_metrics_v4 _test_pymongo_client_rollup_metrics = _test_pymongo_rollup_metrics_v4 - txn_name = "test_pymongo:test_mongodb_client_operation.._test" if six.PY3 else "test_pymongo:_test" - @validate_transaction_errors(errors=[]) @validate_transaction_metrics( - txn_name, + "test_pymongo:test_mongodb_client_operation.._test", scoped_metrics=_test_pymongo_client_scoped_metrics, rollup_metrics=_test_pymongo_client_rollup_metrics, background_task=True, ) @background_task() def _test(): - client = pymongo.MongoClient(MONGODB_HOST, MONGODB_PORT) + client = MongoClient(MONGODB_HOST, MONGODB_PORT) db = client.test _exercise_mongo(db) @@ -289,7 +295,7 @@ def _test(): @validate_database_duration() @background_task() def test_mongodb_database_duration(): - client = pymongo.MongoClient(MONGODB_HOST, MONGODB_PORT) + client = MongoClient(MONGODB_HOST, MONGODB_PORT) db = client.test _exercise_mongo(db) @@ -300,7 +306,7 @@ def test_mongodb_and_sqlite_database_duration(): # Make mongodb queries - client = pymongo.MongoClient(MONGODB_HOST, MONGODB_PORT) + client = MongoClient(MONGODB_HOST, MONGODB_PORT) db = client.test _exercise_mongo(db) diff --git a/tests/datastore_pymssql/conftest.py b/tests/datastore_pymssql/conftest.py new file mode 100644 index 0000000000..70285fe136 --- /dev/null +++ b/tests/datastore_pymssql/conftest.py @@ -0,0 +1,34 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_explain_plan_queries": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (datastore_pymssql)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_pymssql/test_database.py b/tests/datastore_pymssql/test_database.py new file mode 100644 index 0000000000..ca11863b3b --- /dev/null +++ b/tests/datastore_pymssql/test_database.py @@ -0,0 +1,114 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pymssql + +from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_database_trace_inputs import validate_database_trace_inputs + +from testing_support.db_settings import mssql_settings + +from newrelic.api.background_task import background_task + +DB_SETTINGS = mssql_settings()[0] +TABLE_NAME = f"datastore_pymssql_{DB_SETTINGS['namespace']}" +PROCEDURE_NAME = f"hello_{DB_SETTINGS['namespace']}" + + +def execute_db_calls_with_cursor(cursor): + cursor.execute(f"""drop table if exists {TABLE_NAME}""") + + cursor.execute(f"create table {TABLE_NAME} (a integer, b real, c text)") + + cursor.executemany( + f"insert into {TABLE_NAME} values (%s, %s, %s)", + [(1, 1.0, "1.0"), (2, 2.2, "2.2"), (3, 3.3, "3.3")], + ) + + cursor.execute(f"""select * from {TABLE_NAME}""") + + for row in cursor: + pass + + cursor.execute(f"update {TABLE_NAME} set a=%s, b=%s, c=%s where a=%s", (4, 4.0, "4.0", 1)) + + cursor.execute(f"""delete from {TABLE_NAME} where a=2""") + cursor.execute(f"""drop procedure if exists {PROCEDURE_NAME}""") + cursor.execute( + f"""CREATE PROCEDURE {PROCEDURE_NAME} AS + BEGIN + SELECT 'Hello World!'; + END""" + ) + + cursor.callproc(PROCEDURE_NAME) + + +_test_scoped_metrics = [ + ("Function/pymssql._pymssql:connect", 1), + (f"Datastore/statement/MSSQL/{TABLE_NAME}/select", 1), + (f"Datastore/statement/MSSQL/{TABLE_NAME}/insert", 1), + (f"Datastore/statement/MSSQL/{TABLE_NAME}/update", 1), + (f"Datastore/statement/MSSQL/{TABLE_NAME}/delete", 1), + ("Datastore/operation/MSSQL/drop", 2), + ("Datastore/operation/MSSQL/create", 2), + (f"Datastore/statement/MSSQL/{PROCEDURE_NAME}/call", 1), + ("Datastore/operation/MSSQL/commit", 2), + ("Datastore/operation/MSSQL/rollback", 1), +] + +_test_rollup_metrics = [ + ("Datastore/all", 13), + ("Datastore/allOther", 13), + ("Datastore/MSSQL/all", 13), + ("Datastore/MSSQL/allOther", 13), + (f"Datastore/statement/MSSQL/{TABLE_NAME}/select", 1), + (f"Datastore/statement/MSSQL/{TABLE_NAME}/insert", 1), + (f"Datastore/statement/MSSQL/{TABLE_NAME}/update", 1), + (f"Datastore/statement/MSSQL/{TABLE_NAME}/delete", 1), + ("Datastore/operation/MSSQL/select", 1), + ("Datastore/operation/MSSQL/insert", 1), + ("Datastore/operation/MSSQL/update", 1), + ("Datastore/operation/MSSQL/delete", 1), + (f"Datastore/statement/MSSQL/{PROCEDURE_NAME}/call", 1), + ("Datastore/operation/MSSQL/call", 1), + ("Datastore/operation/MSSQL/drop", 2), + ("Datastore/operation/MSSQL/create", 2), + ("Datastore/operation/MSSQL/commit", 2), + ("Datastore/operation/MSSQL/rollback", 1), +] + + +@validate_transaction_metrics( + "test_database:test_execute_via_cursor_context_manager", + scoped_metrics=_test_scoped_metrics, + rollup_metrics=_test_rollup_metrics, + background_task=True, +) +@validate_database_trace_inputs(sql_parameters_type=tuple) +@background_task() +def test_execute_via_cursor_context_manager(): + connection = pymssql.connect( + user=DB_SETTINGS["user"], password=DB_SETTINGS["password"], host=DB_SETTINGS["host"], port=DB_SETTINGS["port"] + ) + + with connection: + cursor = connection.cursor() + + with cursor: + execute_db_calls_with_cursor(cursor) + + connection.commit() + connection.rollback() + connection.commit() diff --git a/tests/datastore_pymysql/conftest.py b/tests/datastore_pymysql/conftest.py index 51d037432d..45d6efc895 100644 --- a/tests/datastore_pymysql/conftest.py +++ b/tests/datastore_pymysql/conftest.py @@ -12,21 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, - 'debug.log_explain_plan_queries': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_explain_plan_queries": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_pymysql)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_pymysql)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_pymysql/test_database.py b/tests/datastore_pymysql/test_database.py index 5943b12665..ef55592f0c 100644 --- a/tests/datastore_pymysql/test_database.py +++ b/tests/datastore_pymysql/test_database.py @@ -13,88 +13,106 @@ # limitations under the License. import pymysql - -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_database_trace_inputs import validate_database_trace_inputs - from testing_support.db_settings import mysql_settings +from testing_support.util import instance_hostname +from testing_support.validators.validate_database_trace_inputs import ( + validate_database_trace_inputs, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.background_task import background_task DB_SETTINGS = mysql_settings()[0] -TABLE_NAME = "datastore_pymysql_" + DB_SETTINGS["namespace"] -PROCEDURE_NAME = "hello_" + DB_SETTINGS["namespace"] +TABLE_NAME = f"datastore_pymysql_{DB_SETTINGS['namespace']}" +PROCEDURE_NAME = f"hello_{DB_SETTINGS['namespace']}" + +HOST = instance_hostname(DB_SETTINGS["host"]) +PORT = DB_SETTINGS["port"] def execute_db_calls_with_cursor(cursor): - cursor.execute("""drop table if exists %s""" % TABLE_NAME) + cursor.execute(f"""drop table if exists {TABLE_NAME}""") - cursor.execute("""create table %s """ % TABLE_NAME + - """(a integer, b real, c text)""") + cursor.execute(f"create table {TABLE_NAME} (a integer, b real, c text)") - cursor.executemany("""insert into %s """ % TABLE_NAME + - """values (%s, %s, %s)""", [(1, 1.0, '1.0'), - (2, 2.2, '2.2'), (3, 3.3, '3.3')]) + cursor.executemany( + f"insert into {TABLE_NAME} values (%s, %s, %s)", + [(1, 1.0, "1.0"), (2, 2.2, "2.2"), (3, 3.3, "3.3")], + ) - cursor.execute("""select * from %s""" % TABLE_NAME) + cursor.execute(f"""select * from {TABLE_NAME}""") - for row in cursor: pass + for row in cursor: + pass - cursor.execute("""update %s""" % TABLE_NAME + """ set a=%s, b=%s, """ - """c=%s where a=%s""", (4, 4.0, '4.0', 1)) + cursor.execute(f"update {TABLE_NAME} set a=%s, b=%s, c=%s where a=%s", (4, 4.0, "4.0", 1)) - cursor.execute("""delete from %s where a=2""" % TABLE_NAME) - cursor.execute("""drop procedure if exists %s""" % PROCEDURE_NAME) - cursor.execute("""CREATE PROCEDURE %s() + cursor.execute(f"""delete from {TABLE_NAME} where a=2""") + cursor.execute(f"""drop procedure if exists {PROCEDURE_NAME}""") + cursor.execute( + f"""CREATE PROCEDURE {PROCEDURE_NAME}() BEGIN SELECT 'Hello World!'; - END""" % PROCEDURE_NAME) + END""" + ) cursor.callproc(PROCEDURE_NAME) _test_execute_via_cursor_scoped_metrics = [ - ('Function/pymysql:Connect', 1), - ('Datastore/statement/MySQL/%s/select' % TABLE_NAME, 1), - ('Datastore/statement/MySQL/%s/insert' % TABLE_NAME, 1), - ('Datastore/statement/MySQL/%s/update' % TABLE_NAME, 1), - ('Datastore/statement/MySQL/%s/delete' % TABLE_NAME, 1), - ('Datastore/operation/MySQL/drop', 2), - ('Datastore/operation/MySQL/create', 2), - ('Datastore/statement/MySQL/%s/call' % PROCEDURE_NAME, 1), - ('Datastore/operation/MySQL/commit', 2), - ('Datastore/operation/MySQL/rollback', 1)] + ("Function/pymysql:Connect", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/select", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/insert", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/update", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/delete", 1), + ("Datastore/operation/MySQL/drop", 2), + ("Datastore/operation/MySQL/create", 2), + (f"Datastore/statement/MySQL/{PROCEDURE_NAME}/call", 1), + ("Datastore/operation/MySQL/commit", 2), + ("Datastore/operation/MySQL/rollback", 1), +] _test_execute_via_cursor_rollup_metrics = [ - ('Datastore/all', 13), - ('Datastore/allOther', 13), - ('Datastore/MySQL/all', 13), - ('Datastore/MySQL/allOther', 13), - ('Datastore/statement/MySQL/%s/select' % TABLE_NAME, 1), - ('Datastore/statement/MySQL/%s/insert' % TABLE_NAME, 1), - ('Datastore/statement/MySQL/%s/update' % TABLE_NAME, 1), - ('Datastore/statement/MySQL/%s/delete' % TABLE_NAME, 1), - ('Datastore/operation/MySQL/select', 1), - ('Datastore/operation/MySQL/insert', 1), - ('Datastore/operation/MySQL/update', 1), - ('Datastore/operation/MySQL/delete', 1), - ('Datastore/statement/MySQL/%s/call' % PROCEDURE_NAME, 1), - ('Datastore/operation/MySQL/call', 1), - ('Datastore/operation/MySQL/drop', 2), - ('Datastore/operation/MySQL/create', 2), - ('Datastore/operation/MySQL/commit', 2), - ('Datastore/operation/MySQL/rollback', 1)] - -@validate_transaction_metrics('test_database:test_execute_via_cursor', - scoped_metrics=_test_execute_via_cursor_scoped_metrics, - rollup_metrics=_test_execute_via_cursor_rollup_metrics, - background_task=True) + ("Datastore/all", 13), + ("Datastore/allOther", 13), + ("Datastore/MySQL/all", 13), + ("Datastore/MySQL/allOther", 13), + (f"Datastore/statement/MySQL/{TABLE_NAME}/select", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/insert", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/update", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/delete", 1), + ("Datastore/operation/MySQL/select", 1), + ("Datastore/operation/MySQL/insert", 1), + ("Datastore/operation/MySQL/update", 1), + ("Datastore/operation/MySQL/delete", 1), + (f"Datastore/statement/MySQL/{PROCEDURE_NAME}/call", 1), + ("Datastore/operation/MySQL/call", 1), + ("Datastore/operation/MySQL/drop", 2), + ("Datastore/operation/MySQL/create", 2), + ("Datastore/operation/MySQL/commit", 2), + ("Datastore/operation/MySQL/rollback", 1), + (f"Datastore/instance/MySQL/{HOST}/{PORT}", 12), +] + + +@validate_transaction_metrics( + "test_database:test_execute_via_cursor", + scoped_metrics=_test_execute_via_cursor_scoped_metrics, + rollup_metrics=_test_execute_via_cursor_rollup_metrics, + background_task=True, +) @validate_database_trace_inputs(sql_parameters_type=tuple) @background_task() def test_execute_via_cursor(): - connection = pymysql.connect(db=DB_SETTINGS['name'], - user=DB_SETTINGS['user'], passwd=DB_SETTINGS['password'], - host=DB_SETTINGS['host'], port=DB_SETTINGS['port']) + connection = pymysql.connect( + db=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + passwd=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ) with connection.cursor() as cursor: execute_db_calls_with_cursor(cursor) @@ -105,49 +123,57 @@ def test_execute_via_cursor(): _test_execute_via_cursor_context_mangaer_scoped_metrics = [ - ('Function/pymysql:Connect', 1), - ('Datastore/statement/MySQL/%s/select' % TABLE_NAME, 1), - ('Datastore/statement/MySQL/%s/insert' % TABLE_NAME, 1), - ('Datastore/statement/MySQL/%s/update' % TABLE_NAME, 1), - ('Datastore/statement/MySQL/%s/delete' % TABLE_NAME, 1), - ('Datastore/operation/MySQL/drop', 2), - ('Datastore/operation/MySQL/create', 2), - ('Datastore/statement/MySQL/%s/call' % PROCEDURE_NAME, 1), - ('Datastore/operation/MySQL/commit', 2), - ('Datastore/operation/MySQL/rollback', 1)] + ("Function/pymysql:Connect", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/select", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/insert", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/update", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/delete", 1), + ("Datastore/operation/MySQL/drop", 2), + ("Datastore/operation/MySQL/create", 2), + (f"Datastore/statement/MySQL/{PROCEDURE_NAME}/call", 1), + ("Datastore/operation/MySQL/commit", 2), + ("Datastore/operation/MySQL/rollback", 1), +] _test_execute_via_cursor_context_mangaer_rollup_metrics = [ - ('Datastore/all', 13), - ('Datastore/allOther', 13), - ('Datastore/MySQL/all', 13), - ('Datastore/MySQL/allOther', 13), - ('Datastore/statement/MySQL/%s/select' % TABLE_NAME, 1), - ('Datastore/statement/MySQL/%s/insert' % TABLE_NAME, 1), - ('Datastore/statement/MySQL/%s/update' % TABLE_NAME, 1), - ('Datastore/statement/MySQL/%s/delete' % TABLE_NAME, 1), - ('Datastore/operation/MySQL/select', 1), - ('Datastore/operation/MySQL/insert', 1), - ('Datastore/operation/MySQL/update', 1), - ('Datastore/operation/MySQL/delete', 1), - ('Datastore/statement/MySQL/%s/call' % PROCEDURE_NAME, 1), - ('Datastore/operation/MySQL/call', 1), - ('Datastore/operation/MySQL/drop', 2), - ('Datastore/operation/MySQL/create', 2), - ('Datastore/operation/MySQL/commit', 2), - ('Datastore/operation/MySQL/rollback', 1)] + ("Datastore/all", 13), + ("Datastore/allOther", 13), + ("Datastore/MySQL/all", 13), + ("Datastore/MySQL/allOther", 13), + (f"Datastore/statement/MySQL/{TABLE_NAME}/select", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/insert", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/update", 1), + (f"Datastore/statement/MySQL/{TABLE_NAME}/delete", 1), + ("Datastore/operation/MySQL/select", 1), + ("Datastore/operation/MySQL/insert", 1), + ("Datastore/operation/MySQL/update", 1), + ("Datastore/operation/MySQL/delete", 1), + (f"Datastore/statement/MySQL/{PROCEDURE_NAME}/call", 1), + ("Datastore/operation/MySQL/call", 1), + ("Datastore/operation/MySQL/drop", 2), + ("Datastore/operation/MySQL/create", 2), + ("Datastore/operation/MySQL/commit", 2), + ("Datastore/operation/MySQL/rollback", 1), + (f"Datastore/instance/MySQL/{HOST}/{PORT}", 12), +] @validate_transaction_metrics( - 'test_database:test_execute_via_cursor_context_manager', - scoped_metrics=_test_execute_via_cursor_context_mangaer_scoped_metrics, - rollup_metrics=_test_execute_via_cursor_context_mangaer_rollup_metrics, - background_task=True) + "test_database:test_execute_via_cursor_context_manager", + scoped_metrics=_test_execute_via_cursor_context_mangaer_scoped_metrics, + rollup_metrics=_test_execute_via_cursor_context_mangaer_rollup_metrics, + background_task=True, +) @validate_database_trace_inputs(sql_parameters_type=tuple) @background_task() def test_execute_via_cursor_context_manager(): - connection = pymysql.connect(db=DB_SETTINGS['name'], - user=DB_SETTINGS['user'], passwd=DB_SETTINGS['password'], - host=DB_SETTINGS['host'], port=DB_SETTINGS['port']) + connection = pymysql.connect( + db=DB_SETTINGS["name"], + user=DB_SETTINGS["user"], + passwd=DB_SETTINGS["password"], + host=DB_SETTINGS["host"], + port=DB_SETTINGS["port"], + ) cursor = connection.cursor() with cursor: diff --git a/tests/datastore_pyodbc/conftest.py b/tests/datastore_pyodbc/conftest.py index b00a0a663f..dd646b0625 100644 --- a/tests/datastore_pyodbc/conftest.py +++ b/tests/datastore_pyodbc/conftest.py @@ -18,6 +18,7 @@ ) _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, diff --git a/tests/datastore_pyodbc/test_pyodbc.py b/tests/datastore_pyodbc/test_pyodbc.py index 119908e4db..96f9b12416 100644 --- a/tests/datastore_pyodbc/test_pyodbc.py +++ b/tests/datastore_pyodbc/test_pyodbc.py @@ -13,6 +13,7 @@ # limitations under the License. import pytest from testing_support.db_settings import postgresql_settings +from testing_support.util import instance_hostname from testing_support.validators.validate_database_trace_inputs import ( validate_database_trace_inputs, ) @@ -55,20 +56,20 @@ def test_execute_via_cursor(pyodbc_driver): ) ) as connection: cursor = connection.cursor() - cursor.execute("""drop table if exists %s""" % DB_SETTINGS["table_name"]) - cursor.execute("""create table %s """ % DB_SETTINGS["table_name"] + """(a integer, b real, c text)""") + cursor.execute(f"""drop table if exists {DB_SETTINGS['table_name']}""") + cursor.execute(f"create table {DB_SETTINGS['table_name']} (a integer, b real, c text)") cursor.executemany( - """insert into %s """ % DB_SETTINGS["table_name"] + """values (?, ?, ?)""", + f"insert into {DB_SETTINGS['table_name']} values (?, ?, ?)", [(1, 1.0, "1.0"), (2, 2.2, "2.2"), (3, 3.3, "3.3")], ) - cursor.execute("""select * from %s""" % DB_SETTINGS["table_name"]) + cursor.execute(f"""select * from {DB_SETTINGS['table_name']}""") for row in cursor: pass cursor.execute( - """update %s """ % DB_SETTINGS["table_name"] + """set a=?, b=?, c=? where a=?""", + f"update {DB_SETTINGS['table_name']} set a=?, b=?, c=? where a=?", (4, 4.0, "4.0", 1), ) - cursor.execute("""delete from %s where a=2""" % DB_SETTINGS["table_name"]) + cursor.execute(f"""delete from {DB_SETTINGS['table_name']} where a=2""") connection.commit() cursor.execute("SELECT now()") diff --git a/tests/datastore_pysolr/conftest.py b/tests/datastore_pysolr/conftest.py index 07851b6981..6886f506a8 100644 --- a/tests/datastore_pysolr/conftest.py +++ b/tests/datastore_pysolr/conftest.py @@ -12,20 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_pysolr)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_pysolr)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_pysolr/test_solr.py b/tests/datastore_pysolr/test_solr.py index a987a29ac9..b47fa4e11b 100644 --- a/tests/datastore_pysolr/test_solr.py +++ b/tests/datastore_pysolr/test_solr.py @@ -13,48 +13,57 @@ # limitations under the License. from pysolr import Solr - -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from testing_support.db_settings import solr_settings +from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.background_task import background_task DB_SETTINGS = solr_settings()[0] SOLR_HOST = DB_SETTINGS["host"] SOLR_PORT = DB_SETTINGS["port"] -SOLR_URL = 'http://%s:%s/solr/collection' % (DB_SETTINGS["host"], DB_SETTINGS["port"]) +SOLR_URL = f"http://{DB_SETTINGS['host']}:{DB_SETTINGS['port']}/solr/collection" + def _exercise_solr(solr): # Construct document names within namespace documents = ["pysolr_doc_1", "pysolr_doc_2"] - documents = [x + "_" + DB_SETTINGS["namespace"] for x in documents] + documents = [f"{x}_{DB_SETTINGS['namespace']}" for x in documents] solr.add([{"id": x} for x in documents]) - solr.search('id:%s' % documents[0]) + solr.search(f"id:{documents[0]}") solr.delete(id=documents[0]) # Delete all documents. - solr.delete(q='id:*_%s' % DB_SETTINGS["namespace"]) + solr.delete(q=f"id:*_{DB_SETTINGS['namespace']}") + _test_solr_search_scoped_metrics = [ - ('Datastore/operation/Solr/add', 1), - ('Datastore/operation/Solr/delete', 2), - ('Datastore/operation/Solr/search', 1)] + ("Datastore/operation/Solr/add", 1), + ("Datastore/operation/Solr/delete", 2), + ("Datastore/operation/Solr/search", 1), +] _test_solr_search_rollup_metrics = [ - ('Datastore/all', 4), - ('Datastore/allOther', 4), - ('Datastore/Solr/all', 4), - ('Datastore/Solr/allOther', 4), - ('Datastore/operation/Solr/add', 1), - ('Datastore/operation/Solr/search', 1), - ('Datastore/operation/Solr/delete', 2)] - -@validate_transaction_metrics('test_solr:test_solr_search', + ("Datastore/all", 4), + ("Datastore/allOther", 4), + ("Datastore/Solr/all", 4), + ("Datastore/Solr/allOther", 4), + ("Datastore/operation/Solr/add", 1), + ("Datastore/operation/Solr/search", 1), + ("Datastore/operation/Solr/delete", 2), +] + + +@validate_transaction_metrics( + "test_solr:test_solr_search", scoped_metrics=_test_solr_search_scoped_metrics, rollup_metrics=_test_solr_search_rollup_metrics, - background_task=True) + background_task=True, +) @background_task() def test_solr_search(): s = Solr(SOLR_URL) diff --git a/tests/datastore_redis/conftest.py b/tests/datastore_redis/conftest.py index 53ff2658de..e939e15c32 100644 --- a/tests/datastore_redis/conftest.py +++ b/tests/datastore_redis/conftest.py @@ -12,20 +12,26 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixture.event_loop import ( # noqa: F401; pylint: disable=W0611 + event_loop as loop, +) +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_redis)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_redis)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_redis/test_asyncio.py b/tests/datastore_redis/test_asyncio.py new file mode 100644 index 0000000000..47eec2d1d9 --- /dev/null +++ b/tests/datastore_redis/test_asyncio.py @@ -0,0 +1,160 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio + +import pytest +from testing_support.db_settings import redis_settings +from testing_support.fixture.event_loop import event_loop as loop # noqa: F401 +from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple + +# Settings + +DB_SETTINGS = redis_settings()[0] +REDIS_PY_VERSION = get_package_version_tuple("redis") + +# Metrics for publish test + +datastore_all_metric_count = 5 if REDIS_PY_VERSION >= (5, 0) else 3 + +_base_scoped_metrics = [("Datastore/operation/Redis/publish", 3)] + +if REDIS_PY_VERSION >= (5, 0): + _base_scoped_metrics.append( + ("Datastore/operation/Redis/client_setinfo", 2), + ) + +_base_rollup_metrics = [ + ("Datastore/all", datastore_all_metric_count), + ("Datastore/allOther", datastore_all_metric_count), + ("Datastore/Redis/all", datastore_all_metric_count), + ("Datastore/Redis/allOther", datastore_all_metric_count), + ("Datastore/operation/Redis/publish", 3), + ( + f"Datastore/instance/Redis/{instance_hostname(DB_SETTINGS['host'])}/{DB_SETTINGS['port']}", + datastore_all_metric_count, + ), +] +if REDIS_PY_VERSION >= (5, 0): + _base_rollup_metrics.append( + ("Datastore/operation/Redis/client_setinfo", 2), + ) + + +# Metrics for connection pool test + +_base_pool_scoped_metrics = [ + ("Datastore/operation/Redis/get", 1), + ("Datastore/operation/Redis/set", 1), + ("Datastore/operation/Redis/client_list", 1), +] + +_base_pool_rollup_metrics = [ + ("Datastore/all", 3), + ("Datastore/allOther", 3), + ("Datastore/Redis/all", 3), + ("Datastore/Redis/allOther", 3), + ("Datastore/operation/Redis/get", 1), + ("Datastore/operation/Redis/set", 1), + ("Datastore/operation/Redis/client_list", 1), + (f"Datastore/instance/Redis/{instance_hostname(DB_SETTINGS['host'])}/{DB_SETTINGS['port']}", 3), +] + + +# Tests + + +@pytest.fixture() +def client(loop): # noqa + import redis.asyncio + + return loop.run_until_complete(redis.asyncio.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0)) + + +@pytest.fixture() +def client_pool(loop): # noqa + import redis.asyncio + + connection_pool = redis.asyncio.ConnectionPool(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) + return loop.run_until_complete(redis.asyncio.Redis(connection_pool=connection_pool)) + + +@pytest.mark.skipif(REDIS_PY_VERSION < (4, 2), reason="This functionality exists in Redis 4.2+") +@validate_transaction_metrics( + "test_asyncio:test_async_connection_pool", + scoped_metrics=_base_pool_scoped_metrics, + rollup_metrics=_base_pool_rollup_metrics, + background_task=True, +) +@background_task() +def test_async_connection_pool(client_pool, loop): # noqa + async def _test_async_pool(client_pool): + await client_pool.set("key1", "value1") + await client_pool.get("key1") + await client_pool.execute_command("CLIENT", "LIST") + + loop.run_until_complete(_test_async_pool(client_pool)) + + +@pytest.mark.skipif(REDIS_PY_VERSION < (4, 2), reason="This functionality exists in Redis 4.2+") +@validate_transaction_metrics("test_asyncio:test_async_pipeline", background_task=True) +@background_task() +def test_async_pipeline(client, loop): # noqa + async def _test_pipeline(client): + async with client.pipeline(transaction=True) as pipe: + await pipe.set("key1", "value1") + await pipe.execute() + + loop.run_until_complete(_test_pipeline(client)) + + +@pytest.mark.skipif(REDIS_PY_VERSION < (4, 2), reason="This functionality exists in Redis 4.2+") +@validate_transaction_metrics( + "test_asyncio:test_async_pubsub", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_base_rollup_metrics, + background_task=True, +) +@background_task() +def test_async_pubsub(client, loop): # noqa + messages_received = [] + + async def reader(pubsub): + while True: + message = await pubsub.get_message(ignore_subscribe_messages=True) + if message: + messages_received.append(message["data"].decode()) + if message["data"].decode() == "NOPE": + break + + async def _test_pubsub(): + async with client.pubsub() as pubsub: + await pubsub.psubscribe("channel:*") + + future = asyncio.create_task(reader(pubsub)) + + await client.publish("channel:1", "Hello") + await client.publish("channel:2", "World") + await client.publish("channel:1", "NOPE") + + await future + + loop.run_until_complete(_test_pubsub()) + assert messages_received == ["Hello", "World", "NOPE"] diff --git a/tests/datastore_redis/test_custom_conn_pool.py b/tests/datastore_redis/test_custom_conn_pool.py index 156c9ce31f..70954f2ce3 100644 --- a/tests/datastore_redis/test_custom_conn_pool.py +++ b/tests/datastore_redis/test_custom_conn_pool.py @@ -12,26 +12,28 @@ # See the License for the specific language governing permissions and # limitations under the License. -''' The purpose of these tests is to confirm that using a non-standard +""" The purpose of these tests is to confirm that using a non-standard connection pool that does not have a `connection_kwargs` attribute will not result in an error. -''' +""" import pytest import redis - -from newrelic.api.background_task import background_task - -from testing_support.fixtures import override_application_settings -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from testing_support.db_settings import redis_settings +from testing_support.fixtures import override_application_settings from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple DB_SETTINGS = redis_settings()[0] -REDIS_PY_VERSION = redis.VERSION +REDIS_PY_VERSION = get_package_version_tuple("redis") -class FakeConnectionPool(object): +class FakeConnectionPool(): """Connection Pool without connection_kwargs attribute.""" def __init__(self, connection): @@ -43,112 +45,120 @@ def get_connection(self, name, *keys, **options): def release(self, connection): self.connection.disconnect() + def disconnect(self): + self.connection.disconnect() + + # Settings _enable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': True, + "datastore_tracer.instance_reporting.enabled": True, } _disable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': False, + "datastore_tracer.instance_reporting.enabled": False, } # Metrics # We don't record instance metrics when using redis blaster, # so we just check for base metrics. - -_base_scoped_metrics = ( - ('Datastore/operation/Redis/get', 1), - ('Datastore/operation/Redis/set', 1), - ('Datastore/operation/Redis/client_list', 1), -) - -_base_rollup_metrics = ( - ('Datastore/all', 3), - ('Datastore/allOther', 3), - ('Datastore/Redis/all', 3), - ('Datastore/Redis/allOther', 3), - ('Datastore/operation/Redis/get', 1), - ('Datastore/operation/Redis/set', 1), - ('Datastore/operation/Redis/client_list', 1), -) - -_disable_scoped_metrics = list(_base_scoped_metrics) -_disable_rollup_metrics = list(_base_rollup_metrics) - -_enable_scoped_metrics = list(_base_scoped_metrics) -_enable_rollup_metrics = list(_base_rollup_metrics) - -_host = instance_hostname(DB_SETTINGS['host']) -_port = DB_SETTINGS['port'] - -_instance_metric_name = 'Datastore/instance/Redis/%s/%s' % (_host, _port) - -_enable_rollup_metrics.append( - (_instance_metric_name, 3) -) - -_disable_rollup_metrics.append( - (_instance_metric_name, None) -) +datastore_all_metric_count = 5 if REDIS_PY_VERSION >= (5, 0) else 3 + +_base_scoped_metrics = [ + ("Datastore/operation/Redis/get", 1), + ("Datastore/operation/Redis/set", 1), + ("Datastore/operation/Redis/client_list", 1), +] +# client_setinfo was introduced in v5.0.0 and assigns info displayed in client_list output +if REDIS_PY_VERSION >= (5, 0): + _base_scoped_metrics.append( + ("Datastore/operation/Redis/client_setinfo", 2), + ) + +_base_rollup_metrics = [ + ("Datastore/all", datastore_all_metric_count), + ("Datastore/allOther", datastore_all_metric_count), + ("Datastore/Redis/all", datastore_all_metric_count), + ("Datastore/Redis/allOther", datastore_all_metric_count), + ("Datastore/operation/Redis/get", 1), + ("Datastore/operation/Redis/set", 1), + ("Datastore/operation/Redis/client_list", 1), +] +if REDIS_PY_VERSION >= (5, 0): + _base_rollup_metrics.append( + ("Datastore/operation/Redis/client_setinfo", 2), + ) + +_host = instance_hostname(DB_SETTINGS["host"]) +_port = DB_SETTINGS["port"] + +_instance_metric_name = f"Datastore/instance/Redis/{_host}/{_port}" + +instance_metric_count = 5 if REDIS_PY_VERSION >= (5, 0) else 3 + +_enable_rollup_metrics = _base_rollup_metrics.append((_instance_metric_name, instance_metric_count)) + +_disable_rollup_metrics = _base_rollup_metrics.append((_instance_metric_name, None)) # Operations + def exercise_redis(client): - client.set('key', 'value') - client.get('key') - client.execute_command('CLIENT', 'LIST', parse='LIST') + client.set("key", "value") + client.get("key") + client.execute_command("CLIENT", "LIST", parse="LIST") + # Tests -@pytest.mark.skipif(REDIS_PY_VERSION < (2, 7), - reason='Client list command introduced in 2.7') + +@pytest.mark.skipif(REDIS_PY_VERSION < (2, 7), reason="Client list command introduced in 2.7") @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( - 'test_custom_conn_pool:test_fake_conn_pool_enable_instance', - scoped_metrics=_enable_scoped_metrics, - rollup_metrics=_enable_rollup_metrics, - background_task=True) + "test_custom_conn_pool:test_fake_conn_pool_enable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) @background_task() def test_fake_conn_pool_enable_instance(): - client = redis.StrictRedis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + client = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) # Get a real connection - conn = client.connection_pool.get_connection('GET') + conn = client.connection_pool.get_connection("GET") # Replace the original connection pool with one that doesn't # have the `connection_kwargs` attribute. fake_pool = FakeConnectionPool(conn) client.connection_pool = fake_pool - assert not hasattr(client.connection_pool, 'connection_kwargs') + assert not hasattr(client.connection_pool, "connection_kwargs") exercise_redis(client) -@pytest.mark.skipif(REDIS_PY_VERSION < (2, 7), - reason='Client list command introduced in 2.7') + +@pytest.mark.skipif(REDIS_PY_VERSION < (2, 7), reason="Client list command introduced in 2.7") @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( - 'test_custom_conn_pool:test_fake_conn_pool_disable_instance', - scoped_metrics=_disable_scoped_metrics, - rollup_metrics=_disable_rollup_metrics, - background_task=True) + "test_custom_conn_pool:test_fake_conn_pool_disable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) @background_task() def test_fake_conn_pool_disable_instance(): - client = redis.StrictRedis(host=DB_SETTINGS['host'], - port=DB_SETTINGS['port'], db=0) + client = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) # Get a real connection - conn = client.connection_pool.get_connection('GET') + conn = client.connection_pool.get_connection("GET") # Replace the original connection pool with one that doesn't # have the `connection_kwargs` attribute. fake_pool = FakeConnectionPool(conn) client.connection_pool = fake_pool - assert not hasattr(client.connection_pool, 'connection_kwargs') + assert not hasattr(client.connection_pool, "connection_kwargs") exercise_redis(client) diff --git a/tests/datastore_redis/test_execute_command.py b/tests/datastore_redis/test_execute_command.py index 7475880725..ebc52d32e0 100644 --- a/tests/datastore_redis/test_execute_command.py +++ b/tests/datastore_redis/test_execute_command.py @@ -16,6 +16,7 @@ import redis from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple from testing_support.fixtures import override_application_settings from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics @@ -23,7 +24,8 @@ from testing_support.util import instance_hostname DB_SETTINGS = redis_settings()[0] -REDIS_PY_VERSION = redis.VERSION +REDIS_PY_VERSION = get_package_version_tuple("redis") + # Settings @@ -36,34 +38,34 @@ # Metrics -_base_scoped_metrics = ( +_base_scoped_metrics = [ ('Datastore/operation/Redis/client_list', 1), -) - -_base_rollup_metrics = ( - ('Datastore/all', 1), - ('Datastore/allOther', 1), - ('Datastore/Redis/all', 1), - ('Datastore/Redis/allOther', 1), +] +if REDIS_PY_VERSION >= (5, 0): + _base_scoped_metrics.append(('Datastore/operation/Redis/client_setinfo', 2),) + +_base_rollup_metrics = [ + ('Datastore/all', 3), + ('Datastore/allOther', 3), + ('Datastore/Redis/all', 3), + ('Datastore/Redis/allOther', 3), ('Datastore/operation/Redis/client_list', 1), -) - -_disable_scoped_metrics = list(_base_scoped_metrics) -_disable_rollup_metrics = list(_base_rollup_metrics) - -_enable_scoped_metrics = list(_base_scoped_metrics) -_enable_rollup_metrics = list(_base_rollup_metrics) +] +if REDIS_PY_VERSION >= (5, 0): + _base_rollup_metrics.append(('Datastore/operation/Redis/client_setinfo', 2),) _host = instance_hostname(DB_SETTINGS['host']) _port = DB_SETTINGS['port'] -_instance_metric_name = 'Datastore/instance/Redis/%s/%s' % (_host, _port) +_instance_metric_name = f'Datastore/instance/Redis/{_host}/{_port}' + +instance_metric_count = 3 if REDIS_PY_VERSION >= (5, 0) else 1 -_enable_rollup_metrics.append( - (_instance_metric_name, 1) +_enable_rollup_metrics = _base_rollup_metrics.append( + (_instance_metric_name, instance_metric_count) ) -_disable_rollup_metrics.append( +_disable_rollup_metrics = _base_rollup_metrics.append( (_instance_metric_name, None) ) @@ -76,7 +78,7 @@ def exercise_redis_single_arg(client): @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( 'test_execute_command:test_strict_redis_execute_command_two_args_enable', - scoped_metrics=_enable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_enable_rollup_metrics, background_task=True) @background_task() @@ -88,7 +90,7 @@ def test_strict_redis_execute_command_two_args_enable(): @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( 'test_execute_command:test_strict_redis_execute_command_two_args_disabled', - scoped_metrics=_disable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_disable_rollup_metrics, background_task=True) @background_task() @@ -100,7 +102,7 @@ def test_strict_redis_execute_command_two_args_disabled(): @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( 'test_execute_command:test_redis_execute_command_two_args_enable', - scoped_metrics=_enable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_enable_rollup_metrics, background_task=True) @background_task() @@ -112,7 +114,7 @@ def test_redis_execute_command_two_args_enable(): @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( 'test_execute_command:test_redis_execute_command_two_args_disabled', - scoped_metrics=_disable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_disable_rollup_metrics, background_task=True) @background_task() @@ -126,7 +128,7 @@ def test_redis_execute_command_two_args_disabled(): @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( 'test_execute_command:test_strict_redis_execute_command_as_one_arg_enable', - scoped_metrics=_enable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_enable_rollup_metrics, background_task=True) @background_task() @@ -140,7 +142,7 @@ def test_strict_redis_execute_command_as_one_arg_enable(): @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( 'test_execute_command:test_strict_redis_execute_command_as_one_arg_disabled', - scoped_metrics=_disable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_disable_rollup_metrics, background_task=True) @background_task() @@ -154,7 +156,7 @@ def test_strict_redis_execute_command_as_one_arg_disabled(): @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( 'test_execute_command:test_redis_execute_command_as_one_arg_enable', - scoped_metrics=_enable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_enable_rollup_metrics, background_task=True) @background_task() @@ -168,7 +170,7 @@ def test_redis_execute_command_as_one_arg_enable(): @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( 'test_execute_command:test_redis_execute_command_as_one_arg_disabled', - scoped_metrics=_disable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_disable_rollup_metrics, background_task=True) @background_task() diff --git a/tests/datastore_redis/test_generators.py b/tests/datastore_redis/test_generators.py new file mode 100644 index 0000000000..13593c1576 --- /dev/null +++ b/tests/datastore_redis/test_generators.py @@ -0,0 +1,258 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +import redis +from testing_support.db_settings import redis_settings +from testing_support.fixtures import override_application_settings +from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.datastore_trace import DatastoreTrace +from newrelic.api.time_trace import current_trace +from newrelic.common.package_version_utils import get_package_version_tuple + +DB_SETTINGS = redis_settings()[0] +REDIS_PY_VERSION = get_package_version_tuple("redis") + +# Settings + +_enable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": True, +} +_disable_instance_settings = { + "datastore_tracer.instance_reporting.enabled": False, +} + +# Metrics + +_base_scoped_metrics = ( + ("Datastore/operation/Redis/scan_iter", 1), + ("Datastore/operation/Redis/sscan_iter", 1), + ("Datastore/operation/Redis/zscan_iter", 1), + ("Datastore/operation/Redis/hscan_iter", 1), + ("Datastore/operation/Redis/set", 1), + ("Datastore/operation/Redis/sadd", 1), + ("Datastore/operation/Redis/zadd", 1), + ("Datastore/operation/Redis/hset", 1), +) + +_base_rollup_metrics = ( + ("Datastore/all", 8), + ("Datastore/allOther", 8), + ("Datastore/Redis/all", 8), + ("Datastore/Redis/allOther", 8), + ("Datastore/operation/Redis/scan_iter", 1), + ("Datastore/operation/Redis/sscan_iter", 1), + ("Datastore/operation/Redis/zscan_iter", 1), + ("Datastore/operation/Redis/hscan_iter", 1), + ("Datastore/operation/Redis/set", 1), + ("Datastore/operation/Redis/sadd", 1), + ("Datastore/operation/Redis/zadd", 1), + ("Datastore/operation/Redis/hset", 1), +) + +_disable_rollup_metrics = list(_base_rollup_metrics) +_enable_rollup_metrics = list(_base_rollup_metrics) + +_host = instance_hostname(DB_SETTINGS["host"]) +_port = DB_SETTINGS["port"] + +_instance_metric_name = f"Datastore/instance/Redis/{_host}/{_port}" + +_enable_rollup_metrics.append((_instance_metric_name, 8)) + +_disable_rollup_metrics.append((_instance_metric_name, None)) + +# Operations + + +def exercise_redis(client): + """ + Exercise client generators by iterating on various methods and ensuring they are + non-empty, and that traces are started and stopped with the generator. + """ + + # Set existing values + client.set("scan-key", "value") + client.sadd("sscan-key", "value") + client.zadd("zscan-key", {"value": 1}) + client.hset("hscan-key", "field", "value") + + # Check generators + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + for k in client.scan_iter("scan-*"): + assert k == b"scan-key" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + for k in client.sscan_iter("sscan-key"): + assert k == b"value" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + for k, _ in client.zscan_iter("zscan-key"): + assert k == b"value" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + for f, v in client.hscan_iter("hscan-key"): + assert f == b"field" + assert v == b"value" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + +async def exercise_redis_async(client): + """ + Exercise client generators by iterating on various methods and ensuring they are + non-empty, and that traces are started and stopped with the generator. + """ + + # Set existing values + await client.set("scan-key", "value") + await client.sadd("sscan-key", "value") + await client.zadd("zscan-key", {"value": 1}) + await client.hset("hscan-key", "field", "value") + + # Check generators + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + async for k in client.scan_iter("scan-*"): + assert k == b"scan-key" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + async for k in client.sscan_iter("sscan-key"): + assert k == b"value" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + async for k, _ in client.zscan_iter("zscan-key"): + assert k == b"value" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + flag = False + assert not isinstance(current_trace(), DatastoreTrace) # Assert no active DatastoreTrace + async for f, v in client.hscan_iter("hscan-key"): + assert f == b"field" + assert v == b"value" + assert isinstance(current_trace(), DatastoreTrace) # Assert DatastoreTrace now active + flag = True + assert flag + + +# Tests + + +@override_application_settings(_enable_instance_settings) +@validate_transaction_metrics( + "test_generators:test_strict_redis_generator_enable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) +@background_task() +def test_strict_redis_generator_enable_instance(): + client = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) + exercise_redis(client) + + +@override_application_settings(_disable_instance_settings) +@validate_transaction_metrics( + "test_generators:test_strict_redis_generator_disable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) +@background_task() +def test_strict_redis_generator_disable_instance(): + client = redis.StrictRedis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) + exercise_redis(client) + + +@override_application_settings(_enable_instance_settings) +@validate_transaction_metrics( + "test_generators:test_redis_generator_enable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) +@background_task() +def test_redis_generator_enable_instance(): + client = redis.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) + exercise_redis(client) + + +@override_application_settings(_disable_instance_settings) +@validate_transaction_metrics( + "test_generators:test_redis_generator_disable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) +@background_task() +def test_redis_generator_disable_instance(): + client = redis.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) + exercise_redis(client) + + +@pytest.mark.skipif(REDIS_PY_VERSION < (4, 2), reason="Redis.asyncio was not added until v4.2") +@override_application_settings(_enable_instance_settings) +@validate_transaction_metrics( + "test_generators:test_redis_async_generator_enable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_enable_rollup_metrics, + background_task=True, +) +@background_task() +def test_redis_async_generator_enable_instance(loop): + client = redis.asyncio.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) + loop.run_until_complete(exercise_redis_async(client)) + + +@pytest.mark.skipif(REDIS_PY_VERSION < (4, 2), reason="Redis.asyncio was not added until v4.2") +@override_application_settings(_disable_instance_settings) +@validate_transaction_metrics( + "test_generators:test_redis_async_generator_disable_instance", + scoped_metrics=_base_scoped_metrics, + rollup_metrics=_disable_rollup_metrics, + background_task=True, +) +@background_task() +def test_redis_async_generator_disable_instance(loop): + client = redis.asyncio.Redis(host=DB_SETTINGS["host"], port=DB_SETTINGS["port"], db=0) + loop.run_until_complete(exercise_redis_async(client)) diff --git a/tests/datastore_redis/test_get_and_set.py b/tests/datastore_redis/test_get_and_set.py index 0e2df4bb1d..3a38ef37e1 100644 --- a/tests/datastore_redis/test_get_and_set.py +++ b/tests/datastore_redis/test_get_and_set.py @@ -48,16 +48,13 @@ ('Datastore/operation/Redis/set', 1), ) -_disable_scoped_metrics = list(_base_scoped_metrics) _disable_rollup_metrics = list(_base_rollup_metrics) - -_enable_scoped_metrics = list(_base_scoped_metrics) _enable_rollup_metrics = list(_base_rollup_metrics) _host = instance_hostname(DB_SETTINGS['host']) _port = DB_SETTINGS['port'] -_instance_metric_name = 'Datastore/instance/Redis/%s/%s' % (_host, _port) +_instance_metric_name = f'Datastore/instance/Redis/{_host}/{_port}' _enable_rollup_metrics.append( (_instance_metric_name, 2) @@ -78,7 +75,7 @@ def exercise_redis(client): @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( 'test_get_and_set:test_strict_redis_operation_enable_instance', - scoped_metrics=_enable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_enable_rollup_metrics, background_task=True) @background_task() @@ -90,7 +87,7 @@ def test_strict_redis_operation_enable_instance(): @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( 'test_get_and_set:test_strict_redis_operation_disable_instance', - scoped_metrics=_disable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_disable_rollup_metrics, background_task=True) @background_task() @@ -102,7 +99,7 @@ def test_strict_redis_operation_disable_instance(): @override_application_settings(_enable_instance_settings) @validate_transaction_metrics( 'test_get_and_set:test_redis_operation_enable_instance', - scoped_metrics=_enable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_enable_rollup_metrics, background_task=True) @background_task() @@ -114,7 +111,7 @@ def test_redis_operation_enable_instance(): @override_application_settings(_disable_instance_settings) @validate_transaction_metrics( 'test_get_and_set:test_redis_operation_disable_instance', - scoped_metrics=_disable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_disable_rollup_metrics, background_task=True) @background_task() diff --git a/tests/datastore_redis/test_instance_info.py b/tests/datastore_redis/test_instance_info.py index b3e9a0d5d7..211e96169a 100644 --- a/tests/datastore_redis/test_instance_info.py +++ b/tests/datastore_redis/test_instance_info.py @@ -15,9 +15,10 @@ import pytest import redis +from newrelic.common.package_version_utils import get_package_version_tuple from newrelic.hooks.datastore_redis import _conn_attrs_to_dict, _instance_info -REDIS_PY_VERSION = redis.VERSION +REDIS_PY_VERSION = get_package_version_tuple("redis") _instance_info_tests = [ ((), {}, ("localhost", "6379", "0")), diff --git a/tests/datastore_redis/test_multiple_dbs.py b/tests/datastore_redis/test_multiple_dbs.py index 15777cc38f..f183e5cc17 100644 --- a/tests/datastore_redis/test_multiple_dbs.py +++ b/tests/datastore_redis/test_multiple_dbs.py @@ -16,6 +16,7 @@ import redis from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple from testing_support.fixtures import override_application_settings from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics @@ -23,6 +24,8 @@ from testing_support.util import instance_hostname DB_MULTIPLE_SETTINGS = redis_settings() +REDIS_PY_VERSION = get_package_version_tuple("redis") + # Settings @@ -35,27 +38,31 @@ # Metrics -_base_scoped_metrics = ( +_base_scoped_metrics = [ ('Datastore/operation/Redis/get', 1), ('Datastore/operation/Redis/set', 1), ('Datastore/operation/Redis/client_list', 1), -) - -_base_rollup_metrics = ( - ('Datastore/all', 3), - ('Datastore/allOther', 3), - ('Datastore/Redis/all', 3), - ('Datastore/Redis/allOther', 3), +] +# client_setinfo was introduced in v5.0.0 and assigns info displayed in client_list output +if REDIS_PY_VERSION >= (5, 0): + _base_scoped_metrics.append(('Datastore/operation/Redis/client_setinfo', 2),) + +datastore_all_metric_count = 5 if REDIS_PY_VERSION >= (5, 0) else 3 + +_base_rollup_metrics = [ + ('Datastore/all', datastore_all_metric_count), + ('Datastore/allOther', datastore_all_metric_count), + ('Datastore/Redis/all', datastore_all_metric_count), + ('Datastore/Redis/allOther', datastore_all_metric_count), ('Datastore/operation/Redis/get', 1), ('Datastore/operation/Redis/set', 1), ('Datastore/operation/Redis/client_list', 1), -) +] -_disable_scoped_metrics = list(_base_scoped_metrics) -_disable_rollup_metrics = list(_base_rollup_metrics) +# client_setinfo was introduced in v5.0.0 and assigns info displayed in client_list output +if REDIS_PY_VERSION >= (5, 0): + _base_rollup_metrics.append(('Datastore/operation/Redis/client_setinfo', 2),) -_enable_scoped_metrics = list(_base_scoped_metrics) -_enable_rollup_metrics = list(_base_rollup_metrics) if len(DB_MULTIPLE_SETTINGS) > 1: redis_1 = DB_MULTIPLE_SETTINGS[0] @@ -67,19 +74,23 @@ host_2 = instance_hostname(redis_2['host']) port_2 = redis_2['port'] - instance_metric_name_1 = 'Datastore/instance/Redis/%s/%s' % (host_1, port_1) - instance_metric_name_2 = 'Datastore/instance/Redis/%s/%s' % (host_2, port_2) + instance_metric_name_1 = f'Datastore/instance/Redis/{host_1}/{port_1}' + instance_metric_name_2 = f'Datastore/instance/Redis/{host_2}/{port_2}' - _enable_rollup_metrics.extend([ - (instance_metric_name_1, 2), - (instance_metric_name_2, 1), + instance_metric_name_1_count = 2 if REDIS_PY_VERSION >= (5, 0) else 2 + instance_metric_name_2_count = 3 if REDIS_PY_VERSION >= (5, 0) else 1 + + _enable_rollup_metrics = _base_rollup_metrics.extend([ + (instance_metric_name_1, instance_metric_name_1_count), + (instance_metric_name_2, instance_metric_name_2_count), ]) - _disable_rollup_metrics.extend([ + _disable_rollup_metrics = _base_rollup_metrics.extend([ (instance_metric_name_1, None), (instance_metric_name_2, None), ]) + def exercise_redis(client_1, client_2): client_1.set('key', 'value') client_1.get('key') @@ -90,7 +101,7 @@ def exercise_redis(client_1, client_2): reason='Test environment not configured with multiple databases.') @override_application_settings(_enable_instance_settings) @validate_transaction_metrics('test_multiple_dbs:test_multiple_datastores_enabled', - scoped_metrics=_enable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_enable_rollup_metrics, background_task=True) @background_task() @@ -106,7 +117,7 @@ def test_multiple_datastores_enabled(): reason='Test environment not configured with multiple databases.') @override_application_settings(_disable_instance_settings) @validate_transaction_metrics('test_multiple_dbs:test_multiple_datastores_disabled', - scoped_metrics=_disable_scoped_metrics, + scoped_metrics=_base_scoped_metrics, rollup_metrics=_disable_rollup_metrics, background_task=True) @background_task() diff --git a/tests/datastore_redis/test_rb.py b/tests/datastore_redis/test_rb.py deleted file mode 100644 index 5678c2787f..0000000000 --- a/tests/datastore_redis/test_rb.py +++ /dev/null @@ -1,138 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -''' The purpose of these tests is to confirm that we will record -record instance info for Redis Blaster commands that go through -redis.Connection:send_command(). Commands that don't use send_command, -like the one that use the fanout client, won't have instance info. -''' - -import pytest -import redis -import six - -from newrelic.api.background_task import background_task - -from testing_support.fixtures import override_application_settings -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.db_settings import redis_settings -from testing_support.util import instance_hostname - -DB_SETTINGS = redis_settings()[0] -REDIS_PY_VERSION = redis.VERSION - - -# Settings - -_enable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': True, -} -_disable_instance_settings = { - 'datastore_tracer.instance_reporting.enabled': False, -} - -# Metrics - -# We don't record instance metrics when using redis blaster, -# so we just check for base metrics. - -_base_scoped_metrics = ( - ('Datastore/operation/Redis/get', 1), - ('Datastore/operation/Redis/set', 1), -) - -_base_rollup_metrics = ( - ('Datastore/all', 2), - ('Datastore/allOther', 2), - ('Datastore/Redis/all', 2), - ('Datastore/Redis/allOther', 2), - ('Datastore/operation/Redis/get', 1), - ('Datastore/operation/Redis/set', 1), -) - -_disable_scoped_metrics = list(_base_scoped_metrics) -_disable_rollup_metrics = list(_base_rollup_metrics) - -_enable_scoped_metrics = list(_base_scoped_metrics) -_enable_rollup_metrics = list(_base_rollup_metrics) - -_host = instance_hostname(DB_SETTINGS['host']) -_port = DB_SETTINGS['port'] - -_instance_metric_name = 'Datastore/instance/Redis/%s/%s' % (_host, _port) - -_enable_rollup_metrics.append( - (_instance_metric_name, 2) -) - -_disable_rollup_metrics.append( - (_instance_metric_name, None) -) - -# Operations - -def exercise_redis(routing_client): - routing_client.set('key', 'value') - routing_client.get('key') - -def exercise_fanout(cluster): - with cluster.fanout(hosts='all') as client: - client.execute_command('CLIENT', 'LIST') - -# Tests - -@pytest.mark.skipif(six.PY3, reason='Redis Blaster is Python 2 only.') -@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10, 2), - reason='Redis Blaster requires redis>=2.10.2') -@override_application_settings(_enable_instance_settings) -@validate_transaction_metrics( - 'test_rb:test_redis_blaster_operation_enable_instance', - scoped_metrics=_enable_scoped_metrics, - rollup_metrics=_enable_rollup_metrics, - background_task=True) -@background_task() -def test_redis_blaster_operation_enable_instance(): - from rb import Cluster - - cluster = Cluster( - hosts={0: {'port': DB_SETTINGS['port']}}, - host_defaults={'host': DB_SETTINGS['host']} - ) - exercise_fanout(cluster) - - client = cluster.get_routing_client() - exercise_redis(client) - - -@pytest.mark.skipif(six.PY3, reason='Redis Blaster is Python 2 only.') -@pytest.mark.skipif(REDIS_PY_VERSION < (2, 10,2 ), - reason='Redis Blaster requires redis>=2.10.2') -@override_application_settings(_disable_instance_settings) -@validate_transaction_metrics( - 'test_rb:test_redis_blaster_operation_disable_instance', - scoped_metrics=_disable_scoped_metrics, - rollup_metrics=_disable_rollup_metrics, - background_task=True) -@background_task() -def test_redis_blaster_operation_disable_instance(): - from rb import Cluster - - cluster = Cluster( - hosts={0: {'port': DB_SETTINGS['port']}}, - host_defaults={'host': DB_SETTINGS['host']} - ) - exercise_fanout(cluster) - - client = cluster.get_routing_client() - exercise_redis(client) diff --git a/tests/datastore_redis/test_span_event.py b/tests/datastore_redis/test_span_event.py index 27103e971f..36293a6f8b 100644 --- a/tests/datastore_redis/test_span_event.py +++ b/tests/datastore_redis/test_span_event.py @@ -78,7 +78,7 @@ def test_span_events(instance_enabled, db_instance_enabled): settings = _enable_instance_settings.copy() hostname = instance_hostname(DB_SETTINGS['host']) exact_agents.update({ - 'peer.address': '%s:%s' % (hostname, DB_SETTINGS['port']), + 'peer.address': f"{hostname}:{DB_SETTINGS['port']}", 'peer.hostname': hostname, }) else: diff --git a/tests/datastore_redis/test_uninstrumented_methods.py b/tests/datastore_redis/test_uninstrumented_methods.py index ccf5a096df..a70dede7f3 100644 --- a/tests/datastore_redis/test_uninstrumented_methods.py +++ b/tests/datastore_redis/test_uninstrumented_methods.py @@ -39,6 +39,7 @@ "append_no_scale", "append_values_and_weights", "append_weights", + "auto_close_connection_pool", "batch_indexer", "BatchIndexer", "bulk", @@ -55,7 +56,9 @@ "edges", "execute_command", "flush", + "from_pool", "from_url", + "get_cache", "get_connection_kwargs", "get_encoder", "get_label", @@ -63,7 +66,6 @@ "get_property", "get_relation", "get_retry", - "hscan_iter", "index_name", "labels", "list_keys", @@ -116,4 +118,4 @@ def test_uninstrumented_methods(client): is_wrapped = lambda m: hasattr(getattr(module_client, m), "__wrapped__") uninstrumented |= {m for m in module_methods - IGNORED_METHODS if not is_wrapped(m)} - assert not uninstrumented, "Uninstrumented methods: %s" % sorted(uninstrumented) + assert not uninstrumented, f"Uninstrumented methods: {sorted(uninstrumented)}" diff --git a/tests/datastore_rediscluster/conftest.py b/tests/datastore_rediscluster/conftest.py new file mode 100644 index 0000000000..424ffc6f63 --- /dev/null +++ b/tests/datastore_rediscluster/conftest.py @@ -0,0 +1,33 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, +} + +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (datastore_redis)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_rediscluster/test_uninstrumented_rediscluster_methods.py b/tests/datastore_rediscluster/test_uninstrumented_rediscluster_methods.py new file mode 100644 index 0000000000..d4c02c690c --- /dev/null +++ b/tests/datastore_rediscluster/test_uninstrumented_rediscluster_methods.py @@ -0,0 +1,168 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import redis +from testing_support.db_settings import redis_cluster_settings + +DB_CLUSTER_SETTINGS = redis_cluster_settings()[0] + +# Set socket_timeout to 5s for fast fail, otherwise the default is to wait forever. +client = redis.RedisCluster(host=DB_CLUSTER_SETTINGS["host"], port=DB_CLUSTER_SETTINGS["port"], socket_timeout=5) + +IGNORED_METHODS = { + "MODULE_CALLBACKS", + "MODULE_VERSION", + "NAME", + "add_edge", + "add_node", + "append_bucket_size", + "append_capacity", + "append_error", + "append_expansion", + "append_items_and_increments", + "append_items", + "append_max_iterations", + "append_no_create", + "append_no_scale", + "append_values_and_weights", + "append_weights", + "batch_indexer", + "BatchIndexer", + "bulk", + "call_procedure", + "client_tracking_off", + "client_tracking_on", + "client", + "close", + "commandmixin", + "connection_pool", + "connection", + "debug_segfault", + "edges", + "execute_command", + "flush", + "from_url", + "get_connection_kwargs", + "get_encoder", + "get_label", + "get_params_args", + "get_property", + "get_relation", + "get_retry", + "hscan_iter", + "index_name", + "labels", + "list_keys", + "load_document", + "load_external_module", + "lock", + "name", + "nodes", + "parse_response", + "pipeline", + "property_keys", + "register_script", + "relationship_types", + "response_callbacks", + "RESPONSE_CALLBACKS", + "sentinel", + "set_file", + "set_path", + "set_response_callback", + "set_retry", + "transaction", + "version", + "ALL_NODES", + "CLUSTER_COMMANDS_RESPONSE_CALLBACKS", + "COMMAND_FLAGS", + "DEFAULT_NODE", + "ERRORS_ALLOW_RETRY", + "NODE_FLAGS", + "PRIMARIES", + "RANDOM", + "REPLICAS", + "RESULT_CALLBACKS", + "RedisClusterRequestTTL", + "SEARCH_COMMANDS", + "client_no_touch", + "cluster_addslotsrange", + "cluster_bumpepoch", + "cluster_delslotsrange", + "cluster_error_retry_attempts", + "cluster_flushslots", + "cluster_links", + "cluster_myid", + "cluster_myshardid", + "cluster_replicas", + "cluster_response_callbacks", + "cluster_setslot_stable", + "cluster_shards", + "command_flags", + "commands_parser", + "determine_slot", + "disconnect_connection_pools", + "encoder", + "get_default_node", + "get_node", + "get_node_from_key", + "get_nodes", + "get_primaries", + "get_random_node", + "get_redis_connection", + "get_replicas", + "keyslot", + "mget_nonatomic", + "monitor", + "mset_nonatomic", + "node_flags", + "nodes_manager", + "on_connect", + "pubsub", + "read_from_replicas", + "reinitialize_counter", + "reinitialize_steps", + "replace_default_node", + "result_callbacks", + "set_default_node", + "user_on_connect_func", +} + +REDIS_MODULES = { + "bf", + "cf", + "cms", + "ft", + "graph", + "json", + "tdigest", + "topk", + "ts", +} + +IGNORED_METHODS |= REDIS_MODULES + + +def test_uninstrumented_methods(): + methods = {m for m in dir(client) if not m[0] == "_"} + is_wrapped = lambda m: hasattr(getattr(client, m), "__wrapped__") + uninstrumented = {m for m in methods - IGNORED_METHODS if not is_wrapped(m)} + + for module in REDIS_MODULES: + if hasattr(client, module): + module_client = getattr(client, module)() + module_methods = {m for m in dir(module_client) if not m[0] == "_"} + is_wrapped = lambda m: hasattr(getattr(module_client, m), "__wrapped__") + uninstrumented |= {m for m in module_methods - IGNORED_METHODS if not is_wrapped(m)} + + assert not uninstrumented, f"Uninstrumented methods: {sorted(uninstrumented)}" diff --git a/tests/datastore_solrpy/conftest.py b/tests/datastore_solrpy/conftest.py index 4418e5d9a4..9fa03f531f 100644 --- a/tests/datastore_solrpy/conftest.py +++ b/tests/datastore_solrpy/conftest.py @@ -12,20 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_solrpy)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_solrpy)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/datastore_solrpy/test_solr.py b/tests/datastore_solrpy/test_solr.py index ee1a7e91ef..3e02f0e401 100644 --- a/tests/datastore_solrpy/test_solr.py +++ b/tests/datastore_solrpy/test_solr.py @@ -13,48 +13,58 @@ # limitations under the License. from solr import SolrConnection - -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics from testing_support.db_settings import solr_settings +from testing_support.util import instance_hostname +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.background_task import background_task DB_SETTINGS = solr_settings()[0] SOLR_HOST = DB_SETTINGS["host"] SOLR_PORT = DB_SETTINGS["port"] -SOLR_URL = 'http://%s:%s/solr/collection' % (DB_SETTINGS["host"], DB_SETTINGS["port"]) +SOLR_URL = f"http://{DB_SETTINGS['host']}:{DB_SETTINGS['port']}/solr/collection" + def _exercise_solr(solr): # Construct document names within namespace documents = ["pysolr_doc_1", "pysolr_doc_2"] - documents = [x + "_" + DB_SETTINGS["namespace"] for x in documents] + documents = [f"{x}_{DB_SETTINGS['namespace']}" for x in documents] solr.add_many([{"id": x} for x in documents]) solr.commit() - solr.query('id:%s' % documents[0]).results - solr.delete('id:*_%s' % DB_SETTINGS["namespace"]) + solr.query(f"id:{documents[0]}").results + solr.delete(f"id:*_{DB_SETTINGS['namespace']}") solr.commit() + _test_solr_search_scoped_metrics = [ - ('Datastore/operation/Solr/add_many', 1), - ('Datastore/operation/Solr/delete', 1), - ('Datastore/operation/Solr/commit', 2), - ('Datastore/operation/Solr/query', 1)] + ("Datastore/operation/Solr/add_many", 1), + ("Datastore/operation/Solr/delete", 1), + ("Datastore/operation/Solr/commit", 2), + ("Datastore/operation/Solr/query", 1), +] _test_solr_search_rollup_metrics = [ - ('Datastore/all', 5), - ('Datastore/allOther', 5), - ('Datastore/Solr/all', 5), - ('Datastore/Solr/allOther', 5), - ('Datastore/operation/Solr/add_many', 1), - ('Datastore/operation/Solr/query', 1), - ('Datastore/operation/Solr/commit', 2), - ('Datastore/operation/Solr/delete', 1)] - -@validate_transaction_metrics('test_solr:test_solr_search', + ("Datastore/all", 5), + ("Datastore/allOther", 5), + ("Datastore/Solr/all", 5), + ("Datastore/Solr/allOther", 5), + (f"Datastore/instance/Solr/{instance_hostname(SOLR_HOST)}/{SOLR_PORT}", 3), + ("Datastore/operation/Solr/add_many", 1), + ("Datastore/operation/Solr/query", 1), + ("Datastore/operation/Solr/commit", 2), + ("Datastore/operation/Solr/delete", 1), +] + + +@validate_transaction_metrics( + "test_solr:test_solr_search", scoped_metrics=_test_solr_search_scoped_metrics, rollup_metrics=_test_solr_search_rollup_metrics, - background_task=True) + background_task=True, +) @background_task() def test_solr_search(): s = SolrConnection(SOLR_URL) diff --git a/tests/datastore_sqlite/conftest.py b/tests/datastore_sqlite/conftest.py index ed695b251f..499dd7b9f8 100644 --- a/tests/datastore_sqlite/conftest.py +++ b/tests/datastore_sqlite/conftest.py @@ -12,21 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, - 'debug.log_explain_plan_queries': True + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_explain_plan_queries": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (datastore_sqlite)', - default_settings=_default_settings, - linked_applications=['Python Agent Test (datastore)']) + app_name="Python Agent Test (datastore_sqlite)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (datastore)"], +) diff --git a/tests/external_aiobotocore/conftest.py b/tests/external_aiobotocore/conftest.py new file mode 100644 index 0000000000..2c54daf31f --- /dev/null +++ b/tests/external_aiobotocore/conftest.py @@ -0,0 +1,152 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import logging +import socket +import threading + +import moto.server +import werkzeug.serving +from testing_support.fixture.event_loop import ( # noqa: F401, pylint: disable=W0611 + event_loop as loop, +) +from testing_support.fixtures import ( # noqa: F401, pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) + +PORT = 4443 +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec +HOST = "127.0.0.1" + + +_default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, +} +collector_agent_registration = collector_agent_registration_fixture( + app_name="Python Agent Test (external_aiobotocore)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (external_aiobotocore)"], +) + + +def get_free_tcp_port(release_socket: bool = False): + sckt = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sckt.bind((HOST, 0)) + _, port = sckt.getsockname() # address, port + if release_socket: + sckt.close() + return port + + return sckt, port + + +class MotoService: + """Will Create MotoService. + Service is ref-counted so there will only be one per process. Real Service will + be returned by `__aenter__`.""" + + _services = {} # {name: instance} + + def __init__(self, service_name: str, port: int = None, ssl: bool = False): + self._service_name = service_name + + if port: + self._socket = None + self._port = port + else: + self._socket, self._port = get_free_tcp_port() + + self._thread = None + self._logger = logging.getLogger("MotoService") + self._refcount = None + self._ip_address = HOST + self._server = None + self._ssl_ctx = werkzeug.serving.generate_adhoc_ssl_context() if ssl else None + self._schema = "http" if not self._ssl_ctx else "https" + + @property + def endpoint_url(self): + return f"{self._schema}://{self._ip_address}:{self._port}" + + def __call__(self, func): + async def wrapper(*args, **kwargs): + await self._start() + try: + result = await func(*args, **kwargs) + finally: + await self._stop() + return result + + functools.update_wrapper(wrapper, func) + wrapper.__wrapped__ = func + return wrapper + + async def __aenter__(self): + svc = self._services.get(self._service_name) + if svc is None: + self._services[self._service_name] = self + self._refcount = 1 + await self._start() + return self + else: + svc._refcount += 1 + return svc + + async def __aexit__(self, exc_type, exc_val, exc_tb): + self._refcount -= 1 + + if self._socket: + self._socket.close() + self._socket = None + + if self._refcount == 0: + del self._services[self._service_name] + await self._stop() + + def _server_entry(self): + self._main_app = moto.server.DomainDispatcherApplication( + moto.server.create_backend_app # , service=self._service_name + ) + self._main_app.debug = True + + if self._socket: + self._socket.close() # release right before we use it + self._socket = None + + self._server = werkzeug.serving.make_server( + self._ip_address, + self._port, + self._main_app, + True, + ssl_context=self._ssl_ctx, + ) + self._server.serve_forever() + + async def _start(self): + self._thread = threading.Thread(target=self._server_entry, daemon=True) + self._thread.start() + + async def _stop(self): + if self._server: + self._server.shutdown() + + self._thread.join() diff --git a/tests/external_aiobotocore/test_aiobotocore_dynamodb.py b/tests/external_aiobotocore/test_aiobotocore_dynamodb.py new file mode 100644 index 0000000000..e4b59513e7 --- /dev/null +++ b/tests/external_aiobotocore/test_aiobotocore_dynamodb.py @@ -0,0 +1,167 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from aiobotocore.session import get_session +from conftest import ( # noqa: F401, pylint: disable=W0611 + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + PORT, + MotoService, + loop, +) +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + +TEST_TABLE = "python-agent-test" + +_dynamodb_scoped_metrics = [ + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/create_table", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/put_item", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/get_item", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/update_item", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/query", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/scan", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/delete_item", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/delete_table", 1), +] + +_dynamodb_rollup_metrics = [ + ("Datastore/all", 8), + ("Datastore/allOther", 8), + ("Datastore/DynamoDB/all", 8), + ("Datastore/DynamoDB/allOther", 8), +] + + +# aws.requestId count disabled due to variability in count. +# Flaky due to waiter function, which "aws.operation" == "DescribeTable" +# This is a polling function, so in real time, this value could fluctuate +# @validate_span_events(expected_agents=("aws.requestId",), count=9) +# @validate_span_events(exact_agents={"aws.operation": "DescribeTable"}, count=2) +@validate_span_events(exact_agents={"aws.operation": "PutItem"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "GetItem"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteItem"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "CreateTable"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteTable"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Query"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Scan"}, count=1) +@validate_transaction_metrics( + "test_aiobotocore_dynamodb:test_aiobotocore_dynamodb", + scoped_metrics=_dynamodb_scoped_metrics, + rollup_metrics=_dynamodb_rollup_metrics, + background_task=True, +) +@background_task() +def test_aiobotocore_dynamodb(loop): + async def _test(): + async with MotoService("dynamodb", port=PORT): + session = get_session() + + async with session.create_client( + "dynamodb", + region_name="us-east-1", + endpoint_url=f"http://localhost:{PORT}", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + ) as client: + + resp = await client.create_table( + TableName=TEST_TABLE, + AttributeDefinitions=[ + {"AttributeName": "Id", "AttributeType": "N"}, + {"AttributeName": "Foo", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "Id", "KeyType": "HASH"}, + {"AttributeName": "Foo", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={ + "ReadCapacityUnits": 5, + "WriteCapacityUnits": 5, + }, + ) + assert resp["TableDescription"]["TableName"] == TEST_TABLE + + # Wait for table to be created + waiter = client.get_waiter("table_exists") + await waiter.wait(TableName=TEST_TABLE) + + # Put item + resp = await client.put_item( + TableName=TEST_TABLE, + Item={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + }, + ) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Get item + resp = await client.get_item( + TableName=TEST_TABLE, + Key={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + }, + ) + assert resp["Item"]["Foo"]["S"] == "hello_world" + + # Update item + resp = await client.update_item( + TableName=TEST_TABLE, + Key={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + }, + AttributeUpdates={ + "Foo2": {"Value": {"S": "hello_world2"}, "Action": "PUT"}, + }, + ReturnValues="ALL_NEW", + ) + assert resp["Attributes"]["Foo2"] + + # Query for item + resp = await client.query( + TableName=TEST_TABLE, + Select="ALL_ATTRIBUTES", + KeyConditionExpression="#Id = :v_id", + ExpressionAttributeNames={"#Id": "Id"}, + ExpressionAttributeValues={":v_id": {"N": "101"}}, + ) + assert len(resp["Items"]) == 1 + assert resp["Items"][0]["Foo"]["S"] == "hello_world" + + # Scan + resp = await client.scan(TableName=TEST_TABLE) + assert len(resp["Items"]) == 1 + + # Delete item + resp = await client.delete_item( + TableName=TEST_TABLE, + Key={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + }, + ) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Delete table + resp = await client.delete_table(TableName=TEST_TABLE) + assert resp["TableDescription"]["TableName"] == TEST_TABLE + + loop.run_until_complete(_test()) diff --git a/tests/external_aiobotocore/test_aiobotocore_s3.py b/tests/external_aiobotocore/test_aiobotocore_s3.py new file mode 100644 index 0000000000..8c8e7eeab8 --- /dev/null +++ b/tests/external_aiobotocore/test_aiobotocore_s3.py @@ -0,0 +1,123 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import aiobotocore +from conftest import ( # noqa: F401, pylint: disable=W0611 + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + PORT, + MotoService, + loop, +) +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + +TEST_BUCKET = "python-agent-test" +FILENAME = "dummy.bin" +FOLDER = "aiobotocore" +ENDPOINT = f"localhost:{PORT}" +KEY = f"{FOLDER}/{FILENAME}" +EXPECTED_BUCKET_URL = f"http://{ENDPOINT}/{TEST_BUCKET}" +EXPECTED_KEY_URL = f"{EXPECTED_BUCKET_URL}/{KEY}" + + +_s3_scoped_metrics = [ + (f"External/{ENDPOINT}/aiobotocore/GET", 5), + (f"External/{ENDPOINT}/aiobotocore/PUT", 2), + (f"External/{ENDPOINT}/aiobotocore/DELETE", 2), +] + +_s3_rollup_metrics = [ + ("External/all", 9), + ("External/allOther", 9), + (f"External/{ENDPOINT}/all", 9), + (f"External/{ENDPOINT}/aiobotocore/GET", 5), + (f"External/{ENDPOINT}/aiobotocore/PUT", 2), + (f"External/{ENDPOINT}/aiobotocore/DELETE", 2), +] + + +@validate_span_events(exact_agents={"aws.operation": "CreateBucket"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "PutObject"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "ListObjects"}, count=2) +@validate_span_events(exact_agents={"aws.operation": "GetObject"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteBucket"}, count=1) +@validate_span_events(exact_agents={"http.url": EXPECTED_BUCKET_URL}, count=4) +@validate_span_events(exact_agents={"http.url": EXPECTED_KEY_URL}, count=4) +@validate_transaction_metrics( + "test_aiobotocore_s3:test_aiobotocore_s3", + scoped_metrics=_s3_scoped_metrics, + rollup_metrics=_s3_rollup_metrics, + background_task=True, +) +@background_task() +def test_aiobotocore_s3(loop): + async def _test(): + + data = b"hello_world" + + async with MotoService("s3", port=PORT): + + session = aiobotocore.session.get_session() + + async with session.create_client( # nosec + "s3", + region_name="us-east-1", + endpoint_url=f"http://localhost:{PORT}", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + ) as client: + + # Create bucket + await client.create_bucket( + Bucket=TEST_BUCKET, + ) + + # List buckets + await client.list_buckets() + + # Upload object to s3 + resp = await client.put_object(Bucket=TEST_BUCKET, Key=KEY, Body=data) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # List objects from bucket + await client.list_objects(Bucket=TEST_BUCKET) + + # Getting s3 object properties of uploaded file + resp = await client.get_object_acl(Bucket=TEST_BUCKET, Key=KEY) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Get object from s3 + response = await client.get_object(Bucket=TEST_BUCKET, Key=KEY) + # this will ensure the connection is correctly re-used/closed + async with response["Body"] as stream: + assert await stream.read() == data + + # List s3 objects using paginator + paginator = client.get_paginator("list_objects") + async for result in paginator.paginate(Bucket=TEST_BUCKET, Prefix=FOLDER): + for content in result.get("Contents", []): + assert content + + # Delete object from s3 + await client.delete_object(Bucket=TEST_BUCKET, Key=KEY) + + # Delete bucket from s3 + await client.delete_bucket(Bucket=TEST_BUCKET) + + loop.run_until_complete(_test()) diff --git a/tests/external_aiobotocore/test_aiobotocore_sns.py b/tests/external_aiobotocore/test_aiobotocore_sns.py new file mode 100644 index 0000000000..31e0db92f7 --- /dev/null +++ b/tests/external_aiobotocore/test_aiobotocore_sns.py @@ -0,0 +1,73 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from aiobotocore.session import get_session +from conftest import ( # noqa: F401, pylint: disable=W0611 + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + PORT, + MotoService, + loop, +) +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + +TOPIC = "arn:aws:sns:us-east-1:123456789012:some-topic" +sns_metrics = [ + (f"MessageBroker/SNS/Topic/Produce/Named/{TOPIC}", 1), + ("MessageBroker/SNS/Topic/Produce/Named/PhoneNumber", 1), +] + + +@validate_span_events(expected_agents=("aws.requestId",), count=4) +@validate_span_events(exact_agents={"aws.operation": "CreateTopic"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Publish"}, count=2) +@validate_transaction_metrics( + "test_aiobotocore_sns:test_publish_to_sns", + scoped_metrics=sns_metrics, + rollup_metrics=sns_metrics, + background_task=True, +) +@background_task() +def test_publish_to_sns(loop): + async def _test(): + + async with MotoService("sns", port=PORT): + session = get_session() + + async with session.create_client( + "sns", + region_name="us-east-1", + endpoint_url=f"http://localhost:{PORT}", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + ) as client: + + topic_arn = await client.create_topic(Name="some-topic") + topic_arn_name = topic_arn["TopicArn"] + + kwargs = {"TopicArn": topic_arn_name} + published_message = await client.publish(Message="my message", **kwargs) + assert "MessageId" in published_message + + await client.subscribe(TopicArn=topic_arn_name, Protocol="sms", Endpoint="5555555555") + + published_message = await client.publish(PhoneNumber="5555555555", Message="my msg") + assert "MessageId" in published_message + + loop.run_until_complete(_test()) diff --git a/tests/external_aiobotocore/test_aiobotocore_sqs.py b/tests/external_aiobotocore/test_aiobotocore_sqs.py new file mode 100644 index 0000000000..bb76a0d12e --- /dev/null +++ b/tests/external_aiobotocore/test_aiobotocore_sqs.py @@ -0,0 +1,114 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from aiobotocore.session import get_session +from conftest import ( # noqa: F401, pylint: disable=W0611 + AWS_ACCESS_KEY_ID, + AWS_SECRET_ACCESS_KEY, + PORT, + MotoService, + loop, +) +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task + +URL = f"localhost:{PORT}" +TEST_QUEUE = "python-agent-test" + +_sqs_scoped_metrics = [ + (f"MessageBroker/SQS/Queue/Produce/Named/{TEST_QUEUE}", 2), + (f"External/{URL}/aiobotocore/POST", 7), +] + +_sqs_rollup_metrics = [ + (f"MessageBroker/SQS/Queue/Produce/Named/{TEST_QUEUE}", 2), + (f"MessageBroker/SQS/Queue/Consume/Named/{TEST_QUEUE}", 1), + ("External/all", 7), + ("External/allOther", 7), + (f"External/{URL}/all", 7), + (f"External/{URL}/aiobotocore/POST", 7), +] + + +@validate_span_events(exact_agents={"aws.operation": "CreateQueue"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "ListQueues"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "SendMessage"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "ReceiveMessage"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "SendMessageBatch"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "PurgeQueue"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteQueue"}, count=1) +@validate_transaction_metrics( + "test_aiobotocore_sqs:test_aiobotocore_sqs", + scoped_metrics=_sqs_scoped_metrics, + rollup_metrics=_sqs_rollup_metrics, + background_task=True, +) +@background_task() +def test_aiobotocore_sqs(loop): + async def _test(): + async with MotoService("sqs", port=PORT): + session = get_session() + + async with session.create_client( + "sqs", + region_name="us-east-1", + endpoint_url=f"http://localhost:{PORT}", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + ) as client: + + response = await client.create_queue(QueueName=TEST_QUEUE) + + queue_url = response["QueueUrl"] + + # List queues + response = await client.list_queues() + for queue_name in response.get("QueueUrls", []): + assert queue_name + + # Send message + resp = await client.send_message( + QueueUrl=queue_url, + MessageBody="hello_world", + ) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Receive message + resp = await client.receive_message( + QueueUrl=queue_url, + ) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Send message batch + messages = [ + {"Id": "1", "MessageBody": "message 1"}, + {"Id": "2", "MessageBody": "message 2"}, + {"Id": "3", "MessageBody": "message 3"}, + ] + resp = await client.send_message_batch(QueueUrl=queue_url, Entries=messages) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Purge queue + resp = await client.purge_queue(QueueUrl=queue_url) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Delete queue + resp = await client.delete_queue(QueueUrl=queue_url) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + loop.run_until_complete(_test()) diff --git a/tests/external_boto3/test_boto3_iam.py b/tests/external_boto3/test_boto3_iam.py deleted file mode 100644 index ac49214f44..0000000000 --- a/tests/external_boto3/test_boto3_iam.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import uuid - -import boto3 -import moto - -from newrelic.api.background_task import background_task -from testing_support.fixtures import ( - validate_tt_segment_params, override_application_settings) -from testing_support.validators.validate_span_events import ( - validate_span_events) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics - -MOTO_VERSION = tuple(int(v) for v in moto.__version__.split('.')[:3]) - -# patch earlier versions of moto to support py37 -if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1): - import re - moto.packages.responses.responses.re._pattern_type = re.Pattern - -AWS_ACCESS_KEY_ID = 'AAAAAAAAAAAACCESSKEY' -AWS_SECRET_ACCESS_KEY = 'AAAAAASECRETKEY' - -TEST_USER = 'python-agent-test-%s' % uuid.uuid4() - -_iam_scoped_metrics = [ - ('External/iam.amazonaws.com/botocore/POST', 3), -] - -_iam_rollup_metrics = [ - ('External/all', 3), - ('External/allOther', 3), - ('External/iam.amazonaws.com/all', 3), - ('External/iam.amazonaws.com/botocore/POST', 3), -] - - -@override_application_settings({'distributed_tracing.enabled': True}) -@validate_span_events( - exact_agents={'http.url': 'https://iam.amazonaws.com/'}, count=3) -@validate_span_events(expected_agents=('aws.requestId',), count=3) -@validate_span_events(exact_agents={'aws.operation': 'CreateUser'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'GetUser'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'DeleteUser'}, count=1) -@validate_tt_segment_params(present_params=('aws.requestId',)) -@validate_transaction_metrics( - 'test_boto3_iam:test_iam', - scoped_metrics=_iam_scoped_metrics, - rollup_metrics=_iam_rollup_metrics, - background_task=True) -@background_task() -@moto.mock_iam -def test_iam(): - iam = boto3.client( - 'iam', - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - ) - - # Create user - resp = iam.create_user(UserName=TEST_USER) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 - - # Get the user - resp = iam.get_user(UserName=TEST_USER) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 - assert resp['User']['UserName'] == TEST_USER - - # Delete the user - resp = iam.delete_user(UserName=TEST_USER) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 diff --git a/tests/external_boto3/test_boto3_sns.py b/tests/external_boto3/test_boto3_sns.py deleted file mode 100644 index 3718d52924..0000000000 --- a/tests/external_boto3/test_boto3_sns.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import sys -import boto3 -import moto -import pytest - -from newrelic.api.background_task import background_task -from testing_support.fixtures import ( - validate_tt_segment_params, override_application_settings) -from testing_support.validators.validate_span_events import validate_span_events -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics - -MOTO_VERSION = tuple(int(v) for v in moto.__version__.split('.')[:3]) - -# patch earlier versions of moto to support py37 -if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1): - import re - moto.packages.responses.responses.re._pattern_type = re.Pattern - -AWS_ACCESS_KEY_ID = 'AAAAAAAAAAAACCESSKEY' -AWS_SECRET_ACCESS_KEY = 'AAAAAASECRETKEY' -AWS_REGION_NAME = 'us-east-1' -SNS_URL = 'sns-us-east-1.amazonaws.com' -TOPIC = 'arn:aws:sns:us-east-1:123456789012:some-topic' -sns_metrics = [ - ('MessageBroker/SNS/Topic' - '/Produce/Named/%s' % TOPIC, 1)] -sns_metrics_phone = [ - ('MessageBroker/SNS/Topic' - '/Produce/Named/PhoneNumber', 1)] - - -@override_application_settings({'distributed_tracing.enabled': True}) -@validate_span_events(expected_agents=('aws.requestId',), count=2) -@validate_span_events(exact_agents={'aws.operation': 'CreateTopic'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'Publish'}, count=1) -@validate_tt_segment_params(present_params=('aws.requestId',)) -@pytest.mark.parametrize('topic_argument', ('TopicArn', 'TargetArn')) -@validate_transaction_metrics('test_boto3_sns:test_publish_to_sns_topic', - scoped_metrics=sns_metrics, rollup_metrics=sns_metrics, - background_task=True) -@background_task() -@moto.mock_sns -def test_publish_to_sns_topic(topic_argument): - conn = boto3.client('sns', - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - region_name=AWS_REGION_NAME) - - topic_arn = conn.create_topic(Name='some-topic')['TopicArn'] - - kwargs = {topic_argument: topic_arn} - published_message = conn.publish(Message='my msg', **kwargs) - assert 'MessageId' in published_message - - -@override_application_settings({'distributed_tracing.enabled': True}) -@validate_span_events(expected_agents=('aws.requestId',), count=3) -@validate_span_events(exact_agents={'aws.operation': 'CreateTopic'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'Subscribe'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'Publish'}, count=1) -@validate_tt_segment_params(present_params=('aws.requestId',)) -@validate_transaction_metrics('test_boto3_sns:test_publish_to_sns_phone', - scoped_metrics=sns_metrics_phone, rollup_metrics=sns_metrics_phone, - background_task=True) -@background_task() -@moto.mock_sns -def test_publish_to_sns_phone(): - conn = boto3.client('sns', - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - region_name=AWS_REGION_NAME) - - topic_arn = conn.create_topic(Name='some-topic')['TopicArn'] - conn.subscribe(TopicArn=topic_arn, Protocol='sms', Endpoint='5555555555') - - published_message = conn.publish( - PhoneNumber='5555555555', Message='my msg') - assert 'MessageId' in published_message diff --git a/tests/external_botocore/_mock_bedrock_encoding_utils.py b/tests/external_botocore/_mock_bedrock_encoding_utils.py new file mode 100644 index 0000000000..6144ebb6a8 --- /dev/null +++ b/tests/external_botocore/_mock_bedrock_encoding_utils.py @@ -0,0 +1,84 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Utilities for encoding and decoding streaming payloads from Bedrock. """ +import base64 +import binascii +import json + + +def crc(b): + """Encode the crc32 of the bytes stream into a 4 byte sequence.""" + return int_to_escaped_bytes(binascii.crc32(b), 4) + + +def int_to_escaped_bytes(i, num_bytes=1): + """Convert an integer into an arbitrary number of bytes.""" + return bytes.fromhex(f"{{:0{str(num_bytes * 2)}x}}".format(i)) + + +def encode_headers(headers): + """Encode a dictionary of headers into bedrock's binary format.""" + new_headers = [] + for h, v in headers.items(): + if not h.startswith(":"): + h = f":{h}" + h = h.encode("utf-8") + v = v.encode("utf-8") + new_headers.append(b"".join((int_to_escaped_bytes(len(h)), h, b"\x07\x00", int_to_escaped_bytes(len(v)), v))) + return b"".join(new_headers) + + +def decode_body(body): + """Decode the mixed JSON and base64 encoded body of a streaming response into a dictionary.""" + body = body.decode("utf-8") + body = json.loads(body) + body = body["bytes"] + body = base64.b64decode(body) + body = body.decode("utf-8") + return json.loads(body) + + +def encode_body(body, malformed_body=False): + """Encode a dictionary body into JSON, base64, then JSON again under a bytes key.""" + + body = json.dumps(body, separators=(",", ":")) + if malformed_body: + # Remove characters from end of body to make it unreadable + body = body[:-4] + + body = body.encode("utf-8") + body = base64.b64encode(body) + body = body.decode("utf-8") + body = {"bytes": body} + body = json.dumps(body, separators=(",", ":")) + body = body.encode("utf-8") + return body + + +def encode_streaming_payload(headers, body, malformed_body=False): + """Encode dictionary headers and dictionary body into bedrock's binary payload format including calculated lengths and CRC32.""" + headers = encode_headers(headers) + body = encode_body(body, malformed_body=malformed_body) + + header_length = len(headers) + payload_length = len(body) + total_length = 16 + payload_length + header_length + + prelude = int_to_escaped_bytes(total_length, 4) + int_to_escaped_bytes(header_length, 4) + prelude_crc = crc(prelude) + + payload = prelude + prelude_crc + headers + body + payload_crc = crc(payload) + + return payload + payload_crc diff --git a/tests/external_botocore/_mock_external_bedrock_server.py b/tests/external_botocore/_mock_external_bedrock_server.py new file mode 100644 index 0000000000..5974cdb406 --- /dev/null +++ b/tests/external_botocore/_mock_external_bedrock_server.py @@ -0,0 +1,6756 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import re + +from _mock_bedrock_encoding_utils import encode_streaming_payload +from testing_support.mock_external_http_server import MockExternalHTTPServer + +# This defines an external server test apps can make requests to instead of +# the real Bedrock backend. This provides 3 features: +# +# 1) This removes dependencies on external websites. +# 2) Provides a better mechanism for making an external call in a test app than +# simple calling another endpoint the test app makes available because this +# server will not be instrumented meaning we don't have to sort through +# transactions to separate the ones created in the test app and the ones +# created by an external call. +# 3) This app runs on a separate thread meaning it won't block the test app. + +STREAMED_RESPONSES = { + "mistral.mistral-7b-instruct-v0%3A2::[INST] What is 212 degrees Fahrenheit converted to Celsius? [/INST]": [ + { + "Content-Type": "application/vnd.amazon.eventstream", + "x-amzn-RequestId": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + }, + 200, + [ + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494652764969776963335276634639795a57467a623234694f6d3531624778395858303d227d30693c02", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949474e76626e5a6c636e51694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227dd3bacdba", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494745694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d7e1af014", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694948526c6258426c636d463064584a6c4969776963335276634639795a57467a623234694f6d3531624778395858303d227d1ac96f27", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949475a79623230694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d79a7bac3", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494559694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d0d3a538e", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69595768795a5734694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d800cf6a1", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f696147567064434973496e4e3062334266636d566863323975496a70756457787366563139227d9261e38d", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494852764969776963335276634639795a57467a623234694f6d3531624778395858303d227d580defbc", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949454d694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227dd42b6945", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f695a57787a4969776963335276634639795a57467a623234694f6d3531624778395858303d227df0e3aab1", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f696158567a4969776963335276634639795a57467a623234694f6d3531624778395858303d227d47cb033c", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694c434973496e4e3062334266636d566863323975496a70756457787366563139227db86ac617", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949486c7664534973496e4e3062334266636d566863323975496a70756457787366563139227d111efa00", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949474e6862694973496e4e3062334266636d566863323975496a70756457787366563139227d33a08812", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694948567a5a534973496e4e3062334266636d566863323975496a70756457787366563139227d60f7f5e7", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694948526f5a534973496e4e3062334266636d566863323975496a70756457787366563139227d5059ad51", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949475a766247787664326c755a794973496e4e3062334266636d566863323975496a70756457787366563139227da7af16c9", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949475a76636d3131624745694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d76a80396", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694f694973496e4e3062334266636d566863323975496a70756457787366563139227dbc0b9457", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949454d694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227dd42b6945", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f695a57787a4969776963335276634639795a57467a623234694f6d3531624778395858303d227df0e3aab1", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f696158567a4969776963335276634639795a57467a623234694f6d3531624778395858303d227d47cb033c", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494430694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d420e6621", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494367694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227df82acb39", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6952694973496e4e3062334266636d566863323975496a70756457787366563139227dd05517eb", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69595768795a5734694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d800cf6a1", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f696147567064434973496e4e3062334266636d566863323975496a70756457787366563139227d9261e38d", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494330694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d763c0ffe", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949434973496e4e3062334266636d566863323975496a70756457787366563139227d0ee1d3f9", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d794973496e4e3062334266636d566863323975496a70756457787366563139227d2cc43589", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d694973496e4e3062334266636d566863323975496a70756457787366563139227df50f363b", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694b534973496e4e3062334266636d566863323975496a70756457787366563139227d9e2e7227", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494d4f584969776963335276634639795a57467a623234694f6d3531624778395858303d227d1309a28c", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949434973496e4e3062334266636d566863323975496a70756457787366563139227d0ee1d3f9", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694e534973496e4e3062334266636d566863323975496a70756457787366563139227d28a567c9", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694c794973496e4e3062334266636d566863323975496a70756457787366563139227d084664bf", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694f534973496e4e3062334266636d566863323975496a70756457787366563139227d0c2736ff", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694c694973496e4e3062334266636d566863323975496a70756457787366563139227dd18d670d", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69584734694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d18050f59", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69584734694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d18050f59", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69553238694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d2acbcc0d", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694c434973496e4e3062334266636d566863323975496a70756457787366563139227db86ac617", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494852764969776963335276634639795a57467a623234694f6d3531624778395858303d227d580defbc", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949474e76626e5a6c636e51694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227dd3bacdba", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949434973496e4e3062334266636d566863323975496a70756457787366563139227d0ee1d3f9", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d694973496e4e3062334266636d566863323975496a70756457787366563139227df50f363b", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d534973496e4e3062334266636d566863323975496a70756457787366563139227d45239493", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d694973496e4e3062334266636d566863323975496a70756457787366563139227df50f363b", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694947526c5a334a6c5a584d694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d3ac15b52", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494559694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d0d3a538e", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69595768795a5734694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d800cf6a1", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f696147567064434973496e4e3062334266636d566863323975496a70756457787366563139227d9261e38d", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494852764969776963335276634639795a57467a623234694f6d3531624778395858303d227d580defbc", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949454d694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227dd42b6945", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f695a57787a4969776963335276634639795a57467a623234694f6d3531624778395858303d227df0e3aab1", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f696158567a4969776963335276634639795a57467a623234694f6d3531624778395858303d227d47cb033c", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694c434973496e4e3062334266636d566863323975496a70756457787366563139227db86ac617", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494752764969776963335276634639795a57467a623234694f6d3531624778395858303d227db0992b1d", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694948526f5a534973496e4e3062334266636d566863323975496a70756457787366563139227d5059ad51", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949475a766247787664326c755a794973496e4e3062334266636d566863323975496a70756457787366563139227da7af16c9", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949474e6862474e3162474630615739754969776963335276634639795a57467a623234694f6d3531624778395858303d227dfaa5d044", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694f694973496e4e3062334266636d566863323975496a70756457787366563139227dbc0b9457", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69584734694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d18050f59", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69584734694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d18050f59", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6951794973496e4e3062334266636d566863323975496a70756457787366563139227d6418e703", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f695a57787a4969776963335276634639795a57467a623234694f6d3531624778395858303d227df0e3aab1", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f696158567a4969776963335276634639795a57467a623234694f6d3531624778395858303d227d47cb033c", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494430694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d420e6621", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494367694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227df82acb39", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d694973496e4e3062334266636d566863323975496a70756457787366563139227df50f363b", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d534973496e4e3062334266636d566863323975496a70756457787366563139227d45239493", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d694973496e4e3062334266636d566863323975496a70756457787366563139227df50f363b", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69777241694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d27bae54a", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6952694973496e4e3062334266636d566863323975496a70756457787366563139227dd05517eb", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494330694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d763c0ffe", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949434973496e4e3062334266636d566863323975496a70756457787366563139227d0ee1d3f9", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d794973496e4e3062334266636d566863323975496a70756457787366563139227d2cc43589", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d694973496e4e3062334266636d566863323975496a70756457787366563139227df50f363b", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694b534973496e4e3062334266636d566863323975496a70756457787366563139227d9e2e7227", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494d4f584969776963335276634639795a57467a623234694f6d3531624778395858303d227d1309a28c", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949434973496e4e3062334266636d566863323975496a70756457787366563139227d0ee1d3f9", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694e534973496e4e3062334266636d566863323975496a70756457787366563139227d28a567c9", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694c794973496e4e3062334266636d566863323975496a70756457787366563139227d084664bf", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694f534973496e4e3062334266636d566863323975496a70756457787366563139227d0c2736ff", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69584734694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d18050f59", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69584734694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d18050f59", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6951794973496e4e3062334266636d566863323975496a70756457787366563139227d6418e703", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f695a57787a4969776963335276634639795a57467a623234694f6d3531624778395858303d227df0e3aab1", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f696158567a4969776963335276634639795a57467a623234694f6d3531624778395858303d227d47cb033c", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494430694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d420e6621", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494367694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227df82acb39", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d694973496e4e3062334266636d566863323975496a70756457787366563139227df50f363b", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d534973496e4e3062334266636d566863323975496a70756457787366563139227d45239493", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d694973496e4e3062334266636d566863323975496a70756457787366563139227df50f363b", + "000000a70000004b462b04f70b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f69494330694c434a7a6447397758334a6c59584e7662694936626e56736248316466513d3d227d763c0ffe", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f6949434973496e4e3062334266636d566863323975496a70756457787366563139227d0ee1d3f9", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d794973496e4e3062334266636d566863323975496a70756457787366563139227d2cc43589", + "000000a30000004bb3aba2370b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694d694973496e4e3062334266636d566863323975496a70756457787366563139227df50f363b", + "000001570000004bc543870f0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a76645852776458527a496a706265794a305a586830496a6f694b534973496e4e3062334266636d566863323975496a6f69624756755a33526f496e31644c434a686257463662323474596d566b636d396a61793170626e5a76593246306157397554575630636d6c6a6379493665794a70626e4231644652766132567551323931626e51694f6a49314c434a7664585277645852556232746c626b4e7664573530496a6f784d444173496d6c75646d396a595852706232354d5958526c626d4e35496a6f784e4451774c434a6d61584a7a64454a356447564d5958526c626d4e35496a6f7a4d6a643966513d3d227d9b82aeca", + ], + ], + "amazon.titan-text-express-v1::User: The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.": [ + { + "Content-Type": "application/vnd.amazon.eventstream", + "x-amzn-RequestId": "884db5c9-18ab-4f27-8892-33656176a2e6", + }, + 200, + [ + "000001d30000004b8125915d0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a7664585277645852555a586830496a6f694945686c624778764c43426f623363675932467549456b676147567363434235623355676447396b59586b2f496977696157356b5a5867694f6a4173496e527664474673543356306348563056475634644652766132567551323931626e51694f6a45784c434a6a623231776247563061573975556d566863323975496a6f69526b6c4f53564e494969776961573577645852555a586830564739725a57354462335675644349364e7a4973496d467459587076626931695a57527962324e724c576c75646d396a595852706232354e5a58527961574e7a496a7037496d6c7563485630564739725a57354462335675644349364e7a4973496d393164484231644652766132567551323931626e51694f6a45784c434a70626e5a765932463061573975544746305a57356a655349364e7a59774c434a6d61584a7a64454a356447564d5958526c626d4e35496a6f334e6a423966513d3d227db357f684" + ], + ], + "anthropic.claude-instant-v1::The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.": [ + { + "Content-Type": "application/vnd.amazon.eventstream", + "x-amzn-RequestId": "1a72a1f6-310f-469c-af1d-2c59eb600089", + }, + 200, + [ + "000001a70000004b8d77d7520b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f696257567a6332466e5a56397a6447467964434973496d316c63334e685a3255694f6e7369615751694f694a6a62323177624638774d56684b4e315a4c4d3052574e304931556e4a6c516e7077595649795a4845694c434a306558426c496a6f696257567a6332466e5a534973496e4a76624755694f694a6863334e7063335268626e51694c434a6a623235305a573530496a7062585377696257396b5a5777694f694a6a624746315a4755746157357a64474675644330784c6a49694c434a7a6447397758334a6c59584e7662694936626e567362437769633352766346397a5a5846315a57356a5a534936626e56736243776964584e685a3255694f6e73696157357764585266644739725a57357a496a6f334d79776962335630634856305833527661325675637949364d58313966513d3d227dd65d4fce", + "000000d70000004bbff9e4380b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131397a6447467964434973496d6c755a475634496a6f774c434a6a623235305a57353058324a7362324e72496a7037496e5235634755694f694a305a58683049697769644756346443493649694a3966513d3d227dcdbf661d", + "000000db0000004b7a0909390b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f694a495a57787362794a3966513d3d227d335563af", + "000000d30000004b4a7942f80b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f694968496e3139227d2d9e8eb6", + "000000d70000004bbff9e4380b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f69496754586b696658303d227d0d0604c6", + "000000db0000004b7a0909390b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f694967626d46745a534a3966513d3d227d625a1a55", + "000000d70000004bbff9e4380b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f69496761584d696658303d227dbcc5266b", + "000000db0000004b7a0909390b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f694967513278686457526c496e3139227dafd2aab1", + "000000d30000004b4a7942f80b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f694975496e3139227d2b9773b8", + "000000db0000004b7a0909390b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f694a63626c787553446f696658303d227d796313c0", + "000000db0000004b7a0909390b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f694967546d6c6a5a534a3966513d3d227d3fc8c903", + "000000d70000004bbff9e4380b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f694967644738696658303d227dcd25ff7b", + "000000db0000004b7a0909390b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f6949676257566c64434a3966513d3d227d9daba9d4", + "000000d70000004bbff9e4380b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f69496765573931496e3139227d35994406", + "000000db0000004b7a0909390b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f694967513278686457526c496e3139227dafd2aab1", + "000000d30000004b4a7942f80b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f694975496e3139227d2b9773b8", + "000000d70000004bbff9e4380b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f69496751324675496e3139227d874170f9", + "000000d70000004bbff9e4380b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f69496765573931496e3139227d35994406", + "000000db0000004b7a0909390b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f6949676447567362434a3966513d3d227de17e04f2", + "000000d70000004bbff9e4380b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f694967625755696658303d227d27157827", + "000000d70000004bbff9e4380b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f69496759534a3966513d3d227d93de2078", + "000000d70000004bbff9e4380b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f694967596d6c30496e3139227d47f89aea", + "000000db0000004b7a0909390b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f69496759574a76645851696658303d227defe54875", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f69496765573931636e4e6c624759696658303d227d2c5dd674", + "000000d30000004b4a7942f80b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131396b5a57783059534973496d6c755a475634496a6f774c434a6b5a5778305953493665794a306558426c496a6f69644756346446396b5a57783059534973496e526c654851694f69492f496e3139227dac32c541", + "0000009b0000004b22fa51700b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a306558426c496a6f695932397564475675644639696247396a6131397a64473977496977696157356b5a5867694f6a4239227dc0567ebe", + ], + ], + "meta.llama2-13b-chat-v1::[INST] The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.": [ + { + "Content-Type": "application/vnd.amazon.eventstream", + "x-amzn-RequestId": "cce6b34c-812c-4f97-8885-515829aa9639", + }, + 200, + [ + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949434973496e427962323177644639306232746c626c396a62335675644349364e7a5973496d646c626d56795958527062323566644739725a57356659323931626e51694f6a4573496e4e3062334266636d566863323975496a70756457787366513d3d227d37a74e44", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694945686c624778764969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f794c434a7a6447397758334a6c59584e7662694936626e56736248303d227d82bd6987", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d79776963335276634639795a57467a623234694f6d353162477839227d69a22395", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949456c304969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f304c434a7a6447397758334a6c59584e7662694936626e56736248303d227d0c311931", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694a794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e53776963335276634639795a57467a623234694f6d353162477839227d208768c0", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6963794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e69776963335276634639795a57467a623234694f6d353162477839227da857c5f9", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494764795a5746304969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f334c434a7a6447397758334a6c59584e7662694936626e56736248303d227d1c1ffb32", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494852764969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f344c434a7a6447397758334a6c59584e7662694936626e56736248303d227dc6a01d05", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949485268624773694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a6b73496e4e3062334266636d566863323975496a70756457787366513d3d227d1ade656f", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494852764969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f784d43776963335276634639795a57467a623234694f6d353162477839227df38e415b", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949486c7664534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d544573496e4e3062334266636d566863323975496a70756457787366513d3d227df8043fc6", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d544973496e4e3062334266636d566863323975496a70756457787366513d3d227da3876c32", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949456b694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a457a4c434a7a6447397758334a6c59584e7662694936626e56736248303d227df2429ce3", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694a794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d545173496e4e3062334266636d566863323975496a70756457787366513d3d227d3e44d0ad", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6962534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d545573496e4e3062334266636d566863323975496a70756457787366513d3d227d43dcbe46", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494746754969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f784e69776963335276634639795a57467a623234694f6d353162477839227ddefd896e", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494545694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a45334c434a7a6447397758334a6c59584e7662694936626e56736248303d227d865b949f", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6953534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d546773496e4e3062334266636d566863323975496a70756457787366513d3d227d87c627db", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694c434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d546b73496e4e3062334266636d566863323975496a70756457787366513d3d227d4efcc97d", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494746755a434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d6a4173496e4e3062334266636d566863323975496a70756457787366513d3d227dc9fb2b36", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949456b694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a49784c434a7a6447397758334a6c59584e7662694936626e56736248303d227d7d13633f", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694a794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d6a4973496e4e3062334266636d566863323975496a70756457787366513d3d227dabef6b74", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6962534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d6a4d73496e4e3062334266636d566863323975496a70756457787366513d3d227dd677059f", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694947686c636d55694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a49304c434a7a6447397758334a6c59584e7662694936626e56736248303d227d778b6773", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494852764969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f794e53776963335276634639795a57467a623234694f6d353162477839227d3f000197", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694947686c624841694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a49324c434a7a6447397758334a6c59584e7662694936626e56736248303d227d262f2f7c", + "000000eb0000004bdb28b1bf0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494746756333646c63694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d6a6373496e4e3062334266636d566863323975496a70756457787366513d3d227dd2632962", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694947467565534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d6a6773496e4e3062334266636d566863323975496a70756457787366513d3d227d2ca266eb", + "000000ef0000004b2ea8177f0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494846315a584e306157397563794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d6a6b73496e4e3062334266636d566863323975496a70756457787366513d3d227d420dc939", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949486c7664534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d7a4173496e4e3062334266636d566863323975496a70756457787366513d3d227d3ba3c73a", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494731705a3268304969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f7a4d53776963335276634639795a57467a623234694f6d353162477839227d742bb4c0", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949476868646d55694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a4d794c434a7a6447397758334a6c59584e7662694936626e56736248303d227d3ba80967", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694c694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d7a4d73496e4e3062334266636d566863323975496a70756457787366513d3d227d7d7ec395", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694946646f595851694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a4d304c434a7a6447397758334a6c59584e7662694936626e56736248303d227d2aff1e3e", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694a794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d7a5573496e4e3062334266636d566863323975496a70756457787366513d3d227dfde32851", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6963794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d7a5973496e4e3062334266636d566863323975496a70756457787366513d3d227de1891392", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494739754969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f7a4e79776963335276634639795a57467a623234694f6d353162477839227d1923b2d1", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949486c76645849694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a4d344c434a7a6447397758334a6c59584e7662694936626e56736248303d227d0475ce8e", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949473170626d51694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a4d354c434a7a6447397758334a6c59584e7662694936626e56736248303d227d39ee9d93", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6950794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e444173496e4e3062334266636d566863323975496a70756457787366513d3d227d23f8fbfc", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e444573496e4e3062334266636d566863323975496a70756457787366513d3d227d034c8c7e", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f304d69776963335276634639795a57467a623234694f6d353162477839227dd1325fa5", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f304d79776963335276634639795a57467a623234694f6d353162477839227d57fad2a0", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f304e43776963335276634639795a57467a623234694f6d353162477839227d2ba0111c", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69384a2b6b6c434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e445573496e4e3062334266636d566863323975496a70756457787366513d3d227d5170fd7c", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494552764969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f304e69776963335276634639795a57467a623234694f6d353162477839227d20bec256", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949486c7664534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e446373496e4e3062334266636d566863323975496a70756457787366513d3d227d3fb405c3", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949476868646d55694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a51344c434a7a6447397758334a6c59584e7662694936626e56736248303d227df9307c2f", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494745694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a51354c434a7a6447397758334a6c59584e7662694936626e56736248303d227daac7d4f1", + "000000eb0000004bdb28b1bf0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949484e775a574e705a6d6c6a4969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f314d43776963335276634639795a57467a623234694f6d353162477839227d8320a489", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694948527663476c6a4969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f314d53776963335276634639795a57467a623234694f6d353162477839227d8422eb4f", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949486c7664534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e544973496e4e3062334266636d566863323975496a70756457787366513d3d227d071ce2e2", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694a794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e544d73496e4e3062334266636d566863323975496a70756457787366513d3d227ddc056360", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f695a434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e545173496e4e3062334266636d566863323975496a70756457787366513d3d227d41081c0a", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949477870613255694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a55314c434a7a6447397758334a6c59584e7662694936626e56736248303d227d25fd8278", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494852764969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f314e69776963335276634639795a57467a623234694f6d353162477839227d7fdc9979", + "000000eb0000004bdb28b1bf0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694947527063324e3163334d694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a55334c434a7a6447397758334a6c59584e7662694936626e56736248303d227dbdcf1f52", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694c434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e546773496e4e3062334266636d566863323975496a70756457787366513d3d227db1e41459", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494739794969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f314f53776963335276634639795a57467a623234694f6d353162477839227d95ce8435", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494752764969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f324d43776963335276634639795a57467a623234694f6d353162477839227da1fc1b06", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949486c7664534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e6a4573496e4e3062334266636d566863323975496a70756457787366513d3d227dfe642df8", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949477031633351694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a59794c434a7a6447397758334a6c59584e7662694936626e56736248303d227d87d3b447", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949486468626e51694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a597a4c434a7a6447397758334a6c59584e7662694936626e56736248303d227db7527265", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494852764969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f324e43776963335276634639795a57467a623234694f6d353162477839227dd2c7cbab", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949474e6f595851694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a59314c434a7a6447397758334a6c59584e7662694936626e56736248303d227d20af0e31", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6950794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e6a5973496e4e3062334266636d566863323975496a70756457787366513d3d227d6f984397", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e6a6373496e4e3062334266636d566863323975496a70756457787366513d3d227dc29ca445", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f324f43776963335276634639795a57467a623234694f6d353162477839227d0a0f23b6", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f324f53776963335276634639795a57467a623234694f6d353162477839227d8cc7aeb3", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f334d43776963335276634639795a57467a623234694f6d353162477839227d7c290d77", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69384a2b5372434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e7a4573496e4e3062334266636d566863323975496a70756457787366513d3d227dc3364864", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949456b694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a63794c434a7a6447397758334a6c59584e7662694936626e56736248303d227d92976f7d", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694a794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e7a4d73496e4e3062334266636d566863323975496a70756457787366513d3d227d6e2881b6", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6962534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e7a5173496e4e3062334266636d566863323975496a70756457787366513d3d227dedfdb5e0", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694947467362434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e7a5573496e4e3062334266636d566863323975496a70756457787366513d3d227dc682e026", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949475668636e4d694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a63324c434a7a6447397758334a6c59584e7662694936626e56736248303d227d8ee6d357", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e7a6373496e4e3062334266636d566863323975496a70756457787366513d3d227d9d4f992d", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e7a6773496e4e3062334266636d566863323975496a70756457787366513d3d227d6addbddd", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f334f53776963335276634639795a57467a623234694f6d353162477839227d82204662", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f344d43776963335276634639795a57467a623234694f6d353162477839227d549e9740", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f344d53776963335276634639795a57467a623234694f6d353162477839227dd2561a45", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69384a2b5267694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f444973496e4e3062334266636d566863323975496a70756457787366513d3d227da5139607", + "0000018f0000004b7cc6b3970b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f344d79776963335276634639795a57467a623234694f694a7a644739774969776959573168656d39754c574a6c5a484a76593273746157353262324e6864476c76626b316c64484a7059334d694f6e736961573577645852556232746c626b4e7664573530496a6f334e6977696233563063485630564739725a57354462335675644349364f444d73496d6c75646d396a595852706232354d5958526c626d4e35496a6f794d7a63314c434a6d61584a7a64454a356447564d5958526c626d4e35496a6f7a4e6a5a3966513d3d227d34e01b75", + ], + ], + "amazon.titan-text-express-v1::What is 212 degrees Fahrenheit converted to Celsius?": [ + { + "Content-Type": "application/vnd.amazon.eventstream", + "x-amzn-RequestId": "b427270f-371a-458d-81b6-a05aafb2704c", + }, + 200, + [ + "000002370000004bdae582ec0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a7664585277645852555a586830496a6f69584734784947526c5a334a6c5a534247595768795a57356f5a576c3049476c7a494441754e5459675a47566e636d566c637942445a57787a6158567a4c694255614756795a575a76636d5573494449784d69426b5a5764795a575567526d466f636d56756147567064434270626942445a57787a6158567a494864766457786b49474a6c494445784e5334334d6934694c434a70626d526c654349364d437769644739305957785064585277645852555a586830564739725a57354462335675644349364d7a5573496d4e76625842735a585270623235535a57467a623234694f694a475355354a553067694c434a70626e42316446526c654852556232746c626b4e7664573530496a6f784d69776959573168656d39754c574a6c5a484a76593273746157353262324e6864476c76626b316c64484a7059334d694f6e736961573577645852556232746c626b4e7664573530496a6f784d6977696233563063485630564739725a57354462335675644349364d7a5573496d6c75646d396a595852706232354d5958526c626d4e35496a6f794d7a4d354c434a6d61584a7a64454a356447564d5958526c626d4e35496a6f794d7a4d356658303d227d358ac004" + ], + ], + "anthropic.claude-instant-v1::Human: What is 212 degrees Fahrenheit converted to Celsius? Assistant:": [ + { + "Content-Type": "application/vnd.amazon.eventstream", + "x-amzn-RequestId": "a645548f-0b3a-47ce-a675-f51e6e9037de", + }, + 200, + [ + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694945686c636d55694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d71ffbf2d", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494746795a534973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d9f82f061", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694948526f5a534973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227dee0662df", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949484e305a58427a4969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227d76bf1639", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494852764969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227daf097af1", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949474e76626e5a6c636e51694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d5955803a", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494449784d694973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227dfa89690e", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694947526c5a334a6c5a584d694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227dbe5287e4", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494559694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d8732a806", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69595768795a57356f5a576c304969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227d066744eb", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494852764969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227daf097af1", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949454e6c62484e7064584d694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227df62aca9e", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694f694973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227de96ff0b6", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6958473563626a45694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227df6e5e085", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694b534973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d82afca0e", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694946526f5a534973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d73834b92", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949475a76636d3131624745694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d3ad98743", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494852764969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227daf097af1", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949474e76626e5a6c636e51694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d5955803a", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949474a6c6448646c5a5734694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227deb6a0bd6", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494559694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d8732a806", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69595768795a57356f5a576c304969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227d066744eb", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494746755a434973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d7e666d0f", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949454e6c62484e7064584d694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227df62aca9e", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949476c7a4969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227d9a64e3c4", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694f694973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227de96ff0b6", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f695847346749434973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d4292c7bb", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949454d694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d1c09da34", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494430694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227db45e8aa5", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494367694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d1f5f0f41", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6952694973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227dc48bec13", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494330694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227dae344b5e", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949444d794969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227d8d3ee747", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694b534973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d82afca0e", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949436f694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d29a16fe0", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494455694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227dc85354c4", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694c794973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d26f20099", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694f534973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227dfff8a709", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6958473563626a49694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d36dc3354", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694b534973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d82afca0e", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494642734969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227de1cc18f5", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69645763694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d7e451c81", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949476c754969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227da2508214", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494449784d694973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227dfa89690e", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694947526c5a334a6c5a584d694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227dbe5287e4", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494559694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d8732a806", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69595768795a57356f5a576c304969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227d066744eb", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949475a7663694973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227dd72b242b", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494559694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d8732a806", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694f694973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227de96ff0b6", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f695847346749434973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d4292c7bb", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949454d694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d1c09da34", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494430694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227db45e8aa5", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494367694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d1f5f0f41", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694d6a45794969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227d184cd7ac", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494330694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227dae344b5e", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949444d794969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227d8d3ee747", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694b534973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d82afca0e", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949436f694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d29a16fe0", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494455694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227dc85354c4", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694c794973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d26f20099", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694f534973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227dfff8a709", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f695847346749434973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d4292c7bb", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949454d694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d1c09da34", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494430694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227db45e8aa5", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494445344d434973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227ddeedbeac", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949436f694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d29a16fe0", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494455694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227dc85354c4", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694c794973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d26f20099", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694f534973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227dfff8a709", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f695847346749434973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d4292c7bb", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949454d694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d1c09da34", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494430694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227db45e8aa5", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494445774d434973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d0c0fabb4", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6958473563626a4d694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227dc01b8024", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694b534973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d82afca0e", + "000000b70000004b26cb93750b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694946526f5a584a6c5a6d39795a534973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d8f97117a", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694c434973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d30655726", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494449784d694973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227dfa89690e", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694947526c5a334a6c5a584d694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227dbe5287e4", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494559694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227d8732a806", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69595768795a57356f5a576c304969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227d066744eb", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949476c7a4969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227d9a64e3c4", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949475678645746734969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227d08092f6c", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494852764969776963335276634639795a57467a623234694f6d353162477773496e4e30623341694f6d353162477839227daf097af1", + "000000af0000004b765b4f360b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f69494445774d434973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227d0c0fabb4", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694947526c5a334a6c5a584d694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227dbe5287e4", + "000000b30000004bd34b35b50b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f6949454e6c62484e7064584d694c434a7a6447397758334a6c59584e7662694936626e5673624377696333527663434936626e56736248303d227df62aca9e", + "000000ab0000004b83dbe9f60b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694c694973496e4e3062334266636d566863323975496a7075645778734c434a7a64473977496a70756457787366513d3d227dbe4ddce4", + "0000016b0000004ba192d2880b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6a623231776247563061573975496a6f694969776963335276634639795a57467a623234694f694a7a6447397758334e6c6358566c626d4e6c496977696333527663434936496c78755847354964573168626a6f694c434a686257463662323474596d566b636d396a61793170626e5a76593246306157397554575630636d6c6a6379493665794a70626e4231644652766132567551323931626e51694f6a45354c434a7664585277645852556232746c626b4e7664573530496a6f354f5377696157353262324e6864476c76626b78686447567559336b694f6a45314e7a4173496d5a70636e4e30516e6c305a5578686447567559336b694f6a51784d583139227d9a4fc171", + ], + ], + "cohere.command-text-v14::What is 212 degrees Fahrenheit converted to Celsius?": [ + { + "Content-Type": "application/vnd.amazon.eventstream", + "x-amzn-RequestId": "4f8ab6c5-42d1-4e35-9573-30f9f41f821e", + }, + 200, + [ + "000003f70000004bf8acb9920b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d46306157397563794936573373695a6d6c7561584e6f58334a6c59584e7662694936496b4e505456424d5256524649697769615751694f694a695a5463794e475a6d5953316c5a6a63344c54517a4d444974595464684e7930794e6a4a6a4e4463784f54566a4e5755694c434a305a586830496a6f694946527649474e76626e5a6c636e5167526d466f636d56756147567064434230627942445a57787a6158567a4c43423562335567593246754948567a5a534230614755675a6d39796258567359547063626c78755132567363326c316379413949436847595768795a57356f5a576c30494330674d7a497049436f674e53383558473563626b6c754948526f61584d675932467a5a537767615759676557393149476868646d55674d6a45794947526c5a334a6c5a584d67526d466f636d567561475670644377676557393149474e686269423163325567644768706379426d62334a74645778684948527649474e6862474e31624746305a534230614755675a58463161585a6862475675644342305a5731775a584a68644856795a534270626942445a57787a6158567a4f6c7875584735445a57787a6158567a494430674b4449784d69417449444d794b534171494455764f534139494445774d434171494455764f5341394944557758473563626c526f5a584a6c5a6d39795a5377674d6a45794947526c5a334a6c5a584d67526d466f636d567561475670644342706379426c63585670646d46735a57353049485276494455774947526c5a334a6c5a584d675132567363326c316379346966563073496d6c6b496a6f694e47593459574932597a55744e444a6b4d5330305a544d314c546b314e7a4d744d7a426d4f5759304d5759344d6a466c4969776963484a7662584230496a6f695632686864434270637941794d5449675a47566e636d566c63794247595768795a57356f5a576c3049474e76626e5a6c636e526c5a434230627942445a57787a6158567a50794973496d467459587076626931695a57527962324e724c576c75646d396a595852706232354e5a58527961574e7a496a7037496d6c7563485630564739725a57354462335675644349364f5377696233563063485630564739725a57354462335675644349364f544573496d6c75646d396a595852706232354d5958526c626d4e35496a6f794f5463794c434a6d61584a7a64454a356447564d5958526c626d4e35496a6f794f5463796658303d227deba065e0" + ], + ], + "meta.llama2-13b-chat-v1::What is 212 degrees Fahrenheit converted to Celsius?": [ + { + "Content-Type": "application/vnd.amazon.eventstream", + "x-amzn-RequestId": "6dd99878-0919-4f92-850c-48f50f923b76", + }, + 200, + [ + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949434973496e427962323177644639306232746c626c396a62335675644349364d546373496d646c626d56795958527062323566644739725a57356659323931626e51694f6a4573496e4e3062334266636d566863323975496a70756457787366513d3d227d8ad5573b", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694946646f595851694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a4973496e4e3062334266636d566863323975496a70756457787366513d3d227dc79406b1", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949476c7a4969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f7a4c434a7a6447397758334a6c59584e7662694936626e56736248303d227d03c98d5f", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694948526f5a534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e43776963335276634639795a57467a623234694f6d353162477839227d87f1a596", + "000000ef0000004b2ea8177f0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949474e76626e5a6c636e4e70623234694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a5573496e4e3062334266636d566863323975496a70756457787366513d3d227d10bef8bd", + "000000eb0000004bdb28b1bf0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949475a76636d3131624745694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a5973496e4e3062334266636d566863323975496a70756457787366513d3d227d587688a5", + "000000df0000004b8f89aff90b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6950794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e79776963335276634639795a57467a623234694f6d353162477839227d2a55ad0a", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69584734694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a6773496e4e3062334266636d566863323975496a70756457787366513d3d227d6967bb80", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69584734694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a6b73496e4e3062334266636d566863323975496a70756457787366513d3d227dfbf995fe", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d544173496e4e3062334266636d566863323975496a70756457787366513d3d227d2d794c92", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d544573496e4e3062334266636d566863323975496a70756457787366513d3d227d28b94ab1", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d544973496e4e3062334266636d566863323975496a70756457787366513d3d227dce6d78c6", + "000000eb0000004bdb28b1bf0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694947526c5a334a6c5a584d694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a457a4c434a7a6447397758334a6c59584e7662694936626e56736248303d227d4f48e7b9", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494559694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a45304c434a7a6447397758334a6c59584e7662694936626e56736248303d227d0992a7e0", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69595768795a5734694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a45314c434a7a6447397758334a6c59584e7662694936626e56736248303d227ddcb2ffa3", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f696147567064434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d545973496e4e3062334266636d566863323975496a70756457787366513d3d227d53be207a", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949476c7a4969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f784e79776963335276634639795a57467a623234694f6d353162477839227d14ca1e11", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949475678645746734969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f784f43776963335276634639795a57467a623234694f6d353162477839227de610d9c7", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494852764969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f784f53776963335276634639795a57467a623234694f6d353162477839227d0d870a4e", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d6a4173496e4e3062334266636d566863323975496a70756457787366513d3d227dad6d87d8", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d6a4573496e4e3062334266636d566863323975496a70756457787366513d3d227d435fabd5", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d6a4973496e4e3062334266636d566863323975496a70756457787366513d3d227d57d9bb71", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d6a4d73496e4e3062334266636d566863323975496a70756457787366513d3d227d2653a15b", + "000000eb0000004bdb28b1bf0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694947526c5a334a6c5a584d694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a49304c434a7a6447397758334a6c59584e7662694936626e56736248303d227db83e662f", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949454e6c62434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d6a5573496e4e3062334266636d566863323975496a70756457787366513d3d227d49763324", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6963326b694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a49324c434a7a6447397758334a6c59584e7662694936626e56736248303d227d57ef43c7", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6964584d694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a49334c434a7a6447397758334a6c59584e7662694936626e56736248303d227d31fa2ded", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694c694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d6a6773496e4e3062334266636d566863323975496a70756457787366513d3d227d45d624b4", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69584734694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a49354c434a7a6447397758334a6c59584e7662694936626e56736248303d227d50170c09", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69584734694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a4d774c434a7a6447397758334a6c59584e7662694936626e56736248303d227d334aff43", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f695647686c4969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f7a4d53776963335276634639795a57467a623234694f6d353162477839227da3b5dcb6", + "000000ef0000004b2ea8177f0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949474e76626e5a6c636e4e70623234694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a4d794c434a7a6447397758334a6c59584e7662694936626e56736248303d227d04d2363e", + "000000eb0000004bdb28b1bf0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949475a76636d3131624745694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a4d7a4c434a7a6447397758334a6c59584e7662694936626e56736248303d227dcdac512b", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949476c7a4969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f7a4e43776963335276634639795a57467a623234694f6d353162477839227d31fe2917", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694f694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d7a5573496e4e3062334266636d566863323975496a70756457787366513d3d227d12ef5dd9", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69584734694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a4d324c434a7a6447397758334a6c59584e7662694936626e56736248303d227d092a8c14", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69584734694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a4d334c434a7a6447397758334a6c59584e7662694936626e56736248303d227d07cd64c5", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69777241694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a4d344c434a7a6447397758334a6c59584e7662694936626e56736248303d227d3ac46cfa", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6951794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364d7a6b73496e4e3062334266636d566863323975496a70756457787366513d3d227db44f4efe", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494430694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a51774c434a7a6447397758334a6c59584e7662694936626e56736248303d227d20eaba75", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494367694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a51784c434a7a6447397758334a6c59584e7662694936626e56736248303d227d7e72f783", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69777241694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a51794c434a7a6447397758334a6c59584e7662694936626e56736248303d227df85c19b2", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6952694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e444d73496e4e3062334266636d566863323975496a70756457787366513d3d227dd6166c06", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494330694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a51304c434a7a6447397758334a6c59584e7662694936626e56736248303d227d73d3f258", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e445573496e4e3062334266636d566863323975496a70756457787366513d3d227d1e15e297", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e445973496e4e3062334266636d566863323975496a70756457787366513d3d227de161d81d", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e446373496e4e3062334266636d566863323975496a70756457787366513d3d227d9b436cbd", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694b534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e446773496e4e3062334266636d566863323975496a70756457787366513d3d227d664be53d", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494d4f584969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f304f53776963335276634639795a57467a623234694f6d353162477839227d437bffb5", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e544173496e4e3062334266636d566863323975496a70756457787366513d3d227dab0d95e6", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694e534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e544573496e4e3062334266636d566863323975496a70756457787366513d3d227dd4e37d1a", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694c794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e544973496e4e3062334266636d566863323975496a70756457787366513d3d227d5547f6e9", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694f534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e544d73496e4e3062334266636d566863323975496a70756457787366513d3d227d47430ae1", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69584734694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a55304c434a7a6447397758334a6c59584e7662694936626e56736248303d227deaa8070b", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69584734694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a55314c434a7a6447397758334a6c59584e7662694936626e56736248303d227de44fefda", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69553238694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a55324c434a7a6447397758334a6c59584e7662694936626e56736248303d227dcc1d90ca", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694c434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e546373496e4e3062334266636d566863323975496a70756457787366513d3d227dc06e0e73", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494852764969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f314f43776963335276634639795a57467a623234694f6d353162477839227dfd8f11d0", + "000000eb0000004bdb28b1bf0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949474e76626e5a6c636e51694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a55354c434a7a6447397758334a6c59584e7662694936626e56736248303d227d9fe07832", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e6a4173496e4e3062334266636d566863323975496a70756457787366513d3d227dc0eb7482", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e6a4573496e4e3062334266636d566863323975496a70756457787366513d3d227d5a934486", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e6a4973496e4e3062334266636d566863323975496a70756457787366513d3d227dbc4776f1", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e6a4d73496e4e3062334266636d566863323975496a70756457787366513d3d227db98770d2", + "000000eb0000004bdb28b1bf0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694947526c5a334a6c5a584d694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a59304c434a7a6447397758334a6c59584e7662694936626e56736248303d227da56708c6", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494559694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a59314c434a7a6447397758334a6c59584e7662694936626e56736248303d227d88b20fa6", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69595768795a5734694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a59324c434a7a6447397758334a6c59584e7662694936626e56736248303d227d405d8647", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f696147567064434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e6a6373496e4e3062334266636d566863323975496a70756457787366513d3d227da9e4b83e", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494852764969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f324f43776963335276634639795a57467a623234694f6d353162477839227deea728a3", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949454e6c62434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e6a6b73496e4e3062334266636d566863323975496a70756457787366513d3d227da940502e", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6963326b694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a63774c434a7a6447397758334a6c59584e7662694936626e56736248303d227d8cecd403", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6964584d694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a63784c434a7a6447397758334a6c59584e7662694936626e56736248303d227dbb968e47", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694c434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e7a4973496e4e3062334266636d566863323975496a70756457787366513d3d227d93200836", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694948646c4969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f334d79776963335276634639795a57467a623234694f6d353162477839227dd66d2ee7", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949474e6862694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e7a5173496e4e3062334266636d566863323975496a70756457787366513d3d227dec7f3121", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694948567a5a534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e7a5573496e4e3062334266636d566863323975496a70756457787366513d3d227dbb1f0174", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694948526f5a534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364e7a5973496e4e3062334266636d566863323975496a70756457787366513d3d227db9f2b3dd", + "000000eb0000004bdb28b1bf0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949475a76636d3131624745694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a63334c434a7a6447397758334a6c59584e7662694936626e56736248303d227d2b853909", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949477870613255694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a63344c434a7a6447397758334a6c59584e7662694936626e56736248303d227d7e216de9", + "000000e70000004b1ed85cbe0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694948526f61584d694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a63354c434a7a6447397758334a6c59584e7662694936626e56736248303d227db006b745", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694f694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f444173496e4e3062334266636d566863323975496a70756457787366513d3d227d5cde6a12", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69584734694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a67784c434a7a6447397758334a6c59584e7662694936626e56736248303d227d8da1c76f", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69584734694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a67794c434a7a6447397758334a6c59584e7662694936626e56736248303d227d83462fbe", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69777241694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a677a4c434a7a6447397758334a6c59584e7662694936626e56736248303d227d85d084c5", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6951794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f445173496e4e3062334266636d566863323975496a70756457787366513d3d227d6a97878c", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494430694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a67314c434a7a6447397758334a6c59584e7662694936626e56736248303d227d67065455", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494367694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a67324c434a7a6447397758334a6c59584e7662694936626e56736248303d227d753efc6f", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f446373496e4e3062334266636d566863323975496a70756457787366513d3d227dbfc13d8b", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f446773496e4e3062334266636d566863323975496a70756457787366513d3d227dba013ba8", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f446b73496e4e3062334266636d566863323975496a70756457787366513d3d227d5cd509df", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494330694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a6b774c434a7a6447397758334a6c59584e7662694936626e56736248303d227da846dad7", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f544573496e4e3062334266636d566863323975496a70756457787366513d3d227dfe05defa", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f544973496e4e3062334266636d566863323975496a70756457787366513d3d227d0171e470", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694d694973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f544d73496e4e3062334266636d566863323975496a70756457787366513d3d227df6e3c080", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694b534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f545173496e4e3062334266636d566863323975496a70756457787366513d3d227df5a613bd", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69494d4f584969776963484a7662584230583352766132567558324e7664573530496a7075645778734c434a6e5a57356c636d463061573975583352766132567558324e7664573530496a6f354e53776963335276634639795a57467a623234694f6d353162477839227d4a635728", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f6949434973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f545973496e4e3062334266636d566863323975496a70756457787366513d3d227d71c29e6d", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694e534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f546373496e4e3062334266636d566863323975496a70756457787366513d3d227d839ce6c1", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694c794973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f546773496e4e3062334266636d566863323975496a70756457787366513d3d227de12c5966", + "000000e30000004beb58fa7e0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f694f534973496e427962323177644639306232746c626c396a6233567564434936626e5673624377695a3256755a584a6864476c76626c39306232746c626c396a62335675644349364f546b73496e4e3062334266636d566863323975496a70756457787366513d3d227d103c913a", + "000001970000004b2c566fd40b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a6e5a57356c636d463061573975496a6f69584734694c434a77636d397463485266644739725a57356659323931626e51694f6d353162477773496d646c626d56795958527062323566644739725a57356659323931626e51694f6a45774d43776963335276634639795a57467a623234694f694a735a57356e644767694c434a686257463662323474596d566b636d396a61793170626e5a76593246306157397554575630636d6c6a6379493665794a70626e4231644652766132567551323931626e51694f6a45334c434a7664585277645852556232746c626b4e7664573530496a6f784d444173496d6c75646d396a595852706232354d5958526c626d4e35496a6f794e7a59304c434a6d61584a7a64454a356447564d5958526c626d4e35496a6f7a4d7a683966513d3d227d7f984a9b", + ], + ], + "amazon.titan-text-express-v1::Malformed Streaming Chunk": [ + { + "Content-Type": "application/vnd.amazon.eventstream", + "x-amzn-RequestId": "a5a8cebb-fd33-4437-8168-5667fbdfc1fb", + }, + 200, + [ + # Payload is intentionally damaged to throw an exception + "00004bdae582ec0b3a6576656e742d747970650700056368756e6b0d3a636f6e74656e742d747970650700106170706c69636174696f6e2f6a736f6e0d3a6d6573736167652d747970650700056576656e747b226279746573223a2265794a7664585277645852555a586830496a6f69584734784947526c5a334a6c5a534247595768795a57356f5a576c3049476c7a494441754e5459675a47566e636d566c637942445a57787a6158567a4c694255614756795a575a76636d5573494449784d69426b5a5764795a575567526d466f636d56756147567064434270626942445a57787a6158567a494864766457786b49474a6c494445784e5334334d6934694c434a70626d526c654349364d437769644739305957785064585277645852555a586830564739725a57354462335675644349364d7a5573496d4e76625842735a585270623235535a57467a623234694f694a475355354a553067694c434a70626e42316446526c654852556232746c626b4e7664573530496a6f784d69776959573168656d39754c574a6c5a484a76593273746157353262324e6864476c76626b316c64484a7059334d694f6e736961573577645852556232746c626b4e7664573530496a6f784d6977696233563063485630564739725a57354462335675644349364d7a5573496d6c75646d396a595852706232354d5958526c626d4e35496a6f794d7a4d354c434a6d61584a7a64454a356447564d5958526c626d4e35496a6f794d7a4d356658303d227d358ac004" + ], + ], + "amazon.titan-text-express-v1::Malformed Streaming Body": [ + { + "Content-Type": "application/vnd.amazon.eventstream", + "x-amzn-RequestId": "a5a8cebb-fd33-4437-8168-5667fbdfc1fb", + "x-amzn-errortype": "ValidationException:http://internal.amazon.com/coral/com.amazon.bedrock/", + }, + 200, + [ + # Computes an example payload for an error inside a stream, from human readable format to a hex string. + encode_streaming_payload( + {"event-type": "chunk", "content-type": "application/json"}, + { + "outputText": "ValidationException", + "index": 0, + "totalOutputTextTokenCount": 35, + "completionReason": "FINISH", + "inputTextTokenCount": 12, + "amazon-bedrock-invocationMetrics": { + "inputTokenCount": 12, + "outputTokenCount": 35, + "invocationLatency": 2339, + "firstByteLatency": 2339, + }, + }, + malformed_body=True, + ).hex() + ], + ], + "amazon.titan-text-express-v1::Streaming Exception": [ + { + "Content-Type": "application/vnd.amazon.eventstream", + "x-amzn-RequestId": "a5a8cebb-fd33-4437-8168-5667fbdfc1fb", + "x-amzn-errortype": "ValidationException:http://internal.amazon.com/coral/com.amazon.bedrock/", + }, + 200, + [ + # Computes an example payload for an error inside a stream, from human readable format to a hex string. + encode_streaming_payload( + { + "event-type": "chunk", + "content-type": "application/json", + "message-type": "error", + "exception-type": "ValidationException", + "error-code": "ValidationException", + "error-message": "Malformed input request, please reformat your input and try again.", + }, + { + "outputText": "ValidationException", + "index": 0, + "totalOutputTextTokenCount": 35, + "completionReason": "FINISH", + "inputTextTokenCount": 12, + "amazon-bedrock-invocationMetrics": { + "inputTokenCount": 12, + "outputTokenCount": 35, + "invocationLatency": 2339, + "firstByteLatency": 2339, + }, + }, + ).hex() + ], + ], +} + +RESPONSES = { + "mistral.mistral-7b-instruct-v0%3A2::[INST] What is 212 degrees Fahrenheit converted to Celsius? [/INST]": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "48c7ee13-7790-461f-959f-04b0a4cf91c8"}, + 200, + { + "outputs": [ + { + "text": " To convert a temperature from Fahrenheit to Celsius, you can use the following formula:\n\nCelsius = (Fahrenheit - 32) \u00d7 5/9\n\nSo, to convert 212 degrees Fahrenheit to Celsius:\n\nCelsius = (212 - 32) \u00d7 5/9\nCelsius = 180.56 \u00d7 5/9\nCelsius", + "stop_reason": "length", + } + ] + }, + ], + "amazon.titan-text-express-v1::User: The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "884db5c9-18ab-4f27-8892-33656176a2e6"}, + 200, + { + "inputTextTokenCount": 72, + "results": [ + {"tokenCount": 23, "outputText": " Hello, how can I help you today?", "completionReason": "FINISH"} + ], + }, + ], + "anthropic.claude-instant-v1::The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "1a72a1f6-310f-469c-af1d-2c59eb600089"}, + 200, + { + "id": "compl_01EGAoDn3azRGBGFLADWEzn7", + "type": "message", + "role": "assistant", + "content": [{"type": "text", "text": "Hello! It's nice to meet you."}], + "model": "claude-instant-1.2", + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 73, "output_tokens": 13}, + }, + ], + "meta.llama2-13b-chat-v1::[INST] The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "cce6b34c-812c-4f97-8885-515829aa9639"}, + 200, + { + "generation": " Hello! It's great to talk to you! I'm an AI, and I'm here to help answer any questions you might have. What's on your mind? \ud83e\udd14 Do you have a specific topic you'd like to discuss, or is there something you'd like to know? \ud83e\udd13 I'm all ears! \ud83d\udc42", + "prompt_token_count": 76, + "generation_token_count": 86, + "stop_reason": "stop", + }, + ], + "ai21.j2-mid-v1::What is 212 degrees Fahrenheit converted to Celsius?": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "228ee63f-4eca-4b7d-b679-bc920de63525"}, + 200, + { + "id": 1234, + "prompt": { + "text": "What is 212 degrees Fahrenheit converted to Celsius?", + "tokens": [ + { + "generatedToken": { + "token": "\u2581What\u2581is", + "logprob": -7.703575134277344, + "raw_logprob": -7.703575134277344, + }, + "topTokens": None, + "textRange": {"start": 0, "end": 7}, + }, + { + "generatedToken": { + "token": "\u2581", + "logprob": -3.4491159915924072, + "raw_logprob": -3.4491159915924072, + }, + "topTokens": None, + "textRange": {"start": 7, "end": 8}, + }, + { + "generatedToken": { + "token": "212", + "logprob": -9.432294845581055, + "raw_logprob": -9.432294845581055, + }, + "topTokens": None, + "textRange": {"start": 8, "end": 11}, + }, + { + "generatedToken": { + "token": "\u2581degrees\u2581Fahrenheit", + "logprob": -9.64009952545166, + "raw_logprob": -9.64009952545166, + }, + "topTokens": None, + "textRange": {"start": 11, "end": 30}, + }, + { + "generatedToken": { + "token": "\u2581converted\u2581to", + "logprob": -8.4347505569458, + "raw_logprob": -8.4347505569458, + }, + "topTokens": None, + "textRange": {"start": 30, "end": 43}, + }, + { + "generatedToken": { + "token": "\u2581Celsius", + "logprob": -0.17065171897411346, + "raw_logprob": -0.17065171897411346, + }, + "topTokens": None, + "textRange": {"start": 43, "end": 51}, + }, + { + "generatedToken": { + "token": "?", + "logprob": -8.000349998474121, + "raw_logprob": -8.000349998474121, + }, + "topTokens": None, + "textRange": {"start": 51, "end": 52}, + }, + ], + }, + "completions": [ + { + "data": { + "text": "\n212 degrees Fahrenheit is equal to 100 degrees Celsius.", + "tokens": [ + { + "generatedToken": { + "token": "<|newline|>", + "logprob": 0.0, + "raw_logprob": -1.0609570381348021e-05, + }, + "topTokens": None, + "textRange": {"start": 0, "end": 1}, + }, + { + "generatedToken": { + "token": "\u2581", + "logprob": -0.0012434140080586076, + "raw_logprob": -0.017860885709524155, + }, + "topTokens": None, + "textRange": {"start": 1, "end": 1}, + }, + { + "generatedToken": { + "token": "212", + "logprob": -0.047134462743997574, + "raw_logprob": -0.12055955827236176, + }, + "topTokens": None, + "textRange": {"start": 1, "end": 4}, + }, + { + "generatedToken": { + "token": "\u2581degrees\u2581Fahrenheit", + "logprob": -0.006817296147346497, + "raw_logprob": -0.052842844277620316, + }, + "topTokens": None, + "textRange": {"start": 4, "end": 23}, + }, + { + "generatedToken": { + "token": "\u2581is\u2581equal\u2581to", + "logprob": -0.008958976715803146, + "raw_logprob": -0.0576501227915287, + }, + "topTokens": None, + "textRange": {"start": 23, "end": 35}, + }, + { + "generatedToken": { + "token": "\u2581", + "logprob": -4.756337511935271e-05, + "raw_logprob": -0.002072569215670228, + }, + "topTokens": None, + "textRange": {"start": 35, "end": 36}, + }, + { + "generatedToken": { + "token": "100", + "logprob": -1.6689286894688848e-06, + "raw_logprob": -0.00014327930693980306, + }, + "topTokens": None, + "textRange": {"start": 36, "end": 39}, + }, + { + "generatedToken": { + "token": "\u2581degrees\u2581Celsius", + "logprob": -0.0009832315845414996, + "raw_logprob": -0.009537134319543839, + }, + "topTokens": None, + "textRange": {"start": 39, "end": 55}, + }, + { + "generatedToken": { + "token": ".", + "logprob": -0.009822133928537369, + "raw_logprob": -0.04313727468252182, + }, + "topTokens": None, + "textRange": {"start": 55, "end": 56}, + }, + { + "generatedToken": { + "token": "<|endoftext|>", + "logprob": -0.10973381996154785, + "raw_logprob": -0.2600202262401581, + }, + "topTokens": None, + "textRange": {"start": 56, "end": 56}, + }, + ], + }, + "finishReason": {"reason": "endoftext"}, + } + ], + }, + ], + "amazon.titan-text-express-v1::What is 212 degrees Fahrenheit converted to Celsius?": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "81508a1c-33a8-4294-8743-f0c629af2f49"}, + 200, + { + "inputTextTokenCount": 12, + "results": [ + { + "tokenCount": 32, + "outputText": "\n1 degree Fahrenheit is 0.56 Celsius. Therefore, 212 degree Fahrenheit in Celsius would be 115.42.", + "completionReason": "FINISH", + } + ], + }, + ], + "anthropic.claude-instant-v1::Human: What is 212 degrees Fahrenheit converted to Celsius? Assistant:": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "6a886158-b39f-46ce-b214-97458ab76f2f"}, + 200, + { + "completion": " Okay, here are the steps to convert 212 degrees Fahrenheit to Celsius:\n\n1) The formula to convert between Fahrenheit and Celsius is:\n C = (F - 32) * 5/9\n\n2) Plug in 212 degrees Fahrenheit for F:\n C = (212 - 32) * 5/9\n C = 180 * 5/9\n C = 100\n\n3) Therefore, 212 degrees Fahrenheit converted to Celsius is 100 degrees Celsius.", + "stop_reason": "max_tokens", + "stop": None, + }, + ], + "cohere.command-text-v14::What is 212 degrees Fahrenheit converted to Celsius?": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "12912a17-aa13-45f3-914c-cc82166f3601"}, + 200, + { + "generations": [ + { + "finish_reason": "MAX_TOKENS", + "id": "deafebbd-7cdb-461f-8d6a-846602141f8f", + "text": " To convert from Fahrenheit to Celsius, you can use the following formula:\n\nCelsius = (Fahrenheit - 32) * 5/9\n\nIn this case, 212 degrees Fahrenheit is converted to Celsius as follows:\n\nCelsius = (212 - 32) * 5/9 = (180) * 5/9 = (180/9) = 20 degrees Celsius\n\nTherefore, 212 degrees Fahrenheit is equivalent to 20 degrees Celsius.\n\nIt's important to note that", + } + ], + "id": "12912a17-aa13-45f3-914c-cc82166f3601", + "prompt": "What is 212 degrees Fahrenheit converted to Celsius?", + }, + ], + "meta.llama2-13b-chat-v1::What is 212 degrees Fahrenheit converted to Celsius?": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "a168214d-742d-4244-bd7f-62214ffa07df"}, + 200, + { + "generation": "\n\n212\u00b0F = ?\u00b0C\n\nPlease help! I'm stuck!\n\nThank you!\n\nI hope this is the correct place to ask this question. Please let me know if it isn't.\n\nI appreciate your help!\n\nBest regards,\n\n[Your Name]", + "prompt_token_count": 17, + "generation_token_count": 69, + "stop_reason": "stop", + }, + ], + "amazon.titan-embed-g1-text-02::This is an embedding test.": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "b10ac895-eae3-4f07-b926-10b2866c55ed"}, + 200, + { + "embedding": [ + -0.14160156, + 0.034423828, + 0.54296875, + 0.10986328, + 0.053466797, + 0.3515625, + 0.12988281, + -0.0002708435, + -0.21484375, + 0.060302734, + 0.58984375, + -0.5859375, + 0.52734375, + 0.82421875, + -0.91015625, + -0.19628906, + 0.45703125, + 0.609375, + -0.67578125, + 0.39453125, + -0.46875, + -0.25390625, + -0.21191406, + 0.114746094, + 0.31640625, + -0.41015625, + -0.32617188, + -0.43554688, + 0.4765625, + -0.4921875, + 0.40429688, + 0.06542969, + 0.859375, + -0.008056641, + -0.19921875, + 0.072753906, + 0.33203125, + 0.69921875, + 0.39453125, + 0.15527344, + 0.08886719, + -0.25, + 0.859375, + 0.22949219, + -0.19042969, + 0.13769531, + -0.078125, + 0.41210938, + 0.875, + 0.5234375, + 0.59765625, + -0.22949219, + -0.22558594, + -0.47460938, + 0.37695312, + 0.51953125, + -0.5703125, + 0.46679688, + 0.43554688, + 0.17480469, + -0.080566406, + -0.16699219, + -0.734375, + -1.0625, + -0.33984375, + 0.390625, + -0.18847656, + -0.5234375, + -0.48828125, + 0.44921875, + -0.09814453, + -0.3359375, + 0.087402344, + 0.36914062, + 1.3203125, + 0.25585938, + 0.14746094, + -0.059570312, + -0.15820312, + -0.037353516, + -0.61328125, + -0.6484375, + -0.35351562, + 0.55078125, + -0.26953125, + 0.90234375, + 0.3671875, + 0.31054688, + 0.00014019012, + -0.171875, + 0.025512695, + 0.5078125, + 0.11621094, + 0.33203125, + 0.8125, + -0.3046875, + -1.078125, + -0.5703125, + 0.26171875, + -0.4609375, + 0.203125, + 0.44726562, + -0.5078125, + 0.41601562, + -0.1953125, + 0.028930664, + -0.57421875, + 0.2265625, + 0.13574219, + -0.040039062, + -0.22949219, + -0.515625, + -0.19042969, + -0.30078125, + 0.10058594, + -0.66796875, + 0.6015625, + 0.296875, + -0.765625, + -0.87109375, + 0.2265625, + 0.068847656, + -0.088378906, + -0.1328125, + -0.796875, + -0.37304688, + 0.47460938, + -0.3515625, + -0.8125, + -0.32226562, + 0.265625, + 0.3203125, + -0.4140625, + -0.49023438, + 0.859375, + -0.19140625, + -0.6328125, + 0.10546875, + -0.5625, + 0.66015625, + 0.26171875, + -0.2109375, + 0.421875, + -0.82421875, + 0.29296875, + 0.17773438, + 0.24023438, + 0.5078125, + -0.49804688, + -0.10205078, + 0.10498047, + -0.36132812, + -0.47460938, + -0.20996094, + 0.010070801, + -0.546875, + 0.66796875, + -0.123046875, + -0.75390625, + 0.19628906, + 0.17480469, + 0.18261719, + -0.96875, + -0.26171875, + 0.4921875, + -0.40039062, + 0.296875, + 0.1640625, + -0.20507812, + -0.36132812, + 0.76171875, + -1.234375, + -0.625, + 0.060058594, + -0.09375, + -0.14746094, + 1.09375, + 0.057861328, + 0.22460938, + -0.703125, + 0.07470703, + 0.23828125, + -0.083984375, + -0.54296875, + 0.5546875, + -0.5, + -0.390625, + 0.106933594, + 0.6640625, + 0.27734375, + -0.953125, + 0.35351562, + -0.7734375, + -0.77734375, + 0.16503906, + -0.42382812, + 0.36914062, + 0.020141602, + -1.3515625, + 0.18847656, + 0.13476562, + -0.034179688, + -0.03930664, + -0.03857422, + -0.027954102, + 0.73828125, + -0.18945312, + -0.09814453, + -0.46289062, + 0.36914062, + 0.033203125, + 0.020874023, + -0.703125, + 0.91796875, + 0.38671875, + 0.625, + -0.19335938, + -0.16796875, + -0.58203125, + 0.21386719, + -0.032470703, + -0.296875, + -0.15625, + -0.1640625, + -0.74609375, + 0.328125, + 0.5546875, + -0.1953125, + 1.0546875, + 0.171875, + -0.099609375, + 0.5234375, + 0.05078125, + -0.35742188, + -0.2734375, + -1.3203125, + -0.8515625, + -0.16015625, + 0.01574707, + 0.29296875, + 0.18457031, + -0.265625, + 0.048339844, + 0.045654297, + -0.32226562, + 0.087890625, + -0.0047302246, + 0.38671875, + 0.10644531, + -0.06225586, + 1.03125, + 0.94140625, + -0.3203125, + 0.20800781, + -1.171875, + 0.48046875, + -0.091796875, + 0.20800781, + -0.1328125, + -0.20507812, + 0.28125, + -0.47070312, + -0.09033203, + 0.0013809204, + -0.08203125, + 0.43359375, + -0.03100586, + -0.060791016, + -0.53515625, + -1.46875, + 0.000101566315, + 0.515625, + 0.40625, + -0.10498047, + -0.15820312, + -0.009460449, + -0.77734375, + -0.5859375, + 0.9765625, + 0.099609375, + 0.51953125, + 0.38085938, + -0.09667969, + -0.100097656, + -0.5, + -1.3125, + -0.18066406, + -0.099121094, + 0.26171875, + -0.14453125, + -0.546875, + 0.17578125, + 0.484375, + 0.765625, + 0.45703125, + 0.2734375, + 0.0028076172, + 0.17089844, + -0.32421875, + -0.37695312, + 0.30664062, + -0.48046875, + 0.07128906, + 0.031982422, + -0.31054688, + -0.055419922, + -0.29296875, + 0.3359375, + -0.296875, + 0.47851562, + -0.05126953, + 0.18457031, + -0.01953125, + -0.35742188, + 0.017944336, + -0.25, + 0.10595703, + 0.17382812, + -0.73828125, + 0.36914062, + -0.15234375, + -0.8125, + 0.17382812, + 0.048095703, + 0.5625, + -0.33789062, + 0.023071289, + -0.21972656, + 0.16015625, + 0.032958984, + -1.1171875, + -0.984375, + 0.83984375, + 0.009033203, + -0.042236328, + -0.46484375, + -0.08203125, + 0.44726562, + -0.765625, + -0.3984375, + -0.40820312, + -0.234375, + 0.044189453, + 0.119628906, + -0.7578125, + -0.55078125, + -0.4453125, + 0.7578125, + 0.34960938, + 0.96484375, + 0.35742188, + 0.36914062, + -0.35351562, + -0.36132812, + 1.109375, + 0.5859375, + 0.85546875, + -0.10644531, + -0.6953125, + -0.0066833496, + 0.042236328, + -0.06689453, + 0.36914062, + 0.9765625, + -0.3046875, + 0.59765625, + -0.6640625, + 0.21484375, + -0.07128906, + 1.1328125, + -0.51953125, + 0.86328125, + -0.11328125, + 0.15722656, + -0.36328125, + -0.04638672, + 1.4375, + 0.18457031, + -0.18359375, + 0.10595703, + -0.49023438, + -0.07324219, + -0.73046875, + -0.119140625, + 0.021118164, + 0.4921875, + -0.46875, + 0.28710938, + 0.3359375, + 0.11767578, + -0.2109375, + -0.14550781, + 0.39648438, + -0.27734375, + 0.48046875, + 0.12988281, + 0.45507812, + -0.375, + -0.84765625, + 0.25585938, + -0.36523438, + 0.8046875, + 0.42382812, + -0.24511719, + 0.54296875, + 0.71875, + 0.010009766, + -0.04296875, + 0.083984375, + -0.52734375, + 0.13964844, + -0.27539062, + -0.30273438, + 1.1484375, + -0.515625, + -0.19335938, + 0.58984375, + 0.049072266, + 0.703125, + -0.04272461, + 0.5078125, + 0.34960938, + -0.3359375, + -0.47460938, + 0.049316406, + 0.36523438, + 0.7578125, + -0.022827148, + -0.71484375, + 0.21972656, + 0.09716797, + -0.203125, + -0.36914062, + 1.34375, + 0.34179688, + 0.46679688, + 1.078125, + 0.26171875, + 0.41992188, + 0.22363281, + -0.515625, + -0.5703125, + 0.13378906, + 0.26757812, + -0.22558594, + -0.5234375, + 0.06689453, + 0.08251953, + -0.625, + 0.16796875, + 0.43164062, + -0.55859375, + 0.28125, + 0.078125, + 0.6328125, + 0.23242188, + -0.064941406, + -0.004486084, + -0.20703125, + 0.2734375, + 0.453125, + -0.734375, + 0.04272461, + 0.36132812, + -0.19628906, + -0.12402344, + 1.3515625, + 0.25585938, + 0.4921875, + -0.29296875, + -0.58984375, + 0.021240234, + -0.044677734, + 0.7578125, + -0.7890625, + 0.10253906, + -0.15820312, + -0.5078125, + -0.39453125, + -0.453125, + 0.35742188, + 0.921875, + 0.44335938, + -0.49804688, + 0.44335938, + 0.31445312, + 0.58984375, + -1.0078125, + -0.22460938, + 0.24121094, + 0.87890625, + 0.66015625, + -0.390625, + -0.05053711, + 0.059570312, + 0.36132812, + -0.00038719177, + -0.017089844, + 0.62890625, + 0.203125, + 0.17480469, + 0.025512695, + 0.47460938, + 0.3125, + 1.140625, + 0.32421875, + -0.057861328, + 0.36914062, + -0.7265625, + -0.51953125, + 0.26953125, + 0.42773438, + 0.064453125, + 0.6328125, + 0.27148438, + -0.11767578, + 0.66796875, + -0.38671875, + 0.5234375, + -0.59375, + 0.5078125, + 0.008239746, + -0.34179688, + -0.27539062, + 0.5234375, + 1.296875, + 0.29492188, + -0.010986328, + -0.41210938, + 0.59375, + 0.061767578, + -0.33398438, + -2.03125, + 0.87890625, + -0.010620117, + 0.53125, + 0.14257812, + -0.515625, + -1.03125, + 0.578125, + 0.1875, + 0.44335938, + -0.33203125, + -0.36328125, + -0.3203125, + 0.29296875, + -0.8203125, + 0.41015625, + -0.48242188, + 0.66015625, + 0.5625, + -0.16503906, + -0.54296875, + -0.38085938, + 0.26171875, + 0.62109375, + 0.29101562, + -0.31054688, + 0.23730469, + -0.8515625, + 0.5234375, + 0.15332031, + 0.52734375, + -0.079589844, + -0.080566406, + -0.15527344, + -0.022827148, + 0.030517578, + -0.1640625, + -0.421875, + 0.09716797, + 0.03930664, + -0.055908203, + -0.546875, + -0.47851562, + 0.091796875, + 0.32226562, + -0.94140625, + -0.04638672, + -1.203125, + -0.39648438, + 0.45507812, + 0.296875, + -0.45703125, + 0.37890625, + -0.122558594, + 0.28320312, + -0.01965332, + -0.11669922, + -0.34570312, + -0.53515625, + -0.091308594, + -0.9375, + -0.32617188, + 0.095214844, + -0.4765625, + 0.37890625, + -0.859375, + 1.1015625, + -0.08935547, + 0.46484375, + -0.19238281, + 0.7109375, + 0.040039062, + -0.5390625, + 0.22363281, + -0.70703125, + 0.4921875, + -0.119140625, + -0.26757812, + -0.08496094, + 0.0859375, + -0.00390625, + -0.013366699, + -0.03955078, + 0.07421875, + -0.13085938, + 0.29101562, + -0.12109375, + 0.45703125, + 0.021728516, + 0.38671875, + -0.3671875, + -0.52734375, + -0.115722656, + 0.125, + 0.5703125, + -1.234375, + 0.06298828, + -0.55859375, + 0.60546875, + 0.8125, + -0.0032958984, + -0.068359375, + -0.21191406, + 0.56640625, + 0.17285156, + -0.3515625, + 0.36328125, + -0.99609375, + 0.43554688, + -0.1015625, + 0.07080078, + -0.66796875, + 1.359375, + 0.41601562, + 0.15917969, + 0.17773438, + -0.28710938, + 0.021850586, + -0.46289062, + 0.17578125, + -0.03955078, + -0.026855469, + 0.5078125, + -0.65625, + 0.0012512207, + 0.044433594, + -0.18652344, + 0.4921875, + -0.75390625, + 0.0072021484, + 0.4375, + -0.31445312, + 0.20214844, + 0.15039062, + -0.63671875, + -0.296875, + -0.375, + -0.027709961, + 0.013427734, + 0.17089844, + 0.89453125, + 0.11621094, + -0.43945312, + -0.30859375, + 0.02709961, + 0.23242188, + -0.64453125, + -0.859375, + 0.22167969, + -0.023071289, + -0.052734375, + 0.3671875, + -0.18359375, + 0.81640625, + -0.11816406, + 0.028320312, + 0.19042969, + 0.012817383, + -0.43164062, + 0.55859375, + -0.27929688, + 0.14257812, + -0.140625, + -0.048583984, + -0.014526367, + 0.35742188, + 0.22753906, + 0.13183594, + 0.04638672, + 0.03930664, + -0.29296875, + -0.2109375, + -0.16308594, + -0.48046875, + -0.13378906, + -0.39257812, + 0.29296875, + -0.047851562, + -0.5546875, + 0.08300781, + -0.14941406, + -0.07080078, + 0.12451172, + 0.1953125, + -0.51171875, + -0.048095703, + 0.1953125, + -0.37695312, + 0.46875, + -0.084472656, + 0.19042969, + -0.39453125, + 0.69921875, + -0.0065307617, + 0.25390625, + -0.16992188, + -0.5078125, + 0.016845703, + 0.27929688, + -0.22070312, + 0.671875, + 0.18652344, + 0.25, + -0.046875, + -0.012023926, + -0.36523438, + 0.36523438, + -0.11279297, + 0.421875, + 0.079589844, + -0.100097656, + 0.37304688, + 0.29882812, + -0.10546875, + -0.36523438, + 0.040039062, + 0.546875, + 0.12890625, + -0.06542969, + -0.38085938, + -0.35742188, + -0.6484375, + -0.28515625, + 0.0107421875, + -0.055664062, + 0.45703125, + 0.33984375, + 0.26367188, + -0.23144531, + 0.012878418, + -0.875, + 0.11035156, + 0.33984375, + 0.203125, + 0.38867188, + 0.24902344, + -0.37304688, + -0.98046875, + -0.122558594, + -0.17871094, + -0.09277344, + 0.1796875, + 0.4453125, + -0.66796875, + 0.78515625, + 0.12988281, + 0.35546875, + 0.44140625, + 0.58984375, + 0.29492188, + 0.7734375, + -0.21972656, + -0.40234375, + -0.22265625, + 0.18359375, + 0.54296875, + 0.17382812, + 0.59375, + -0.390625, + -0.92578125, + -0.017456055, + -0.25, + 0.73828125, + 0.7578125, + -0.3828125, + -0.25976562, + 0.049072266, + 0.046875, + -0.3515625, + 0.30078125, + -1.03125, + -0.48828125, + 0.0017929077, + -0.26171875, + 0.20214844, + 0.29882812, + 0.064941406, + 0.21484375, + -0.55078125, + -0.021362305, + 0.12988281, + 0.27148438, + 0.38867188, + -0.19726562, + -0.55078125, + 0.1640625, + 0.32226562, + -0.72265625, + 0.36132812, + 1.21875, + -0.22070312, + -0.32421875, + -0.29882812, + 0.0024414062, + 0.19921875, + 0.734375, + 0.16210938, + 0.17871094, + -0.19140625, + 0.38476562, + -0.06591797, + -0.47070312, + -0.040039062, + -0.33007812, + -0.07910156, + -0.2890625, + 0.00970459, + 0.12695312, + -0.12060547, + -0.18847656, + 1.015625, + -0.032958984, + 0.12451172, + -0.38476562, + 0.063964844, + 1.0859375, + 0.067871094, + -0.24511719, + 0.125, + 0.10546875, + -0.22460938, + -0.29101562, + 0.24414062, + -0.017944336, + -0.15625, + -0.60546875, + -0.25195312, + -0.46875, + 0.80859375, + -0.34960938, + 0.42382812, + 0.796875, + 0.296875, + -0.067871094, + 0.39453125, + 0.07470703, + 0.033935547, + 0.24414062, + 0.32617188, + 0.023925781, + 0.73046875, + 0.2109375, + -0.43164062, + 0.14453125, + 0.63671875, + 0.21972656, + -0.1875, + -0.18066406, + -0.22167969, + -1.3359375, + 0.52734375, + -0.40625, + -0.12988281, + 0.17480469, + -0.18066406, + 0.58984375, + -0.32421875, + -0.13476562, + 0.39257812, + -0.19238281, + 0.068359375, + 0.7265625, + -0.7109375, + -0.125, + 0.328125, + 0.34179688, + -0.48828125, + -0.10058594, + -0.83984375, + 0.30273438, + 0.008239746, + -1.390625, + 0.171875, + 0.34960938, + 0.44921875, + 0.22167969, + 0.60546875, + -0.36914062, + -0.028808594, + -0.19921875, + 0.6875, + 0.52734375, + -0.07421875, + 0.35546875, + 0.546875, + 0.08691406, + 0.23339844, + -0.984375, + -0.20507812, + 0.08544922, + 0.453125, + -0.07421875, + -0.953125, + 0.74609375, + -0.796875, + 0.47851562, + 0.81640625, + -0.44921875, + -0.33398438, + -0.54296875, + 0.46484375, + -0.390625, + -0.24121094, + -0.0115356445, + 1.1328125, + 1.0390625, + 0.6484375, + 0.35742188, + -0.29492188, + -0.0007095337, + -0.060302734, + 0.21777344, + 0.15136719, + -0.6171875, + 0.11328125, + -0.025878906, + 0.19238281, + 0.140625, + 0.171875, + 0.25195312, + 0.10546875, + 0.0008354187, + -0.13476562, + -0.26953125, + 0.025024414, + -0.28320312, + -0.107910156, + 1.015625, + 0.05493164, + -0.12988281, + 0.30859375, + 0.22558594, + -0.60546875, + 0.11328125, + -1.203125, + 0.6484375, + 0.087402344, + 0.32226562, + 0.63671875, + -0.07714844, + -1.390625, + -0.71875, + -0.34179688, + -0.10546875, + -0.37304688, + -0.09863281, + -0.41210938, + -0.14941406, + 0.41210938, + -0.20898438, + 0.18261719, + 0.67578125, + 0.41601562, + 0.32617188, + 0.2421875, + -0.14257812, + -0.6796875, + 0.01953125, + 0.34179688, + 0.20800781, + -0.123046875, + 0.087402344, + 0.85546875, + 0.33984375, + 0.33203125, + -0.68359375, + 0.44921875, + 0.50390625, + 0.083496094, + 0.10888672, + -0.09863281, + 0.55078125, + 0.09765625, + -0.50390625, + 0.13378906, + -0.29882812, + 0.030761719, + -0.64453125, + 0.22949219, + 0.43945312, + 0.16503906, + 0.10888672, + -0.12792969, + -0.039794922, + -0.111328125, + -0.35742188, + 0.053222656, + -0.78125, + -0.4375, + 0.359375, + -0.88671875, + -0.21972656, + -0.053710938, + 0.91796875, + -0.10644531, + 0.55859375, + -0.7734375, + 0.5078125, + 0.46484375, + 0.32226562, + 0.16796875, + -0.28515625, + 0.045410156, + -0.45117188, + 0.38867188, + -0.33398438, + -0.5234375, + 0.296875, + 0.6015625, + 0.3515625, + -0.734375, + 0.3984375, + -0.08251953, + 0.359375, + -0.28515625, + -0.88671875, + 0.0051879883, + 0.045166016, + -0.7421875, + -0.36523438, + 0.140625, + 0.18066406, + -0.171875, + -0.15625, + -0.53515625, + 0.2421875, + -0.19140625, + -0.18066406, + 0.25390625, + 0.6875, + -0.01965332, + -0.33203125, + 0.29492188, + 0.107421875, + -0.048339844, + -0.82421875, + 0.52734375, + 0.78125, + 0.8203125, + -0.90625, + 0.765625, + 0.0390625, + 0.045410156, + 0.26367188, + -0.14355469, + -0.26367188, + 0.390625, + -0.10888672, + 0.33007812, + -0.5625, + 0.08105469, + -0.13769531, + 0.8515625, + -0.14453125, + 0.77734375, + -0.48046875, + -0.3515625, + -0.25390625, + -0.09277344, + 0.23925781, + -0.022338867, + -0.45898438, + 0.36132812, + -0.23828125, + 0.265625, + -0.48632812, + -0.46875, + -0.75390625, + 1.3125, + 0.78125, + -0.63671875, + -1.21875, + 0.5078125, + -0.27734375, + -0.118652344, + 0.041992188, + -0.14648438, + -0.8046875, + 0.21679688, + -0.79296875, + 0.28320312, + -0.09667969, + 0.42773438, + 0.49414062, + 0.44726562, + 0.21972656, + -0.02746582, + -0.03540039, + -0.14941406, + -0.515625, + -0.27929688, + 0.9609375, + -0.007598877, + 0.34765625, + -0.060546875, + -0.44726562, + 0.7421875, + 0.15332031, + 0.45117188, + -0.4921875, + 0.07080078, + 0.5625, + 0.3984375, + -0.20019531, + 0.014892578, + 0.63671875, + -0.0071411133, + 0.016357422, + 1.0625, + 0.049316406, + 0.18066406, + 0.09814453, + -0.52734375, + -0.359375, + -0.072265625, + -0.41992188, + 0.39648438, + 0.38671875, + -0.30273438, + -0.056640625, + -0.640625, + -0.44921875, + 0.49414062, + 0.29101562, + 0.49609375, + 0.40429688, + -0.10205078, + 0.49414062, + -0.28125, + -0.12695312, + -0.0022735596, + -0.37304688, + 0.122558594, + 0.07519531, + -0.12597656, + -0.38085938, + -0.19824219, + -0.40039062, + 0.56640625, + -1.140625, + -0.515625, + -0.17578125, + -0.765625, + -0.43945312, + 0.3359375, + -0.24707031, + 0.32617188, + -0.45117188, + -0.37109375, + 0.45117188, + -0.27539062, + -0.38867188, + 0.09082031, + 0.17675781, + 0.49414062, + 0.19921875, + 0.17480469, + 0.8515625, + -0.23046875, + -0.234375, + -0.28515625, + 0.10253906, + 0.29101562, + -0.3359375, + -0.203125, + 0.6484375, + 0.11767578, + -0.20214844, + -0.42382812, + 0.26367188, + 0.6328125, + 0.0059509277, + 0.08691406, + -1.5625, + -0.43554688, + 0.17675781, + 0.091796875, + -0.5234375, + -0.09863281, + 0.20605469, + 0.16601562, + -0.578125, + 0.017700195, + 0.41015625, + 1.03125, + -0.55078125, + 0.21289062, + -0.35351562, + 0.24316406, + -0.123535156, + 0.11035156, + -0.48242188, + -0.34179688, + 0.45117188, + 0.3125, + -0.071777344, + 0.12792969, + 0.55859375, + 0.063964844, + -0.21191406, + 0.01965332, + -1.359375, + -0.21582031, + -0.019042969, + 0.16308594, + -0.3671875, + -0.40625, + -1.0234375, + -0.21289062, + 0.24023438, + -0.28125, + 0.26953125, + -0.14550781, + -0.087890625, + 0.16113281, + -0.49804688, + -0.17675781, + -0.890625, + 0.27929688, + 0.484375, + 0.27148438, + 0.11816406, + 0.83984375, + 0.029052734, + -0.890625, + 0.66796875, + 0.78515625, + -0.953125, + 0.49414062, + -0.546875, + 0.106933594, + -0.08251953, + 0.2890625, + -0.1484375, + -0.85546875, + 0.32421875, + -0.0040893555, + -0.16601562, + -0.16699219, + 0.24414062, + -0.5078125, + 0.25390625, + -0.10253906, + 0.15625, + 0.140625, + -0.27539062, + -0.546875, + -0.5546875, + -0.71875, + 0.37304688, + 0.060058594, + -0.076171875, + 0.44921875, + 0.06933594, + -0.28710938, + -0.22949219, + 0.17578125, + 0.09814453, + 0.4765625, + -0.95703125, + -0.03540039, + 0.21289062, + -0.7578125, + -0.07373047, + 0.10546875, + 0.07128906, + 0.76171875, + 0.4296875, + -0.09375, + 0.27539062, + -0.55078125, + 0.29882812, + -0.42382812, + 0.32617188, + -0.39648438, + 0.12451172, + 0.16503906, + -0.22460938, + -0.65625, + -0.022094727, + 0.61328125, + -0.024780273, + 0.62109375, + -0.033447266, + 0.515625, + 0.12890625, + -0.21875, + -0.08642578, + 0.49804688, + -0.2265625, + -0.29296875, + 0.19238281, + 0.3515625, + -1.265625, + 0.57421875, + 0.20117188, + -0.28320312, + 0.1953125, + -0.30664062, + 0.2265625, + -0.11230469, + 0.83984375, + 0.111328125, + 0.265625, + 0.71484375, + -0.625, + 0.38867188, + 0.47070312, + -0.32617188, + -0.171875, + 1.0078125, + 0.19726562, + -0.118652344, + 0.63671875, + -0.068359375, + -0.25585938, + 0.4140625, + -0.29296875, + 0.21386719, + -0.064453125, + 0.15820312, + -0.89453125, + -0.16308594, + 0.48046875, + 0.14648438, + -0.5703125, + 0.84765625, + -0.19042969, + 0.03515625, + 0.42578125, + -0.27539062, + -0.5390625, + 0.95703125, + 0.2734375, + 0.16699219, + -0.328125, + 0.11279297, + 0.003250122, + 0.47265625, + -0.31640625, + 0.546875, + 0.55859375, + 0.06933594, + -0.61328125, + -0.16210938, + -0.375, + 0.100097656, + -0.088378906, + 0.12695312, + 0.079589844, + 0.123535156, + -1.0078125, + 0.6875, + 0.022949219, + -0.40039062, + -0.09863281, + 0.29101562, + -1.2890625, + -0.20996094, + 0.36328125, + -0.3515625, + 0.7890625, + 0.12207031, + 0.48046875, + -0.13671875, + -0.041015625, + 0.19824219, + 0.19921875, + 0.01171875, + -0.37695312, + -0.62890625, + 0.9375, + -0.671875, + 0.24609375, + 0.6484375, + -0.29101562, + 0.076171875, + 0.62109375, + -0.5546875, + 0.36523438, + 0.75390625, + -0.19140625, + -0.875, + -0.8203125, + -0.24414062, + -0.625, + 0.1796875, + -0.40039062, + 0.25390625, + -0.14550781, + -0.21679688, + -0.828125, + 0.3359375, + 0.43554688, + 0.55078125, + -0.44921875, + -0.28710938, + 0.24023438, + 0.18066406, + -0.6953125, + 0.020385742, + -0.11376953, + 0.13867188, + -0.92578125, + 0.33398438, + -0.328125, + 0.78125, + -0.45507812, + -0.07470703, + 0.34179688, + 0.07080078, + 0.76171875, + 0.37890625, + -0.10644531, + 0.90234375, + -0.21875, + -0.15917969, + -0.36132812, + 0.2109375, + -0.45703125, + -0.76953125, + 0.21289062, + 0.26367188, + 0.49804688, + 0.35742188, + -0.20019531, + 0.31054688, + 0.34179688, + 0.17089844, + -0.15429688, + 0.39648438, + -0.5859375, + 0.20996094, + -0.40039062, + 0.5703125, + -0.515625, + 0.5234375, + 0.049560547, + 0.328125, + 0.24804688, + 0.42578125, + 0.609375, + 0.19238281, + 0.27929688, + 0.19335938, + 0.78125, + -0.9921875, + 0.23925781, + -1.3828125, + -0.22949219, + -0.578125, + -0.13964844, + -0.17382812, + -0.011169434, + 0.26171875, + -0.73046875, + -1.4375, + 0.6953125, + -0.7421875, + 0.052246094, + 0.12207031, + 1.3046875, + 0.38867188, + 0.040283203, + -0.546875, + -0.0021514893, + 0.18457031, + -0.5546875, + -0.51171875, + -0.16308594, + -0.104003906, + -0.38867188, + -0.20996094, + -0.8984375, + 0.6015625, + -0.30078125, + -0.13769531, + 0.16113281, + 0.58203125, + -0.23730469, + -0.125, + -1.0234375, + 0.875, + -0.7109375, + 0.29101562, + 0.09667969, + -0.3203125, + -0.48046875, + 0.37890625, + 0.734375, + -0.28710938, + -0.29882812, + -0.05493164, + 0.34765625, + -0.84375, + 0.65625, + 0.578125, + -0.20019531, + 0.13769531, + 0.10058594, + -0.37109375, + 0.36523438, + -0.22167969, + 0.72265625, + ], + "inputTextTokenCount": 6, + }, + ], + "amazon.titan-embed-text-v1::This is an embedding test.": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "11233989-07e8-4ecb-9ba6-79601ba6d8cc"}, + 200, + { + "embedding": [ + -0.14160156, + 0.034423828, + 0.54296875, + 0.10986328, + 0.053466797, + 0.3515625, + 0.12988281, + -0.0002708435, + -0.21484375, + 0.060302734, + 0.58984375, + -0.5859375, + 0.52734375, + 0.82421875, + -0.91015625, + -0.19628906, + 0.45703125, + 0.609375, + -0.67578125, + 0.39453125, + -0.46875, + -0.25390625, + -0.21191406, + 0.114746094, + 0.31640625, + -0.41015625, + -0.32617188, + -0.43554688, + 0.4765625, + -0.4921875, + 0.40429688, + 0.06542969, + 0.859375, + -0.008056641, + -0.19921875, + 0.072753906, + 0.33203125, + 0.69921875, + 0.39453125, + 0.15527344, + 0.08886719, + -0.25, + 0.859375, + 0.22949219, + -0.19042969, + 0.13769531, + -0.078125, + 0.41210938, + 0.875, + 0.5234375, + 0.59765625, + -0.22949219, + -0.22558594, + -0.47460938, + 0.37695312, + 0.51953125, + -0.5703125, + 0.46679688, + 0.43554688, + 0.17480469, + -0.080566406, + -0.16699219, + -0.734375, + -1.0625, + -0.33984375, + 0.390625, + -0.18847656, + -0.5234375, + -0.48828125, + 0.44921875, + -0.09814453, + -0.3359375, + 0.087402344, + 0.36914062, + 1.3203125, + 0.25585938, + 0.14746094, + -0.059570312, + -0.15820312, + -0.037353516, + -0.61328125, + -0.6484375, + -0.35351562, + 0.55078125, + -0.26953125, + 0.90234375, + 0.3671875, + 0.31054688, + 0.00014019012, + -0.171875, + 0.025512695, + 0.5078125, + 0.11621094, + 0.33203125, + 0.8125, + -0.3046875, + -1.078125, + -0.5703125, + 0.26171875, + -0.4609375, + 0.203125, + 0.44726562, + -0.5078125, + 0.41601562, + -0.1953125, + 0.028930664, + -0.57421875, + 0.2265625, + 0.13574219, + -0.040039062, + -0.22949219, + -0.515625, + -0.19042969, + -0.30078125, + 0.10058594, + -0.66796875, + 0.6015625, + 0.296875, + -0.765625, + -0.87109375, + 0.2265625, + 0.068847656, + -0.088378906, + -0.1328125, + -0.796875, + -0.37304688, + 0.47460938, + -0.3515625, + -0.8125, + -0.32226562, + 0.265625, + 0.3203125, + -0.4140625, + -0.49023438, + 0.859375, + -0.19140625, + -0.6328125, + 0.10546875, + -0.5625, + 0.66015625, + 0.26171875, + -0.2109375, + 0.421875, + -0.82421875, + 0.29296875, + 0.17773438, + 0.24023438, + 0.5078125, + -0.49804688, + -0.10205078, + 0.10498047, + -0.36132812, + -0.47460938, + -0.20996094, + 0.010070801, + -0.546875, + 0.66796875, + -0.123046875, + -0.75390625, + 0.19628906, + 0.17480469, + 0.18261719, + -0.96875, + -0.26171875, + 0.4921875, + -0.40039062, + 0.296875, + 0.1640625, + -0.20507812, + -0.36132812, + 0.76171875, + -1.234375, + -0.625, + 0.060058594, + -0.09375, + -0.14746094, + 1.09375, + 0.057861328, + 0.22460938, + -0.703125, + 0.07470703, + 0.23828125, + -0.083984375, + -0.54296875, + 0.5546875, + -0.5, + -0.390625, + 0.106933594, + 0.6640625, + 0.27734375, + -0.953125, + 0.35351562, + -0.7734375, + -0.77734375, + 0.16503906, + -0.42382812, + 0.36914062, + 0.020141602, + -1.3515625, + 0.18847656, + 0.13476562, + -0.034179688, + -0.03930664, + -0.03857422, + -0.027954102, + 0.73828125, + -0.18945312, + -0.09814453, + -0.46289062, + 0.36914062, + 0.033203125, + 0.020874023, + -0.703125, + 0.91796875, + 0.38671875, + 0.625, + -0.19335938, + -0.16796875, + -0.58203125, + 0.21386719, + -0.032470703, + -0.296875, + -0.15625, + -0.1640625, + -0.74609375, + 0.328125, + 0.5546875, + -0.1953125, + 1.0546875, + 0.171875, + -0.099609375, + 0.5234375, + 0.05078125, + -0.35742188, + -0.2734375, + -1.3203125, + -0.8515625, + -0.16015625, + 0.01574707, + 0.29296875, + 0.18457031, + -0.265625, + 0.048339844, + 0.045654297, + -0.32226562, + 0.087890625, + -0.0047302246, + 0.38671875, + 0.10644531, + -0.06225586, + 1.03125, + 0.94140625, + -0.3203125, + 0.20800781, + -1.171875, + 0.48046875, + -0.091796875, + 0.20800781, + -0.1328125, + -0.20507812, + 0.28125, + -0.47070312, + -0.09033203, + 0.0013809204, + -0.08203125, + 0.43359375, + -0.03100586, + -0.060791016, + -0.53515625, + -1.46875, + 0.000101566315, + 0.515625, + 0.40625, + -0.10498047, + -0.15820312, + -0.009460449, + -0.77734375, + -0.5859375, + 0.9765625, + 0.099609375, + 0.51953125, + 0.38085938, + -0.09667969, + -0.100097656, + -0.5, + -1.3125, + -0.18066406, + -0.099121094, + 0.26171875, + -0.14453125, + -0.546875, + 0.17578125, + 0.484375, + 0.765625, + 0.45703125, + 0.2734375, + 0.0028076172, + 0.17089844, + -0.32421875, + -0.37695312, + 0.30664062, + -0.48046875, + 0.07128906, + 0.031982422, + -0.31054688, + -0.055419922, + -0.29296875, + 0.3359375, + -0.296875, + 0.47851562, + -0.05126953, + 0.18457031, + -0.01953125, + -0.35742188, + 0.017944336, + -0.25, + 0.10595703, + 0.17382812, + -0.73828125, + 0.36914062, + -0.15234375, + -0.8125, + 0.17382812, + 0.048095703, + 0.5625, + -0.33789062, + 0.023071289, + -0.21972656, + 0.16015625, + 0.032958984, + -1.1171875, + -0.984375, + 0.83984375, + 0.009033203, + -0.042236328, + -0.46484375, + -0.08203125, + 0.44726562, + -0.765625, + -0.3984375, + -0.40820312, + -0.234375, + 0.044189453, + 0.119628906, + -0.7578125, + -0.55078125, + -0.4453125, + 0.7578125, + 0.34960938, + 0.96484375, + 0.35742188, + 0.36914062, + -0.35351562, + -0.36132812, + 1.109375, + 0.5859375, + 0.85546875, + -0.10644531, + -0.6953125, + -0.0066833496, + 0.042236328, + -0.06689453, + 0.36914062, + 0.9765625, + -0.3046875, + 0.59765625, + -0.6640625, + 0.21484375, + -0.07128906, + 1.1328125, + -0.51953125, + 0.86328125, + -0.11328125, + 0.15722656, + -0.36328125, + -0.04638672, + 1.4375, + 0.18457031, + -0.18359375, + 0.10595703, + -0.49023438, + -0.07324219, + -0.73046875, + -0.119140625, + 0.021118164, + 0.4921875, + -0.46875, + 0.28710938, + 0.3359375, + 0.11767578, + -0.2109375, + -0.14550781, + 0.39648438, + -0.27734375, + 0.48046875, + 0.12988281, + 0.45507812, + -0.375, + -0.84765625, + 0.25585938, + -0.36523438, + 0.8046875, + 0.42382812, + -0.24511719, + 0.54296875, + 0.71875, + 0.010009766, + -0.04296875, + 0.083984375, + -0.52734375, + 0.13964844, + -0.27539062, + -0.30273438, + 1.1484375, + -0.515625, + -0.19335938, + 0.58984375, + 0.049072266, + 0.703125, + -0.04272461, + 0.5078125, + 0.34960938, + -0.3359375, + -0.47460938, + 0.049316406, + 0.36523438, + 0.7578125, + -0.022827148, + -0.71484375, + 0.21972656, + 0.09716797, + -0.203125, + -0.36914062, + 1.34375, + 0.34179688, + 0.46679688, + 1.078125, + 0.26171875, + 0.41992188, + 0.22363281, + -0.515625, + -0.5703125, + 0.13378906, + 0.26757812, + -0.22558594, + -0.5234375, + 0.06689453, + 0.08251953, + -0.625, + 0.16796875, + 0.43164062, + -0.55859375, + 0.28125, + 0.078125, + 0.6328125, + 0.23242188, + -0.064941406, + -0.004486084, + -0.20703125, + 0.2734375, + 0.453125, + -0.734375, + 0.04272461, + 0.36132812, + -0.19628906, + -0.12402344, + 1.3515625, + 0.25585938, + 0.4921875, + -0.29296875, + -0.58984375, + 0.021240234, + -0.044677734, + 0.7578125, + -0.7890625, + 0.10253906, + -0.15820312, + -0.5078125, + -0.39453125, + -0.453125, + 0.35742188, + 0.921875, + 0.44335938, + -0.49804688, + 0.44335938, + 0.31445312, + 0.58984375, + -1.0078125, + -0.22460938, + 0.24121094, + 0.87890625, + 0.66015625, + -0.390625, + -0.05053711, + 0.059570312, + 0.36132812, + -0.00038719177, + -0.017089844, + 0.62890625, + 0.203125, + 0.17480469, + 0.025512695, + 0.47460938, + 0.3125, + 1.140625, + 0.32421875, + -0.057861328, + 0.36914062, + -0.7265625, + -0.51953125, + 0.26953125, + 0.42773438, + 0.064453125, + 0.6328125, + 0.27148438, + -0.11767578, + 0.66796875, + -0.38671875, + 0.5234375, + -0.59375, + 0.5078125, + 0.008239746, + -0.34179688, + -0.27539062, + 0.5234375, + 1.296875, + 0.29492188, + -0.010986328, + -0.41210938, + 0.59375, + 0.061767578, + -0.33398438, + -2.03125, + 0.87890625, + -0.010620117, + 0.53125, + 0.14257812, + -0.515625, + -1.03125, + 0.578125, + 0.1875, + 0.44335938, + -0.33203125, + -0.36328125, + -0.3203125, + 0.29296875, + -0.8203125, + 0.41015625, + -0.48242188, + 0.66015625, + 0.5625, + -0.16503906, + -0.54296875, + -0.38085938, + 0.26171875, + 0.62109375, + 0.29101562, + -0.31054688, + 0.23730469, + -0.8515625, + 0.5234375, + 0.15332031, + 0.52734375, + -0.079589844, + -0.080566406, + -0.15527344, + -0.022827148, + 0.030517578, + -0.1640625, + -0.421875, + 0.09716797, + 0.03930664, + -0.055908203, + -0.546875, + -0.47851562, + 0.091796875, + 0.32226562, + -0.94140625, + -0.04638672, + -1.203125, + -0.39648438, + 0.45507812, + 0.296875, + -0.45703125, + 0.37890625, + -0.122558594, + 0.28320312, + -0.01965332, + -0.11669922, + -0.34570312, + -0.53515625, + -0.091308594, + -0.9375, + -0.32617188, + 0.095214844, + -0.4765625, + 0.37890625, + -0.859375, + 1.1015625, + -0.08935547, + 0.46484375, + -0.19238281, + 0.7109375, + 0.040039062, + -0.5390625, + 0.22363281, + -0.70703125, + 0.4921875, + -0.119140625, + -0.26757812, + -0.08496094, + 0.0859375, + -0.00390625, + -0.013366699, + -0.03955078, + 0.07421875, + -0.13085938, + 0.29101562, + -0.12109375, + 0.45703125, + 0.021728516, + 0.38671875, + -0.3671875, + -0.52734375, + -0.115722656, + 0.125, + 0.5703125, + -1.234375, + 0.06298828, + -0.55859375, + 0.60546875, + 0.8125, + -0.0032958984, + -0.068359375, + -0.21191406, + 0.56640625, + 0.17285156, + -0.3515625, + 0.36328125, + -0.99609375, + 0.43554688, + -0.1015625, + 0.07080078, + -0.66796875, + 1.359375, + 0.41601562, + 0.15917969, + 0.17773438, + -0.28710938, + 0.021850586, + -0.46289062, + 0.17578125, + -0.03955078, + -0.026855469, + 0.5078125, + -0.65625, + 0.0012512207, + 0.044433594, + -0.18652344, + 0.4921875, + -0.75390625, + 0.0072021484, + 0.4375, + -0.31445312, + 0.20214844, + 0.15039062, + -0.63671875, + -0.296875, + -0.375, + -0.027709961, + 0.013427734, + 0.17089844, + 0.89453125, + 0.11621094, + -0.43945312, + -0.30859375, + 0.02709961, + 0.23242188, + -0.64453125, + -0.859375, + 0.22167969, + -0.023071289, + -0.052734375, + 0.3671875, + -0.18359375, + 0.81640625, + -0.11816406, + 0.028320312, + 0.19042969, + 0.012817383, + -0.43164062, + 0.55859375, + -0.27929688, + 0.14257812, + -0.140625, + -0.048583984, + -0.014526367, + 0.35742188, + 0.22753906, + 0.13183594, + 0.04638672, + 0.03930664, + -0.29296875, + -0.2109375, + -0.16308594, + -0.48046875, + -0.13378906, + -0.39257812, + 0.29296875, + -0.047851562, + -0.5546875, + 0.08300781, + -0.14941406, + -0.07080078, + 0.12451172, + 0.1953125, + -0.51171875, + -0.048095703, + 0.1953125, + -0.37695312, + 0.46875, + -0.084472656, + 0.19042969, + -0.39453125, + 0.69921875, + -0.0065307617, + 0.25390625, + -0.16992188, + -0.5078125, + 0.016845703, + 0.27929688, + -0.22070312, + 0.671875, + 0.18652344, + 0.25, + -0.046875, + -0.012023926, + -0.36523438, + 0.36523438, + -0.11279297, + 0.421875, + 0.079589844, + -0.100097656, + 0.37304688, + 0.29882812, + -0.10546875, + -0.36523438, + 0.040039062, + 0.546875, + 0.12890625, + -0.06542969, + -0.38085938, + -0.35742188, + -0.6484375, + -0.28515625, + 0.0107421875, + -0.055664062, + 0.45703125, + 0.33984375, + 0.26367188, + -0.23144531, + 0.012878418, + -0.875, + 0.11035156, + 0.33984375, + 0.203125, + 0.38867188, + 0.24902344, + -0.37304688, + -0.98046875, + -0.122558594, + -0.17871094, + -0.09277344, + 0.1796875, + 0.4453125, + -0.66796875, + 0.78515625, + 0.12988281, + 0.35546875, + 0.44140625, + 0.58984375, + 0.29492188, + 0.7734375, + -0.21972656, + -0.40234375, + -0.22265625, + 0.18359375, + 0.54296875, + 0.17382812, + 0.59375, + -0.390625, + -0.92578125, + -0.017456055, + -0.25, + 0.73828125, + 0.7578125, + -0.3828125, + -0.25976562, + 0.049072266, + 0.046875, + -0.3515625, + 0.30078125, + -1.03125, + -0.48828125, + 0.0017929077, + -0.26171875, + 0.20214844, + 0.29882812, + 0.064941406, + 0.21484375, + -0.55078125, + -0.021362305, + 0.12988281, + 0.27148438, + 0.38867188, + -0.19726562, + -0.55078125, + 0.1640625, + 0.32226562, + -0.72265625, + 0.36132812, + 1.21875, + -0.22070312, + -0.32421875, + -0.29882812, + 0.0024414062, + 0.19921875, + 0.734375, + 0.16210938, + 0.17871094, + -0.19140625, + 0.38476562, + -0.06591797, + -0.47070312, + -0.040039062, + -0.33007812, + -0.07910156, + -0.2890625, + 0.00970459, + 0.12695312, + -0.12060547, + -0.18847656, + 1.015625, + -0.032958984, + 0.12451172, + -0.38476562, + 0.063964844, + 1.0859375, + 0.067871094, + -0.24511719, + 0.125, + 0.10546875, + -0.22460938, + -0.29101562, + 0.24414062, + -0.017944336, + -0.15625, + -0.60546875, + -0.25195312, + -0.46875, + 0.80859375, + -0.34960938, + 0.42382812, + 0.796875, + 0.296875, + -0.067871094, + 0.39453125, + 0.07470703, + 0.033935547, + 0.24414062, + 0.32617188, + 0.023925781, + 0.73046875, + 0.2109375, + -0.43164062, + 0.14453125, + 0.63671875, + 0.21972656, + -0.1875, + -0.18066406, + -0.22167969, + -1.3359375, + 0.52734375, + -0.40625, + -0.12988281, + 0.17480469, + -0.18066406, + 0.58984375, + -0.32421875, + -0.13476562, + 0.39257812, + -0.19238281, + 0.068359375, + 0.7265625, + -0.7109375, + -0.125, + 0.328125, + 0.34179688, + -0.48828125, + -0.10058594, + -0.83984375, + 0.30273438, + 0.008239746, + -1.390625, + 0.171875, + 0.34960938, + 0.44921875, + 0.22167969, + 0.60546875, + -0.36914062, + -0.028808594, + -0.19921875, + 0.6875, + 0.52734375, + -0.07421875, + 0.35546875, + 0.546875, + 0.08691406, + 0.23339844, + -0.984375, + -0.20507812, + 0.08544922, + 0.453125, + -0.07421875, + -0.953125, + 0.74609375, + -0.796875, + 0.47851562, + 0.81640625, + -0.44921875, + -0.33398438, + -0.54296875, + 0.46484375, + -0.390625, + -0.24121094, + -0.0115356445, + 1.1328125, + 1.0390625, + 0.6484375, + 0.35742188, + -0.29492188, + -0.0007095337, + -0.060302734, + 0.21777344, + 0.15136719, + -0.6171875, + 0.11328125, + -0.025878906, + 0.19238281, + 0.140625, + 0.171875, + 0.25195312, + 0.10546875, + 0.0008354187, + -0.13476562, + -0.26953125, + 0.025024414, + -0.28320312, + -0.107910156, + 1.015625, + 0.05493164, + -0.12988281, + 0.30859375, + 0.22558594, + -0.60546875, + 0.11328125, + -1.203125, + 0.6484375, + 0.087402344, + 0.32226562, + 0.63671875, + -0.07714844, + -1.390625, + -0.71875, + -0.34179688, + -0.10546875, + -0.37304688, + -0.09863281, + -0.41210938, + -0.14941406, + 0.41210938, + -0.20898438, + 0.18261719, + 0.67578125, + 0.41601562, + 0.32617188, + 0.2421875, + -0.14257812, + -0.6796875, + 0.01953125, + 0.34179688, + 0.20800781, + -0.123046875, + 0.087402344, + 0.85546875, + 0.33984375, + 0.33203125, + -0.68359375, + 0.44921875, + 0.50390625, + 0.083496094, + 0.10888672, + -0.09863281, + 0.55078125, + 0.09765625, + -0.50390625, + 0.13378906, + -0.29882812, + 0.030761719, + -0.64453125, + 0.22949219, + 0.43945312, + 0.16503906, + 0.10888672, + -0.12792969, + -0.039794922, + -0.111328125, + -0.35742188, + 0.053222656, + -0.78125, + -0.4375, + 0.359375, + -0.88671875, + -0.21972656, + -0.053710938, + 0.91796875, + -0.10644531, + 0.55859375, + -0.7734375, + 0.5078125, + 0.46484375, + 0.32226562, + 0.16796875, + -0.28515625, + 0.045410156, + -0.45117188, + 0.38867188, + -0.33398438, + -0.5234375, + 0.296875, + 0.6015625, + 0.3515625, + -0.734375, + 0.3984375, + -0.08251953, + 0.359375, + -0.28515625, + -0.88671875, + 0.0051879883, + 0.045166016, + -0.7421875, + -0.36523438, + 0.140625, + 0.18066406, + -0.171875, + -0.15625, + -0.53515625, + 0.2421875, + -0.19140625, + -0.18066406, + 0.25390625, + 0.6875, + -0.01965332, + -0.33203125, + 0.29492188, + 0.107421875, + -0.048339844, + -0.82421875, + 0.52734375, + 0.78125, + 0.8203125, + -0.90625, + 0.765625, + 0.0390625, + 0.045410156, + 0.26367188, + -0.14355469, + -0.26367188, + 0.390625, + -0.10888672, + 0.33007812, + -0.5625, + 0.08105469, + -0.13769531, + 0.8515625, + -0.14453125, + 0.77734375, + -0.48046875, + -0.3515625, + -0.25390625, + -0.09277344, + 0.23925781, + -0.022338867, + -0.45898438, + 0.36132812, + -0.23828125, + 0.265625, + -0.48632812, + -0.46875, + -0.75390625, + 1.3125, + 0.78125, + -0.63671875, + -1.21875, + 0.5078125, + -0.27734375, + -0.118652344, + 0.041992188, + -0.14648438, + -0.8046875, + 0.21679688, + -0.79296875, + 0.28320312, + -0.09667969, + 0.42773438, + 0.49414062, + 0.44726562, + 0.21972656, + -0.02746582, + -0.03540039, + -0.14941406, + -0.515625, + -0.27929688, + 0.9609375, + -0.007598877, + 0.34765625, + -0.060546875, + -0.44726562, + 0.7421875, + 0.15332031, + 0.45117188, + -0.4921875, + 0.07080078, + 0.5625, + 0.3984375, + -0.20019531, + 0.014892578, + 0.63671875, + -0.0071411133, + 0.016357422, + 1.0625, + 0.049316406, + 0.18066406, + 0.09814453, + -0.52734375, + -0.359375, + -0.072265625, + -0.41992188, + 0.39648438, + 0.38671875, + -0.30273438, + -0.056640625, + -0.640625, + -0.44921875, + 0.49414062, + 0.29101562, + 0.49609375, + 0.40429688, + -0.10205078, + 0.49414062, + -0.28125, + -0.12695312, + -0.0022735596, + -0.37304688, + 0.122558594, + 0.07519531, + -0.12597656, + -0.38085938, + -0.19824219, + -0.40039062, + 0.56640625, + -1.140625, + -0.515625, + -0.17578125, + -0.765625, + -0.43945312, + 0.3359375, + -0.24707031, + 0.32617188, + -0.45117188, + -0.37109375, + 0.45117188, + -0.27539062, + -0.38867188, + 0.09082031, + 0.17675781, + 0.49414062, + 0.19921875, + 0.17480469, + 0.8515625, + -0.23046875, + -0.234375, + -0.28515625, + 0.10253906, + 0.29101562, + -0.3359375, + -0.203125, + 0.6484375, + 0.11767578, + -0.20214844, + -0.42382812, + 0.26367188, + 0.6328125, + 0.0059509277, + 0.08691406, + -1.5625, + -0.43554688, + 0.17675781, + 0.091796875, + -0.5234375, + -0.09863281, + 0.20605469, + 0.16601562, + -0.578125, + 0.017700195, + 0.41015625, + 1.03125, + -0.55078125, + 0.21289062, + -0.35351562, + 0.24316406, + -0.123535156, + 0.11035156, + -0.48242188, + -0.34179688, + 0.45117188, + 0.3125, + -0.071777344, + 0.12792969, + 0.55859375, + 0.063964844, + -0.21191406, + 0.01965332, + -1.359375, + -0.21582031, + -0.019042969, + 0.16308594, + -0.3671875, + -0.40625, + -1.0234375, + -0.21289062, + 0.24023438, + -0.28125, + 0.26953125, + -0.14550781, + -0.087890625, + 0.16113281, + -0.49804688, + -0.17675781, + -0.890625, + 0.27929688, + 0.484375, + 0.27148438, + 0.11816406, + 0.83984375, + 0.029052734, + -0.890625, + 0.66796875, + 0.78515625, + -0.953125, + 0.49414062, + -0.546875, + 0.106933594, + -0.08251953, + 0.2890625, + -0.1484375, + -0.85546875, + 0.32421875, + -0.0040893555, + -0.16601562, + -0.16699219, + 0.24414062, + -0.5078125, + 0.25390625, + -0.10253906, + 0.15625, + 0.140625, + -0.27539062, + -0.546875, + -0.5546875, + -0.71875, + 0.37304688, + 0.060058594, + -0.076171875, + 0.44921875, + 0.06933594, + -0.28710938, + -0.22949219, + 0.17578125, + 0.09814453, + 0.4765625, + -0.95703125, + -0.03540039, + 0.21289062, + -0.7578125, + -0.07373047, + 0.10546875, + 0.07128906, + 0.76171875, + 0.4296875, + -0.09375, + 0.27539062, + -0.55078125, + 0.29882812, + -0.42382812, + 0.32617188, + -0.39648438, + 0.12451172, + 0.16503906, + -0.22460938, + -0.65625, + -0.022094727, + 0.61328125, + -0.024780273, + 0.62109375, + -0.033447266, + 0.515625, + 0.12890625, + -0.21875, + -0.08642578, + 0.49804688, + -0.2265625, + -0.29296875, + 0.19238281, + 0.3515625, + -1.265625, + 0.57421875, + 0.20117188, + -0.28320312, + 0.1953125, + -0.30664062, + 0.2265625, + -0.11230469, + 0.83984375, + 0.111328125, + 0.265625, + 0.71484375, + -0.625, + 0.38867188, + 0.47070312, + -0.32617188, + -0.171875, + 1.0078125, + 0.19726562, + -0.118652344, + 0.63671875, + -0.068359375, + -0.25585938, + 0.4140625, + -0.29296875, + 0.21386719, + -0.064453125, + 0.15820312, + -0.89453125, + -0.16308594, + 0.48046875, + 0.14648438, + -0.5703125, + 0.84765625, + -0.19042969, + 0.03515625, + 0.42578125, + -0.27539062, + -0.5390625, + 0.95703125, + 0.2734375, + 0.16699219, + -0.328125, + 0.11279297, + 0.003250122, + 0.47265625, + -0.31640625, + 0.546875, + 0.55859375, + 0.06933594, + -0.61328125, + -0.16210938, + -0.375, + 0.100097656, + -0.088378906, + 0.12695312, + 0.079589844, + 0.123535156, + -1.0078125, + 0.6875, + 0.022949219, + -0.40039062, + -0.09863281, + 0.29101562, + -1.2890625, + -0.20996094, + 0.36328125, + -0.3515625, + 0.7890625, + 0.12207031, + 0.48046875, + -0.13671875, + -0.041015625, + 0.19824219, + 0.19921875, + 0.01171875, + -0.37695312, + -0.62890625, + 0.9375, + -0.671875, + 0.24609375, + 0.6484375, + -0.29101562, + 0.076171875, + 0.62109375, + -0.5546875, + 0.36523438, + 0.75390625, + -0.19140625, + -0.875, + -0.8203125, + -0.24414062, + -0.625, + 0.1796875, + -0.40039062, + 0.25390625, + -0.14550781, + -0.21679688, + -0.828125, + 0.3359375, + 0.43554688, + 0.55078125, + -0.44921875, + -0.28710938, + 0.24023438, + 0.18066406, + -0.6953125, + 0.020385742, + -0.11376953, + 0.13867188, + -0.92578125, + 0.33398438, + -0.328125, + 0.78125, + -0.45507812, + -0.07470703, + 0.34179688, + 0.07080078, + 0.76171875, + 0.37890625, + -0.10644531, + 0.90234375, + -0.21875, + -0.15917969, + -0.36132812, + 0.2109375, + -0.45703125, + -0.76953125, + 0.21289062, + 0.26367188, + 0.49804688, + 0.35742188, + -0.20019531, + 0.31054688, + 0.34179688, + 0.17089844, + -0.15429688, + 0.39648438, + -0.5859375, + 0.20996094, + -0.40039062, + 0.5703125, + -0.515625, + 0.5234375, + 0.049560547, + 0.328125, + 0.24804688, + 0.42578125, + 0.609375, + 0.19238281, + 0.27929688, + 0.19335938, + 0.78125, + -0.9921875, + 0.23925781, + -1.3828125, + -0.22949219, + -0.578125, + -0.13964844, + -0.17382812, + -0.011169434, + 0.26171875, + -0.73046875, + -1.4375, + 0.6953125, + -0.7421875, + 0.052246094, + 0.12207031, + 1.3046875, + 0.38867188, + 0.040283203, + -0.546875, + -0.0021514893, + 0.18457031, + -0.5546875, + -0.51171875, + -0.16308594, + -0.104003906, + -0.38867188, + -0.20996094, + -0.8984375, + 0.6015625, + -0.30078125, + -0.13769531, + 0.16113281, + 0.58203125, + -0.23730469, + -0.125, + -1.0234375, + 0.875, + -0.7109375, + 0.29101562, + 0.09667969, + -0.3203125, + -0.48046875, + 0.37890625, + 0.734375, + -0.28710938, + -0.29882812, + -0.05493164, + 0.34765625, + -0.84375, + 0.65625, + 0.578125, + -0.20019531, + 0.13769531, + 0.10058594, + -0.37109375, + 0.36523438, + -0.22167969, + 0.72265625, + ], + "inputTextTokenCount": 6, + }, + ], + "cohere.embed-english-v3::This is an embedding test.": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "11233989-07e8-4ecb-9ba6-79601ba6d8cc"}, + 200, + { + "embeddings": [ + [ + 0.03390503, + 0.010032654, + 0.020904541, + -0.017105103, + -0.020050049, + -0.015411377, + -0.012001038, + -0.019744873, + -0.0107803345, + 0.012702942, + -0.043273926, + 0.003583908, + -0.023422241, + -0.017440796, + 0.03277588, + -0.04321289, + 0.018661499, + 0.05697632, + 0.041107178, + -0.007549286, + -0.026916504, + 0.012214661, + 0.00012886524, + -0.03010559, + -0.011680603, + -0.008392334, + 0.06222534, + -0.026260376, + 0.026947021, + -0.018692017, + 0.0016307831, + -0.018798828, + -0.00025224686, + 0.03982544, + -0.017501831, + 0.03665161, + -0.040985107, + 0.029296875, + 0.025283813, + -0.013618469, + 0.0038471222, + -0.007751465, + 0.017501831, + -0.03756714, + -0.012863159, + -0.03781128, + 0.043945312, + -0.018112183, + 0.01713562, + -0.04736328, + 0.005264282, + 0.0018386841, + -0.0018186569, + 0.016540527, + -0.012268066, + 0.061462402, + 0.006629944, + 0.02671814, + 0.039733887, + -0.0026226044, + -0.037628174, + 0.007675171, + 0.02418518, + -0.039855957, + 0.016586304, + -0.034820557, + -0.016113281, + 0.03262329, + 0.07537842, + -0.00554657, + -0.014450073, + 0.013061523, + 0.0056991577, + 0.007396698, + -0.055114746, + 0.032684326, + 0.0051460266, + 0.046173096, + 0.009277344, + -0.052337646, + -0.022872925, + 0.013946533, + -0.03643799, + 0.004562378, + -0.018218994, + 0.019851685, + 0.0038719177, + -0.032958984, + 0.04147339, + -0.016616821, + -0.010231018, + 0.099121094, + 0.0015497208, + 0.06021118, + -0.045562744, + 0.010559082, + 0.021408081, + -0.009635925, + -0.04067993, + -0.03060913, + -0.020248413, + -0.049346924, + 0.017562866, + -0.019943237, + 0.008331299, + -0.027755737, + -0.016693115, + -0.052368164, + 0.044647217, + 0.027954102, + -0.034332275, + 0.09460449, + -0.013832092, + -0.022888184, + 0.0033721924, + 0.015457153, + -0.03062439, + 0.026794434, + 0.0104904175, + -0.018737793, + 0.0060920715, + 0.027297974, + 0.027786255, + -0.016799927, + 0.022872925, + 0.043640137, + 0.0036582947, + -0.05267334, + 0.010169983, + -0.030258179, + -0.021530151, + 0.0496521, + -0.05883789, + -0.03439331, + -0.017944336, + -0.006149292, + -0.066223145, + -0.017593384, + 0.0016317368, + -0.0027751923, + -0.0028533936, + -0.057434082, + 0.04800415, + 0.01374054, + -0.06756592, + 0.087768555, + 0.04269409, + -0.032226562, + -0.04321289, + -0.08850098, + -0.022201538, + 0.0009198189, + -0.0046043396, + 0.029724121, + 0.01979065, + 0.03753662, + -0.05343628, + 0.033111572, + 0.034332275, + 0.071777344, + 0.0063934326, + -0.034606934, + -0.003545761, + 0.008972168, + 0.008232117, + 0.033447266, + 0.015823364, + 0.027297974, + 0.018981934, + 0.120910645, + -0.037872314, + 0.0038814545, + -0.0031642914, + 0.0071029663, + -0.022644043, + -0.008758545, + 0.0023460388, + -0.025802612, + 0.034332275, + -0.0021533966, + 0.02268982, + -0.008506775, + 0.010147095, + 0.022827148, + 0.0007414818, + -0.055999756, + 0.03237915, + -0.01083374, + -0.014343262, + 0.028793335, + 0.0068511963, + 0.031402588, + 0.023269653, + -0.013748169, + 0.00014042854, + 0.0007624626, + -0.03111267, + 0.007549286, + -0.0236969, + -0.00043439865, + -0.0058670044, + 0.013587952, + -0.0029067993, + -0.0052948, + 0.015701294, + -0.005924225, + 0.032104492, + -0.0017576218, + 0.052947998, + 0.011299133, + -0.03152466, + -0.027526855, + 0.031051636, + -0.04232788, + -0.048217773, + 0.04055786, + -0.038482666, + -0.06088257, + -0.016540527, + -0.027114868, + 0.008636475, + 0.06008911, + -0.038513184, + 0.023330688, + 0.0054473877, + 0.018325806, + -0.017288208, + -7.2062016e-05, + 0.0064430237, + 0.02357483, + 0.02166748, + -0.043060303, + -0.009613037, + 0.013504028, + -0.010856628, + -0.018585205, + -0.00041294098, + -0.012687683, + 0.019302368, + 0.03250122, + 0.03503418, + 0.037353516, + -0.01272583, + -0.039215088, + 0.05230713, + 0.008918762, + 0.020614624, + -0.012039185, + -0.041534424, + 0.0317688, + 0.012168884, + -0.0027694702, + -0.023773193, + 0.0068855286, + -0.04309082, + 0.034820557, + 0.018463135, + 0.048736572, + -0.0016841888, + 0.032836914, + -0.070617676, + 0.04473877, + 0.052581787, + -0.042114258, + -0.017456055, + -0.03945923, + -0.0040626526, + 0.016433716, + 0.02368164, + -0.04034424, + 0.006038666, + 0.005634308, + -5.722046e-06, + -0.01864624, + 0.0635376, + -0.041229248, + -0.026809692, + -0.009262085, + 0.0011701584, + -0.0053367615, + 0.020935059, + 0.04473877, + 0.03665161, + 0.01121521, + 0.017486572, + 0.061920166, + 0.020812988, + 0.013786316, + 0.0006785393, + 0.0027122498, + 0.012237549, + 0.07446289, + -0.021011353, + 0.06921387, + 0.046966553, + 0.028945923, + 0.00044202805, + 0.03488159, + 0.0034942627, + -0.0038585663, + -0.023269653, + 0.04852295, + -0.01525116, + 0.032836914, + 0.013153076, + 0.0014123917, + -0.005718231, + 0.038024902, + 0.015182495, + -0.0143585205, + -0.008659363, + 0.024093628, + 0.008972168, + -0.011962891, + 0.005367279, + -0.027297974, + 0.02696228, + -0.0063972473, + -0.008087158, + -0.015899658, + 0.07122803, + 0.0463562, + -0.06713867, + -0.02230835, + 0.011940002, + -0.0015964508, + -0.049438477, + -0.04864502, + -0.06262207, + -0.015029907, + -0.0049057007, + -0.084472656, + -0.011177063, + -0.031555176, + 0.0035552979, + -0.028427124, + 0.021759033, + 0.016174316, + 0.0390625, + 0.04168701, + 0.07714844, + -0.0064086914, + 0.013000488, + -0.011512756, + -0.0021686554, + -0.032196045, + -0.057678223, + 0.010017395, + -0.06793213, + -0.04220581, + 0.06793213, + -0.029144287, + -0.02229309, + 0.03074646, + 0.03265381, + -0.020050049, + 0.021911621, + 0.055236816, + 0.05480957, + -0.015823364, + 0.04815674, + 0.009384155, + 0.024383545, + -0.034484863, + -0.042114258, + -0.06744385, + -0.011207581, + 0.010749817, + 0.005012512, + 0.029510498, + 0.04977417, + 0.0070648193, + 0.00050497055, + -0.005710602, + -0.063964844, + -0.030807495, + -0.0013856888, + -0.026794434, + -0.024383545, + -0.025817871, + 0.00945282, + -0.008171082, + 0.071777344, + -0.018493652, + 0.041778564, + 0.0012550354, + 0.024902344, + 0.07366943, + 0.02381897, + -0.0016851425, + -0.015945435, + 0.035461426, + -0.038391113, + -0.02961731, + 0.020401001, + 0.0063171387, + 0.035308838, + 0.016586304, + -0.036590576, + -0.04522705, + 0.046722412, + -0.04901123, + -0.028076172, + -0.025787354, + -0.022567749, + -0.00843811, + 0.03778076, + 0.00020611286, + -0.006668091, + 0.027648926, + 0.027008057, + -0.011711121, + -0.0019445419, + 0.030456543, + 0.0038223267, + -0.037872314, + -0.019805908, + 0.017333984, + -0.023986816, + 0.0012874603, + -0.0053596497, + 0.02305603, + -0.03012085, + -0.013389587, + 0.016159058, + 0.020629883, + 0.04159546, + -0.008338928, + 0.029571533, + -0.0005707741, + 0.0231781, + -0.040863037, + -0.012886047, + -0.011627197, + -0.0574646, + 0.0011692047, + -0.0060691833, + -0.010749817, + 0.03567505, + -0.051757812, + 0.009735107, + 0.016159058, + 0.037139893, + -0.013214111, + 0.013938904, + -0.025482178, + 0.04647827, + 0.016418457, + -0.018936157, + 0.040008545, + -0.054595947, + 0.007865906, + -0.022872925, + 0.02508545, + -0.033935547, + 0.004310608, + 0.027008057, + 0.03010559, + 0.020736694, + 0.020111084, + 0.037719727, + -0.015487671, + -0.04598999, + 0.016189575, + -0.009643555, + 0.022399902, + 0.027786255, + 0.013580322, + -0.013595581, + -0.004825592, + 0.039855957, + -0.05834961, + -0.016906738, + -0.016235352, + -0.01826477, + -0.0053520203, + 0.031402588, + 0.023986816, + -0.012367249, + -0.02835083, + -0.004310608, + -0.025115967, + -0.05899048, + -0.036987305, + 0.01574707, + -0.007926941, + -0.030853271, + 0.04458618, + 0.00818634, + 0.017059326, + 4.7802925e-05, + 0.0062294006, + 0.028930664, + -0.027618408, + -0.013557434, + -0.0093307495, + -0.012741089, + -0.009307861, + 0.0032444, + -0.09460449, + -0.0552063, + 0.034576416, + 0.02178955, + 0.024612427, + 0.013587952, + -0.041656494, + -0.029647827, + 0.010848999, + 0.045959473, + -0.001698494, + 0.031341553, + 0.016693115, + 0.027145386, + -0.029541016, + -0.011222839, + 0.08703613, + -0.017303467, + -0.009376526, + 0.025436401, + -0.020217896, + 0.06939697, + 0.023651123, + 0.05065918, + -0.010749817, + -8.738041e-05, + 0.019195557, + 0.024917603, + -0.009590149, + -0.033172607, + -0.025314331, + 0.0049819946, + -0.0070266724, + 0.019622803, + -0.023605347, + 0.030258179, + 0.03869629, + -0.036834717, + -0.0025596619, + 0.007320404, + -0.021438599, + -0.0044021606, + -0.0052604675, + -0.050109863, + 0.0051498413, + -0.011734009, + -0.027770996, + -0.0043258667, + 0.07495117, + 0.007820129, + 0.03930664, + 0.058563232, + -0.006385803, + -0.04055786, + -0.02609253, + -0.03265381, + -0.02670288, + -0.013587952, + 0.015548706, + -0.047790527, + -0.010292053, + -0.02508545, + -0.005592346, + -0.025299072, + 0.023254395, + 0.0043945312, + 0.0062408447, + 0.006996155, + -0.015060425, + -0.0059814453, + -0.033325195, + -0.024520874, + -0.015472412, + 0.01676941, + -0.011817932, + 0.03173828, + 0.018981934, + -0.03488159, + -0.005340576, + -0.003358841, + 0.045715332, + 0.03314209, + 0.050964355, + -0.018859863, + 0.0541687, + 0.025115967, + 0.025894165, + -0.028366089, + -0.0070533752, + -0.022506714, + 0.018463135, + 0.0068588257, + -0.023742676, + -0.011627197, + -0.05935669, + 0.026000977, + -0.013893127, + 0.06555176, + -0.010292053, + -0.020202637, + 0.018432617, + -0.0043754578, + 0.030548096, + 0.0262146, + 0.027801514, + -0.039001465, + 0.026412964, + 0.028793335, + 0.0063476562, + -0.0027694702, + -0.014305115, + 0.022003174, + 0.0017242432, + -0.02116394, + 0.028152466, + -0.027023315, + -0.008705139, + -0.0037574768, + 0.048034668, + 0.010238647, + -0.020324707, + 0.03086853, + -0.031066895, + 0.0146102905, + 0.014930725, + -0.014785767, + -0.010292053, + 0.017929077, + -0.010429382, + -0.00019311905, + -0.0012149811, + -0.026733398, + -0.026031494, + -0.050048828, + -0.07861328, + -0.017684937, + 0.061706543, + -0.011001587, + -0.041168213, + 0.003314972, + 0.029876709, + -0.009559631, + 0.032348633, + 0.0635376, + -0.040252686, + 0.056365967, + -0.0446167, + 0.026031494, + 0.017089844, + -0.04397583, + 0.044311523, + 0.0068740845, + 0.034454346, + 0.025848389, + 0.02027893, + 0.005153656, + 0.04159546, + -0.008239746, + 0.0056381226, + -0.0033721924, + -0.0692749, + -0.0038280487, + 0.022140503, + -0.008087158, + 0.0051727295, + -0.0102005005, + 0.0009098053, + 0.04067993, + -0.0065193176, + 0.026031494, + -0.08728027, + -0.027648926, + 0.04373169, + -0.048187256, + -0.033233643, + -0.014953613, + 0.022720337, + -0.004333496, + 0.02609253, + 0.0017251968, + -0.017868042, + -0.036956787, + -0.01838684, + 0.06665039, + 0.0259552, + -0.053497314, + -0.03111267, + 0.050872803, + 0.036895752, + -0.030944824, + -0.031921387, + -0.0569458, + 0.020248413, + -0.02229309, + 0.002916336, + 0.0076942444, + -0.0060691833, + -0.0317688, + -0.013793945, + 0.015068054, + -0.004508972, + -0.0047798157, + -0.0021457672, + 0.0003311634, + -0.036346436, + 0.0023174286, + 0.018096924, + 0.0063323975, + -0.014152527, + 0.0023460388, + 0.019836426, + -0.00233078, + 0.009048462, + -0.04812622, + -0.028442383, + 0.04925537, + 0.0043754578, + 0.04650879, + 0.055358887, + 0.036499023, + -0.044677734, + -0.012786865, + -0.013916016, + 0.025985718, + -0.033691406, + 0.010375977, + 0.036590576, + -0.036376953, + 0.009384155, + 0.0012626648, + -0.017623901, + -0.00032114983, + -0.026428223, + 0.018112183, + -0.016098022, + 0.0066375732, + -0.08355713, + 0.024291992, + 0.043670654, + -0.0067367554, + 0.01763916, + -0.0057640076, + -0.0154953, + 0.04196167, + 0.005542755, + 0.026901245, + 0.06427002, + 0.010612488, + 0.040222168, + 0.033966064, + 0.017028809, + 0.02748108, + 0.007980347, + -0.045013428, + -0.001121521, + 0.001408577, + 0.037750244, + 0.013549805, + -0.016967773, + -0.047729492, + 0.027496338, + -0.064331055, + 0.010917664, + -0.013870239, + 0.03668213, + 0.0055236816, + 0.0087509155, + -0.0847168, + -0.009521484, + 0.0703125, + -0.03338623, + -0.011062622, + 0.06555176, + -0.011268616, + -0.08477783, + 0.014633179, + 0.0045928955, + -0.0029029846, + -0.0050849915, + -0.016082764, + 0.037017822, + 0.023406982, + 0.03765869, + -0.032714844, + -0.03692627, + -0.0057411194, + -0.026748657, + 0.0107040405, + 0.033050537, + 0.018829346, + 0.058685303, + 0.0005726814, + 0.026947021, + 0.004272461, + -0.006614685, + -0.0018100739, + -0.024353027, + -0.007835388, + 0.0016746521, + 0.00806427, + -0.008636475, + 0.031188965, + -0.08416748, + 0.05014038, + 0.0073242188, + 0.017822266, + -0.08282471, + 0.010810852, + 0.07312012, + 0.014053345, + 0.00025081635, + 0.0015468597, + 0.00020134449, + -0.0043296814, + -0.050750732, + -0.05758667, + 0.002746582, + 0.030395508, + 0.014060974, + -0.047302246, + -0.045776367, + 0.0045928955, + 0.01739502, + 0.010063171, + 0.0031433105, + -0.005428314, + 0.0031604767, + 0.018371582, + -0.025680542, + 0.0076446533, + 0.0026683807, + 0.025604248, + -0.025741577, + 0.05001831, + 0.06768799, + 0.049713135, + 0.016220093, + -0.06008911, + -0.034942627, + 0.024490356, + -0.01651001, + 0.026443481, + -0.06097412, + 0.04675293, + 0.034240723, + -0.06555176, + 0.02267456, + 0.012382507, + -0.023132324, + 0.015914917, + 0.027236938, + 0.033081055, + 0.025436401, + -0.018951416, + 0.015510559, + 0.0289917, + 0.06317139, + 0.02935791, + -0.03189087, + -0.015930176, + 0.0011873245, + -0.028625488, + 0.013977051, + -0.0012779236, + 0.04220581, + 0.025772095, + 0.009117126, + -0.052642822, + -0.009880066, + -0.032836914, + -0.028945923, + 0.027267456, + 0.07165527, + -0.005748749, + 0.01701355, + 0.0049972534, + -0.005130768, + 0.049835205, + -0.02015686, + 0.03857422, + 0.014305115, + 0.022415161, + 0.025924683, + 0.04031372, + -0.00015962124, + -0.02267456, + 0.003648758, + -0.008026123, + 0.042755127, + -0.004512787, + -0.022079468, + 0.010383606, + -0.014602661, + 0.026138306, + 0.020751953, + -0.025787354, + -0.000538826, + -0.013442993, + -0.00869751, + -0.017547607, + -0.03704834, + -0.010871887, + -0.0012283325, + 0.008880615, + -0.047088623, + -0.008216858, + 0.014083862, + -0.0015964508, + -0.028839111, + -0.00017225742, + -0.038604736, + 0.00187397, + 0.00504303, + 0.017990112, + 0.036224365, + -0.011581421, + -0.01436615, + 0.01626587, + 0.0026187897, + 0.064086914, + 0.016433716, + -0.010345459, + -0.036102295, + 0.025878906, + -0.04260254, + -0.0109939575, + 0.010246277, + 0.006877899, + -0.04071045, + -0.021224976, + -0.003982544, + 0.010421753, + -0.0345459, + -0.073791504, + -0.008987427, + 0.01260376, + -0.043762207, + 0.01210022, + -0.011390686, + -0.0007429123, + -0.027786255, + -0.023620605, + 0.019165039, + -0.010894775, + 0.004272461, + -0.0597229, + 0.036499023, + -0.049224854, + -0.04663086, + -0.02243042, + -0.0018253326, + 0.027572632, + -0.015159607, + -0.014411926, + -0.0033721924, + 0.032470703, + 0.041168213, + -0.021713257, + -0.027160645, + 0.025726318, + 0.048431396, + -0.031829834, + 0.037841797, + 0.04638672, + 0.014976501, + -0.024612427, + 0.0014600754, + -0.04031372, + -0.0011501312, + 0.004142761, + 0.012207031, + -0.00806427, + -0.009025574, + -0.051513672, + 0.030807495, + 0.016998291, + -0.049194336, + 0.0038776398, + -0.0042533875, + -0.04260254, + -0.008239746, + -0.0060195923, + 0.01473999, + 0.0034885406, + -0.0063171387, + -0.048614502, + 0.037628174, + -0.022247314, + -0.018951416, + 0.02192688, + -0.0065994263, + -0.02519226, + -0.0004734993, + -0.036102295, + 0.009109497, + -0.0029640198, + -0.012290955, + 0.011711121, + -0.034942627, + 0.043273926, + 0.022644043, + -0.026351929, + -0.014381409, + 0.044433594, + -0.04949951, + -0.025878906, + -0.01890564, + 0.010566711, + -0.017684937, + -0.06555176, + 0.047912598, + -0.031921387, + 0.047943115, + -0.061584473, + 0.051605225, + 0.009773254, + 0.016525269, + 0.0025367737, + -0.064086914, + 0.031311035, + -0.041778564, + -0.03250122, + -0.044158936, + -0.0135650635, + 0.008224487, + ] + ], + "id": "d26b1832-cd83-40cf-91e9-d96505b89ae8", + "response_type": "embeddings_floats", + "texts": ["This is an embedding test."], + }, + ], + "does-not-exist::": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "f4908827-3db9-4742-9103-2bbc34578b03", + "x-amzn-ErrorType": "ValidationException:http://internal.amazon.com/coral/com.amazon.bedrock/", + }, + 400, + {"message": "The provided model identifier is invalid."}, + ], + "mistral.mistral-7b-instruct-v0%3A2::[INST] Invalid Token [/INST]": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "ai21.j2-mid-v1::Invalid Token": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "9021791d-3797-493d-9277-e33aa6f6d544", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "amazon.titan-embed-g1-text-02::Invalid Token": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "73328313-506e-4da8-af0f-51017fa6ca3f", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "amazon.titan-embed-text-v1::Invalid Token": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "aece6ad7-e2ff-443b-a953-ba7d385fd0cc", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "cohere.embed-english-v3::Invalid Token": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "73328313-506e-4da8-af0f-51017fa6ca3f", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "amazon.titan-text-express-v1::Invalid Token": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "15b39c8b-8e85-42c9-9623-06720301bda3", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "anthropic.claude-instant-v1::Human: Invalid Token Assistant:": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "37396f55-b721-4bae-9461-4c369f5a080d", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "cohere.command-text-v14::Invalid Token": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "22476490-a0d6-42db-b5ea-32d0b8a7f751", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "meta.llama2-13b-chat-v1::Invalid Token": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "22476490-a0d6-42db-b5ea-32d0b8a7f751", + "x-amzn-ErrorType": "UnrecognizedClientException:http://internal.amazon.com/coral/com.amazon.coral.service/", + }, + 403, + {"message": "The security token included in the request is invalid."}, + ], + "amazon.titan-text-express-v1::Malformed Body": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "81508a1c-33a8-4294-8743-f0c629af2f49"}, + 200, + { + "inputTextTokenCount": 12, + "results": [ + { + "tokenCount": 32, + "outputText": "\n1 degree Fahrenheit is 0.56 Celsius. Therefore, 212 degree Fahrenheit in Celsius would be 115.42.", + "completionReason": "FINISH", + } + ], + }, + ], + "amazon.titan-embed-g1-text-02::Malformed Body": [ + {"Content-Type": "application/json", "x-amzn-RequestId": "b10ac895-eae3-4f07-b926-10b2866c55ed"}, + 200, + { + "embedding": [ + -0.14160156, + 0.034423828, + 0.54296875, + 0.10986328, + 0.053466797, + 0.3515625, + 0.12988281, + -0.0002708435, + -0.21484375, + 0.060302734, + 0.58984375, + -0.5859375, + 0.52734375, + 0.82421875, + -0.91015625, + -0.19628906, + 0.45703125, + 0.609375, + -0.67578125, + 0.39453125, + -0.46875, + -0.25390625, + -0.21191406, + 0.114746094, + 0.31640625, + -0.41015625, + -0.32617188, + -0.43554688, + 0.4765625, + -0.4921875, + 0.40429688, + 0.06542969, + 0.859375, + -0.008056641, + -0.19921875, + 0.072753906, + 0.33203125, + 0.69921875, + 0.39453125, + 0.15527344, + 0.08886719, + -0.25, + 0.859375, + 0.22949219, + -0.19042969, + 0.13769531, + -0.078125, + 0.41210938, + 0.875, + 0.5234375, + 0.59765625, + -0.22949219, + -0.22558594, + -0.47460938, + 0.37695312, + 0.51953125, + -0.5703125, + 0.46679688, + 0.43554688, + 0.17480469, + -0.080566406, + -0.16699219, + -0.734375, + -1.0625, + -0.33984375, + 0.390625, + -0.18847656, + -0.5234375, + -0.48828125, + 0.44921875, + -0.09814453, + -0.3359375, + 0.087402344, + 0.36914062, + 1.3203125, + 0.25585938, + 0.14746094, + -0.059570312, + -0.15820312, + -0.037353516, + -0.61328125, + -0.6484375, + -0.35351562, + 0.55078125, + -0.26953125, + 0.90234375, + 0.3671875, + 0.31054688, + 0.00014019012, + -0.171875, + 0.025512695, + 0.5078125, + 0.11621094, + 0.33203125, + 0.8125, + -0.3046875, + -1.078125, + -0.5703125, + 0.26171875, + -0.4609375, + 0.203125, + 0.44726562, + -0.5078125, + 0.41601562, + -0.1953125, + 0.028930664, + -0.57421875, + 0.2265625, + 0.13574219, + -0.040039062, + -0.22949219, + -0.515625, + -0.19042969, + -0.30078125, + 0.10058594, + -0.66796875, + 0.6015625, + 0.296875, + -0.765625, + -0.87109375, + 0.2265625, + 0.068847656, + -0.088378906, + -0.1328125, + -0.796875, + -0.37304688, + 0.47460938, + -0.3515625, + -0.8125, + -0.32226562, + 0.265625, + 0.3203125, + -0.4140625, + -0.49023438, + 0.859375, + -0.19140625, + -0.6328125, + 0.10546875, + -0.5625, + 0.66015625, + 0.26171875, + -0.2109375, + 0.421875, + -0.82421875, + 0.29296875, + 0.17773438, + 0.24023438, + 0.5078125, + -0.49804688, + -0.10205078, + 0.10498047, + -0.36132812, + -0.47460938, + -0.20996094, + 0.010070801, + -0.546875, + 0.66796875, + -0.123046875, + -0.75390625, + 0.19628906, + 0.17480469, + 0.18261719, + -0.96875, + -0.26171875, + 0.4921875, + -0.40039062, + 0.296875, + 0.1640625, + -0.20507812, + -0.36132812, + 0.76171875, + -1.234375, + -0.625, + 0.060058594, + -0.09375, + -0.14746094, + 1.09375, + 0.057861328, + 0.22460938, + -0.703125, + 0.07470703, + 0.23828125, + -0.083984375, + -0.54296875, + 0.5546875, + -0.5, + -0.390625, + 0.106933594, + 0.6640625, + 0.27734375, + -0.953125, + 0.35351562, + -0.7734375, + -0.77734375, + 0.16503906, + -0.42382812, + 0.36914062, + 0.020141602, + -1.3515625, + 0.18847656, + 0.13476562, + -0.034179688, + -0.03930664, + -0.03857422, + -0.027954102, + 0.73828125, + -0.18945312, + -0.09814453, + -0.46289062, + 0.36914062, + 0.033203125, + 0.020874023, + -0.703125, + 0.91796875, + 0.38671875, + 0.625, + -0.19335938, + -0.16796875, + -0.58203125, + 0.21386719, + -0.032470703, + -0.296875, + -0.15625, + -0.1640625, + -0.74609375, + 0.328125, + 0.5546875, + -0.1953125, + 1.0546875, + 0.171875, + -0.099609375, + 0.5234375, + 0.05078125, + -0.35742188, + -0.2734375, + -1.3203125, + -0.8515625, + -0.16015625, + 0.01574707, + 0.29296875, + 0.18457031, + -0.265625, + 0.048339844, + 0.045654297, + -0.32226562, + 0.087890625, + -0.0047302246, + 0.38671875, + 0.10644531, + -0.06225586, + 1.03125, + 0.94140625, + -0.3203125, + 0.20800781, + -1.171875, + 0.48046875, + -0.091796875, + 0.20800781, + -0.1328125, + -0.20507812, + 0.28125, + -0.47070312, + -0.09033203, + 0.0013809204, + -0.08203125, + 0.43359375, + -0.03100586, + -0.060791016, + -0.53515625, + -1.46875, + 0.000101566315, + 0.515625, + 0.40625, + -0.10498047, + -0.15820312, + -0.009460449, + -0.77734375, + -0.5859375, + 0.9765625, + 0.099609375, + 0.51953125, + 0.38085938, + -0.09667969, + -0.100097656, + -0.5, + -1.3125, + -0.18066406, + -0.099121094, + 0.26171875, + -0.14453125, + -0.546875, + 0.17578125, + 0.484375, + 0.765625, + 0.45703125, + 0.2734375, + 0.0028076172, + 0.17089844, + -0.32421875, + -0.37695312, + 0.30664062, + -0.48046875, + 0.07128906, + 0.031982422, + -0.31054688, + -0.055419922, + -0.29296875, + 0.3359375, + -0.296875, + 0.47851562, + -0.05126953, + 0.18457031, + -0.01953125, + -0.35742188, + 0.017944336, + -0.25, + 0.10595703, + 0.17382812, + -0.73828125, + 0.36914062, + -0.15234375, + -0.8125, + 0.17382812, + 0.048095703, + 0.5625, + -0.33789062, + 0.023071289, + -0.21972656, + 0.16015625, + 0.032958984, + -1.1171875, + -0.984375, + 0.83984375, + 0.009033203, + -0.042236328, + -0.46484375, + -0.08203125, + 0.44726562, + -0.765625, + -0.3984375, + -0.40820312, + -0.234375, + 0.044189453, + 0.119628906, + -0.7578125, + -0.55078125, + -0.4453125, + 0.7578125, + 0.34960938, + 0.96484375, + 0.35742188, + 0.36914062, + -0.35351562, + -0.36132812, + 1.109375, + 0.5859375, + 0.85546875, + -0.10644531, + -0.6953125, + -0.0066833496, + 0.042236328, + -0.06689453, + 0.36914062, + 0.9765625, + -0.3046875, + 0.59765625, + -0.6640625, + 0.21484375, + -0.07128906, + 1.1328125, + -0.51953125, + 0.86328125, + -0.11328125, + 0.15722656, + -0.36328125, + -0.04638672, + 1.4375, + 0.18457031, + -0.18359375, + 0.10595703, + -0.49023438, + -0.07324219, + -0.73046875, + -0.119140625, + 0.021118164, + 0.4921875, + -0.46875, + 0.28710938, + 0.3359375, + 0.11767578, + -0.2109375, + -0.14550781, + 0.39648438, + -0.27734375, + 0.48046875, + 0.12988281, + 0.45507812, + -0.375, + -0.84765625, + 0.25585938, + -0.36523438, + 0.8046875, + 0.42382812, + -0.24511719, + 0.54296875, + 0.71875, + 0.010009766, + -0.04296875, + 0.083984375, + -0.52734375, + 0.13964844, + -0.27539062, + -0.30273438, + 1.1484375, + -0.515625, + -0.19335938, + 0.58984375, + 0.049072266, + 0.703125, + -0.04272461, + 0.5078125, + 0.34960938, + -0.3359375, + -0.47460938, + 0.049316406, + 0.36523438, + 0.7578125, + -0.022827148, + -0.71484375, + 0.21972656, + 0.09716797, + -0.203125, + -0.36914062, + 1.34375, + 0.34179688, + 0.46679688, + 1.078125, + 0.26171875, + 0.41992188, + 0.22363281, + -0.515625, + -0.5703125, + 0.13378906, + 0.26757812, + -0.22558594, + -0.5234375, + 0.06689453, + 0.08251953, + -0.625, + 0.16796875, + 0.43164062, + -0.55859375, + 0.28125, + 0.078125, + 0.6328125, + 0.23242188, + -0.064941406, + -0.004486084, + -0.20703125, + 0.2734375, + 0.453125, + -0.734375, + 0.04272461, + 0.36132812, + -0.19628906, + -0.12402344, + 1.3515625, + 0.25585938, + 0.4921875, + -0.29296875, + -0.58984375, + 0.021240234, + -0.044677734, + 0.7578125, + -0.7890625, + 0.10253906, + -0.15820312, + -0.5078125, + -0.39453125, + -0.453125, + 0.35742188, + 0.921875, + 0.44335938, + -0.49804688, + 0.44335938, + 0.31445312, + 0.58984375, + -1.0078125, + -0.22460938, + 0.24121094, + 0.87890625, + 0.66015625, + -0.390625, + -0.05053711, + 0.059570312, + 0.36132812, + -0.00038719177, + -0.017089844, + 0.62890625, + 0.203125, + 0.17480469, + 0.025512695, + 0.47460938, + 0.3125, + 1.140625, + 0.32421875, + -0.057861328, + 0.36914062, + -0.7265625, + -0.51953125, + 0.26953125, + 0.42773438, + 0.064453125, + 0.6328125, + 0.27148438, + -0.11767578, + 0.66796875, + -0.38671875, + 0.5234375, + -0.59375, + 0.5078125, + 0.008239746, + -0.34179688, + -0.27539062, + 0.5234375, + 1.296875, + 0.29492188, + -0.010986328, + -0.41210938, + 0.59375, + 0.061767578, + -0.33398438, + -2.03125, + 0.87890625, + -0.010620117, + 0.53125, + 0.14257812, + -0.515625, + -1.03125, + 0.578125, + 0.1875, + 0.44335938, + -0.33203125, + -0.36328125, + -0.3203125, + 0.29296875, + -0.8203125, + 0.41015625, + -0.48242188, + 0.66015625, + 0.5625, + -0.16503906, + -0.54296875, + -0.38085938, + 0.26171875, + 0.62109375, + 0.29101562, + -0.31054688, + 0.23730469, + -0.8515625, + 0.5234375, + 0.15332031, + 0.52734375, + -0.079589844, + -0.080566406, + -0.15527344, + -0.022827148, + 0.030517578, + -0.1640625, + -0.421875, + 0.09716797, + 0.03930664, + -0.055908203, + -0.546875, + -0.47851562, + 0.091796875, + 0.32226562, + -0.94140625, + -0.04638672, + -1.203125, + -0.39648438, + 0.45507812, + 0.296875, + -0.45703125, + 0.37890625, + -0.122558594, + 0.28320312, + -0.01965332, + -0.11669922, + -0.34570312, + -0.53515625, + -0.091308594, + -0.9375, + -0.32617188, + 0.095214844, + -0.4765625, + 0.37890625, + -0.859375, + 1.1015625, + -0.08935547, + 0.46484375, + -0.19238281, + 0.7109375, + 0.040039062, + -0.5390625, + 0.22363281, + -0.70703125, + 0.4921875, + -0.119140625, + -0.26757812, + -0.08496094, + 0.0859375, + -0.00390625, + -0.013366699, + -0.03955078, + 0.07421875, + -0.13085938, + 0.29101562, + -0.12109375, + 0.45703125, + 0.021728516, + 0.38671875, + -0.3671875, + -0.52734375, + -0.115722656, + 0.125, + 0.5703125, + -1.234375, + 0.06298828, + -0.55859375, + 0.60546875, + 0.8125, + -0.0032958984, + -0.068359375, + -0.21191406, + 0.56640625, + 0.17285156, + -0.3515625, + 0.36328125, + -0.99609375, + 0.43554688, + -0.1015625, + 0.07080078, + -0.66796875, + 1.359375, + 0.41601562, + 0.15917969, + 0.17773438, + -0.28710938, + 0.021850586, + -0.46289062, + 0.17578125, + -0.03955078, + -0.026855469, + 0.5078125, + -0.65625, + 0.0012512207, + 0.044433594, + -0.18652344, + 0.4921875, + -0.75390625, + 0.0072021484, + 0.4375, + -0.31445312, + 0.20214844, + 0.15039062, + -0.63671875, + -0.296875, + -0.375, + -0.027709961, + 0.013427734, + 0.17089844, + 0.89453125, + 0.11621094, + -0.43945312, + -0.30859375, + 0.02709961, + 0.23242188, + -0.64453125, + -0.859375, + 0.22167969, + -0.023071289, + -0.052734375, + 0.3671875, + -0.18359375, + 0.81640625, + -0.11816406, + 0.028320312, + 0.19042969, + 0.012817383, + -0.43164062, + 0.55859375, + -0.27929688, + 0.14257812, + -0.140625, + -0.048583984, + -0.014526367, + 0.35742188, + 0.22753906, + 0.13183594, + 0.04638672, + 0.03930664, + -0.29296875, + -0.2109375, + -0.16308594, + -0.48046875, + -0.13378906, + -0.39257812, + 0.29296875, + -0.047851562, + -0.5546875, + 0.08300781, + -0.14941406, + -0.07080078, + 0.12451172, + 0.1953125, + -0.51171875, + -0.048095703, + 0.1953125, + -0.37695312, + 0.46875, + -0.084472656, + 0.19042969, + -0.39453125, + 0.69921875, + -0.0065307617, + 0.25390625, + -0.16992188, + -0.5078125, + 0.016845703, + 0.27929688, + -0.22070312, + 0.671875, + 0.18652344, + 0.25, + -0.046875, + -0.012023926, + -0.36523438, + 0.36523438, + -0.11279297, + 0.421875, + 0.079589844, + -0.100097656, + 0.37304688, + 0.29882812, + -0.10546875, + -0.36523438, + 0.040039062, + 0.546875, + 0.12890625, + -0.06542969, + -0.38085938, + -0.35742188, + -0.6484375, + -0.28515625, + 0.0107421875, + -0.055664062, + 0.45703125, + 0.33984375, + 0.26367188, + -0.23144531, + 0.012878418, + -0.875, + 0.11035156, + 0.33984375, + 0.203125, + 0.38867188, + 0.24902344, + -0.37304688, + -0.98046875, + -0.122558594, + -0.17871094, + -0.09277344, + 0.1796875, + 0.4453125, + -0.66796875, + 0.78515625, + 0.12988281, + 0.35546875, + 0.44140625, + 0.58984375, + 0.29492188, + 0.7734375, + -0.21972656, + -0.40234375, + -0.22265625, + 0.18359375, + 0.54296875, + 0.17382812, + 0.59375, + -0.390625, + -0.92578125, + -0.017456055, + -0.25, + 0.73828125, + 0.7578125, + -0.3828125, + -0.25976562, + 0.049072266, + 0.046875, + -0.3515625, + 0.30078125, + -1.03125, + -0.48828125, + 0.0017929077, + -0.26171875, + 0.20214844, + 0.29882812, + 0.064941406, + 0.21484375, + -0.55078125, + -0.021362305, + 0.12988281, + 0.27148438, + 0.38867188, + -0.19726562, + -0.55078125, + 0.1640625, + 0.32226562, + -0.72265625, + 0.36132812, + 1.21875, + -0.22070312, + -0.32421875, + -0.29882812, + 0.0024414062, + 0.19921875, + 0.734375, + 0.16210938, + 0.17871094, + -0.19140625, + 0.38476562, + -0.06591797, + -0.47070312, + -0.040039062, + -0.33007812, + -0.07910156, + -0.2890625, + 0.00970459, + 0.12695312, + -0.12060547, + -0.18847656, + 1.015625, + -0.032958984, + 0.12451172, + -0.38476562, + 0.063964844, + 1.0859375, + 0.067871094, + -0.24511719, + 0.125, + 0.10546875, + -0.22460938, + -0.29101562, + 0.24414062, + -0.017944336, + -0.15625, + -0.60546875, + -0.25195312, + -0.46875, + 0.80859375, + -0.34960938, + 0.42382812, + 0.796875, + 0.296875, + -0.067871094, + 0.39453125, + 0.07470703, + 0.033935547, + 0.24414062, + 0.32617188, + 0.023925781, + 0.73046875, + 0.2109375, + -0.43164062, + 0.14453125, + 0.63671875, + 0.21972656, + -0.1875, + -0.18066406, + -0.22167969, + -1.3359375, + 0.52734375, + -0.40625, + -0.12988281, + 0.17480469, + -0.18066406, + 0.58984375, + -0.32421875, + -0.13476562, + 0.39257812, + -0.19238281, + 0.068359375, + 0.7265625, + -0.7109375, + -0.125, + 0.328125, + 0.34179688, + -0.48828125, + -0.10058594, + -0.83984375, + 0.30273438, + 0.008239746, + -1.390625, + 0.171875, + 0.34960938, + 0.44921875, + 0.22167969, + 0.60546875, + -0.36914062, + -0.028808594, + -0.19921875, + 0.6875, + 0.52734375, + -0.07421875, + 0.35546875, + 0.546875, + 0.08691406, + 0.23339844, + -0.984375, + -0.20507812, + 0.08544922, + 0.453125, + -0.07421875, + -0.953125, + 0.74609375, + -0.796875, + 0.47851562, + 0.81640625, + -0.44921875, + -0.33398438, + -0.54296875, + 0.46484375, + -0.390625, + -0.24121094, + -0.0115356445, + 1.1328125, + 1.0390625, + 0.6484375, + 0.35742188, + -0.29492188, + -0.0007095337, + -0.060302734, + 0.21777344, + 0.15136719, + -0.6171875, + 0.11328125, + -0.025878906, + 0.19238281, + 0.140625, + 0.171875, + 0.25195312, + 0.10546875, + 0.0008354187, + -0.13476562, + -0.26953125, + 0.025024414, + -0.28320312, + -0.107910156, + 1.015625, + 0.05493164, + -0.12988281, + 0.30859375, + 0.22558594, + -0.60546875, + 0.11328125, + -1.203125, + 0.6484375, + 0.087402344, + 0.32226562, + 0.63671875, + -0.07714844, + -1.390625, + -0.71875, + -0.34179688, + -0.10546875, + -0.37304688, + -0.09863281, + -0.41210938, + -0.14941406, + 0.41210938, + -0.20898438, + 0.18261719, + 0.67578125, + 0.41601562, + 0.32617188, + 0.2421875, + -0.14257812, + -0.6796875, + 0.01953125, + 0.34179688, + 0.20800781, + -0.123046875, + 0.087402344, + 0.85546875, + 0.33984375, + 0.33203125, + -0.68359375, + 0.44921875, + 0.50390625, + 0.083496094, + 0.10888672, + -0.09863281, + 0.55078125, + 0.09765625, + -0.50390625, + 0.13378906, + -0.29882812, + 0.030761719, + -0.64453125, + 0.22949219, + 0.43945312, + 0.16503906, + 0.10888672, + -0.12792969, + -0.039794922, + -0.111328125, + -0.35742188, + 0.053222656, + -0.78125, + -0.4375, + 0.359375, + -0.88671875, + -0.21972656, + -0.053710938, + 0.91796875, + -0.10644531, + 0.55859375, + -0.7734375, + 0.5078125, + 0.46484375, + 0.32226562, + 0.16796875, + -0.28515625, + 0.045410156, + -0.45117188, + 0.38867188, + -0.33398438, + -0.5234375, + 0.296875, + 0.6015625, + 0.3515625, + -0.734375, + 0.3984375, + -0.08251953, + 0.359375, + -0.28515625, + -0.88671875, + 0.0051879883, + 0.045166016, + -0.7421875, + -0.36523438, + 0.140625, + 0.18066406, + -0.171875, + -0.15625, + -0.53515625, + 0.2421875, + -0.19140625, + -0.18066406, + 0.25390625, + 0.6875, + -0.01965332, + -0.33203125, + 0.29492188, + 0.107421875, + -0.048339844, + -0.82421875, + 0.52734375, + 0.78125, + 0.8203125, + -0.90625, + 0.765625, + 0.0390625, + 0.045410156, + 0.26367188, + -0.14355469, + -0.26367188, + 0.390625, + -0.10888672, + 0.33007812, + -0.5625, + 0.08105469, + -0.13769531, + 0.8515625, + -0.14453125, + 0.77734375, + -0.48046875, + -0.3515625, + -0.25390625, + -0.09277344, + 0.23925781, + -0.022338867, + -0.45898438, + 0.36132812, + -0.23828125, + 0.265625, + -0.48632812, + -0.46875, + -0.75390625, + 1.3125, + 0.78125, + -0.63671875, + -1.21875, + 0.5078125, + -0.27734375, + -0.118652344, + 0.041992188, + -0.14648438, + -0.8046875, + 0.21679688, + -0.79296875, + 0.28320312, + -0.09667969, + 0.42773438, + 0.49414062, + 0.44726562, + 0.21972656, + -0.02746582, + -0.03540039, + -0.14941406, + -0.515625, + -0.27929688, + 0.9609375, + -0.007598877, + 0.34765625, + -0.060546875, + -0.44726562, + 0.7421875, + 0.15332031, + 0.45117188, + -0.4921875, + 0.07080078, + 0.5625, + 0.3984375, + -0.20019531, + 0.014892578, + 0.63671875, + -0.0071411133, + 0.016357422, + 1.0625, + 0.049316406, + 0.18066406, + 0.09814453, + -0.52734375, + -0.359375, + -0.072265625, + -0.41992188, + 0.39648438, + 0.38671875, + -0.30273438, + -0.056640625, + -0.640625, + -0.44921875, + 0.49414062, + 0.29101562, + 0.49609375, + 0.40429688, + -0.10205078, + 0.49414062, + -0.28125, + -0.12695312, + -0.0022735596, + -0.37304688, + 0.122558594, + 0.07519531, + -0.12597656, + -0.38085938, + -0.19824219, + -0.40039062, + 0.56640625, + -1.140625, + -0.515625, + -0.17578125, + -0.765625, + -0.43945312, + 0.3359375, + -0.24707031, + 0.32617188, + -0.45117188, + -0.37109375, + 0.45117188, + -0.27539062, + -0.38867188, + 0.09082031, + 0.17675781, + 0.49414062, + 0.19921875, + 0.17480469, + 0.8515625, + -0.23046875, + -0.234375, + -0.28515625, + 0.10253906, + 0.29101562, + -0.3359375, + -0.203125, + 0.6484375, + 0.11767578, + -0.20214844, + -0.42382812, + 0.26367188, + 0.6328125, + 0.0059509277, + 0.08691406, + -1.5625, + -0.43554688, + 0.17675781, + 0.091796875, + -0.5234375, + -0.09863281, + 0.20605469, + 0.16601562, + -0.578125, + 0.017700195, + 0.41015625, + 1.03125, + -0.55078125, + 0.21289062, + -0.35351562, + 0.24316406, + -0.123535156, + 0.11035156, + -0.48242188, + -0.34179688, + 0.45117188, + 0.3125, + -0.071777344, + 0.12792969, + 0.55859375, + 0.063964844, + -0.21191406, + 0.01965332, + -1.359375, + -0.21582031, + -0.019042969, + 0.16308594, + -0.3671875, + -0.40625, + -1.0234375, + -0.21289062, + 0.24023438, + -0.28125, + 0.26953125, + -0.14550781, + -0.087890625, + 0.16113281, + -0.49804688, + -0.17675781, + -0.890625, + 0.27929688, + 0.484375, + 0.27148438, + 0.11816406, + 0.83984375, + 0.029052734, + -0.890625, + 0.66796875, + 0.78515625, + -0.953125, + 0.49414062, + -0.546875, + 0.106933594, + -0.08251953, + 0.2890625, + -0.1484375, + -0.85546875, + 0.32421875, + -0.0040893555, + -0.16601562, + -0.16699219, + 0.24414062, + -0.5078125, + 0.25390625, + -0.10253906, + 0.15625, + 0.140625, + -0.27539062, + -0.546875, + -0.5546875, + -0.71875, + 0.37304688, + 0.060058594, + -0.076171875, + 0.44921875, + 0.06933594, + -0.28710938, + -0.22949219, + 0.17578125, + 0.09814453, + 0.4765625, + -0.95703125, + -0.03540039, + 0.21289062, + -0.7578125, + -0.07373047, + 0.10546875, + 0.07128906, + 0.76171875, + 0.4296875, + -0.09375, + 0.27539062, + -0.55078125, + 0.29882812, + -0.42382812, + 0.32617188, + -0.39648438, + 0.12451172, + 0.16503906, + -0.22460938, + -0.65625, + -0.022094727, + 0.61328125, + -0.024780273, + 0.62109375, + -0.033447266, + 0.515625, + 0.12890625, + -0.21875, + -0.08642578, + 0.49804688, + -0.2265625, + -0.29296875, + 0.19238281, + 0.3515625, + -1.265625, + 0.57421875, + 0.20117188, + -0.28320312, + 0.1953125, + -0.30664062, + 0.2265625, + -0.11230469, + 0.83984375, + 0.111328125, + 0.265625, + 0.71484375, + -0.625, + 0.38867188, + 0.47070312, + -0.32617188, + -0.171875, + 1.0078125, + 0.19726562, + -0.118652344, + 0.63671875, + -0.068359375, + -0.25585938, + 0.4140625, + -0.29296875, + 0.21386719, + -0.064453125, + 0.15820312, + -0.89453125, + -0.16308594, + 0.48046875, + 0.14648438, + -0.5703125, + 0.84765625, + -0.19042969, + 0.03515625, + 0.42578125, + -0.27539062, + -0.5390625, + 0.95703125, + 0.2734375, + 0.16699219, + -0.328125, + 0.11279297, + 0.003250122, + 0.47265625, + -0.31640625, + 0.546875, + 0.55859375, + 0.06933594, + -0.61328125, + -0.16210938, + -0.375, + 0.100097656, + -0.088378906, + 0.12695312, + 0.079589844, + 0.123535156, + -1.0078125, + 0.6875, + 0.022949219, + -0.40039062, + -0.09863281, + 0.29101562, + -1.2890625, + -0.20996094, + 0.36328125, + -0.3515625, + 0.7890625, + 0.12207031, + 0.48046875, + -0.13671875, + -0.041015625, + 0.19824219, + 0.19921875, + 0.01171875, + -0.37695312, + -0.62890625, + 0.9375, + -0.671875, + 0.24609375, + 0.6484375, + -0.29101562, + 0.076171875, + 0.62109375, + -0.5546875, + 0.36523438, + 0.75390625, + -0.19140625, + -0.875, + -0.8203125, + -0.24414062, + -0.625, + 0.1796875, + -0.40039062, + 0.25390625, + -0.14550781, + -0.21679688, + -0.828125, + 0.3359375, + 0.43554688, + 0.55078125, + -0.44921875, + -0.28710938, + 0.24023438, + 0.18066406, + -0.6953125, + 0.020385742, + -0.11376953, + 0.13867188, + -0.92578125, + 0.33398438, + -0.328125, + 0.78125, + -0.45507812, + -0.07470703, + 0.34179688, + 0.07080078, + 0.76171875, + 0.37890625, + -0.10644531, + 0.90234375, + -0.21875, + -0.15917969, + -0.36132812, + 0.2109375, + -0.45703125, + -0.76953125, + 0.21289062, + 0.26367188, + 0.49804688, + 0.35742188, + -0.20019531, + 0.31054688, + 0.34179688, + 0.17089844, + -0.15429688, + 0.39648438, + -0.5859375, + 0.20996094, + -0.40039062, + 0.5703125, + -0.515625, + 0.5234375, + 0.049560547, + 0.328125, + 0.24804688, + 0.42578125, + 0.609375, + 0.19238281, + 0.27929688, + 0.19335938, + 0.78125, + -0.9921875, + 0.23925781, + -1.3828125, + -0.22949219, + -0.578125, + -0.13964844, + -0.17382812, + -0.011169434, + 0.26171875, + -0.73046875, + -1.4375, + 0.6953125, + -0.7421875, + 0.052246094, + 0.12207031, + 1.3046875, + 0.38867188, + 0.040283203, + -0.546875, + -0.0021514893, + 0.18457031, + -0.5546875, + -0.51171875, + -0.16308594, + -0.104003906, + -0.38867188, + -0.20996094, + -0.8984375, + 0.6015625, + -0.30078125, + -0.13769531, + 0.16113281, + 0.58203125, + -0.23730469, + -0.125, + -1.0234375, + 0.875, + -0.7109375, + 0.29101562, + 0.09667969, + -0.3203125, + -0.48046875, + 0.37890625, + 0.734375, + -0.28710938, + -0.29882812, + -0.05493164, + 0.34765625, + -0.84375, + 0.65625, + 0.578125, + -0.20019531, + 0.13769531, + 0.10058594, + -0.37109375, + 0.36523438, + -0.22167969, + 0.72265625, + ], + "inputTextTokenCount": 6, + }, + ], + "amazon.titan-text-express-v1::{ Malformed Request Body": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "e72d1b46-9f16-4bf0-8eee-f7778f32e5a5", + "x-amzn-ErrorType": "ValidationException:http://internal.amazon.com/coral/com.amazon.bedrock/", + }, + 400, + {"message": "Malformed input request, please reformat your input and try again."}, + ], + "amazon.titan-embed-g1-text-02::{ Malformed Request Body": [ + { + "Content-Type": "application/json", + "x-amzn-RequestId": "b3646569-18c5-4173-a9fa-bbe9c648f636", + "x-amzn-ErrorType": "ValidationException:http://internal.amazon.com/coral/com.amazon.bedrock/", + }, + 400, + {"message": "Malformed input request, please reformat your input and try again."}, + ], +} + + +MODEL_PATH_RE = re.compile(r"/model/([^/]+)/invoke") + + +def simple_get(self): + content_len = int(self.headers.get("content-length")) + body = self.rfile.read(content_len).decode("utf-8") + try: + content = json.loads(body) + except Exception: + content = body + + stream = self.path.endswith("invoke-with-response-stream") + model = MODEL_PATH_RE.match(self.path).group(1) + prompt = extract_shortened_prompt(content, model) + if not prompt: + self.send_response(500) + self.end_headers() + self.wfile.write("Could not parse prompt.".encode("utf-8")) + return + + headers, status_code, response = ({}, 0, "") + if stream: + for k, v in STREAMED_RESPONSES.items(): + if prompt.startswith(k): + headers, status_code, response = v + break + if not response: + for k, v in RESPONSES.items(): + # Only look for error responses returned immediately instead of in a stream + if prompt.startswith(k) and v[1] >= 400: + headers, status_code, response = v + stream = False # Response will not be streamed + break + else: + for k, v in RESPONSES.items(): + if prompt.startswith(k): + headers, status_code, response = v + break + + if not response: + # If no matches found + self.send_response(500) + self.end_headers() + self.wfile.write(f"Unknown Prompt:\n{prompt}".encode("utf-8")) + return + + if stream: + # Send response code + self.send_response(status_code) + + # Send headers + for k, v in headers.items(): + self.send_header(k, v) + self.end_headers() + + # Send response body + for resp in response: + self.wfile.write(bytes.fromhex(resp)) + else: + # Send response code + self.send_response(status_code) + + # Send headers + for k, v in headers.items(): + self.send_header(k, v) + self.end_headers() + + # Send response body + response_body = json.dumps(response).encode("utf-8") + + if "Malformed Body" in prompt: + # Remove end of response to make invalid JSON + response_body = response_body[:-4] + + self.wfile.write(response_body) + return + + +def extract_shortened_prompt(content, model): + if isinstance(content, str): + prompt = content + elif "messages" in content: + prompt = content["messages"][0].get("content") + else: + prompt = content.get("inputText", "") or content.get("prompt", "") or content.get("texts", [""])[0] + # Sometimes there are leading whitespaces in the prompt. + prompt = prompt.strip() + prompt = f"{model}::{prompt}" # Prepend model name to prompt key to keep separate copies + return prompt.lstrip().split("\n")[0] + + +class MockExternalBedrockServer(MockExternalHTTPServer): + # To use this class in a test one needs to start and stop this server + # before and after making requests to the test app that makes the external + # calls. + + def __init__(self, handler=simple_get, port=None, *args, **kwargs): + super(MockExternalBedrockServer, self).__init__(handler=handler, port=port, *args, **kwargs) + + +if __name__ == "__main__": + # Use this to sort dict for easier future incremental updates + print(f"RESPONSES = {dict(sorted(RESPONSES.items(), key=lambda i: (i[1][1], i[0])))}") + + with MockExternalBedrockServer() as server: + print(f"MockExternalBedrockServer serving on port {str(server.port)}") + while True: + pass # Serve forever diff --git a/tests/external_botocore/_test_bedrock_chat_completion.py b/tests/external_botocore/_test_bedrock_chat_completion.py new file mode 100644 index 0000000000..f7c536c4e7 --- /dev/null +++ b/tests/external_botocore/_test_bedrock_chat_completion.py @@ -0,0 +1,1587 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +chat_completion_payload_templates = { + "amazon.titan-text-express-v1": '{ "inputText": "%s", "textGenerationConfig": {"temperature": %f, "maxTokenCount": %d }}', + "ai21.j2-mid-v1": '{"prompt": "%s", "temperature": %f, "maxTokens": %d}', + "anthropic.claude-instant-v1": '{"prompt": "Human: %s Assistant:", "temperature": %f, "max_tokens_to_sample": %d}', + "cohere.command-text-v14": '{"prompt": "%s", "temperature": %f, "max_tokens": %d}', + "meta.llama2-13b-chat-v1": '{"prompt": "%s", "temperature": %f, "max_gen_len": %d}', + "mistral.mistral-7b-instruct-v0:2": '{"prompt": "[INST] %s [/INST]", "temperature": %f, "max_tokens": %d}', +} + +chat_completion_expected_events = { + "mistral.mistral-7b-instruct-v0:2": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "duration": None, # Response time varies each test run + "request.model": "mistral.mistral-7b-instruct-v0:2", + "response.model": "mistral.mistral-7b-instruct-v0:2", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "length", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "span_id": None, + "trace_id": "trace-id", + "content": "[INST] What is 212 degrees Fahrenheit converted to Celsius? [/INST]", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "mistral.mistral-7b-instruct-v0:2", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "span_id": None, + "trace_id": "trace-id", + "content": " To convert a temperature from Fahrenheit to Celsius, you can use the following formula:\n\nCelsius = (Fahrenheit - 32) \u00d7 5/9\n\nSo, to convert 212 degrees Fahrenheit to Celsius:\n\nCelsius = (212 - 32) \u00d7 5/9\nCelsius = 180.56 \u00d7 5/9\nCelsius", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "mistral.mistral-7b-instruct-v0:2", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "amazon.titan-text-express-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "81508a1c-33a8-4294-8743-f0c629af2f49", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-text-express-v1", + "response.model": "amazon.titan-text-express-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "FINISH", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "81508a1c-33a8-4294-8743-f0c629af2f49", + "span_id": None, + "trace_id": "trace-id", + "content": "What is 212 degrees Fahrenheit converted to Celsius?", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "81508a1c-33a8-4294-8743-f0c629af2f49", + "span_id": None, + "trace_id": "trace-id", + "content": "\n1 degree Fahrenheit is 0.56 Celsius. Therefore, 212 degree Fahrenheit in Celsius would be 115.42.", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "ai21.j2-mid-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "228ee63f-4eca-4b7d-b679-bc920de63525", + "response_id": "1234", + "duration": None, # Response time varies each test run + "request.model": "ai21.j2-mid-v1", + "response.model": "ai21.j2-mid-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "endoftext", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": "1234-0", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "228ee63f-4eca-4b7d-b679-bc920de63525", + "span_id": None, + "trace_id": "trace-id", + "content": "What is 212 degrees Fahrenheit converted to Celsius?", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "ai21.j2-mid-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": "1234-1", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "228ee63f-4eca-4b7d-b679-bc920de63525", + "span_id": None, + "trace_id": "trace-id", + "content": "\n212 degrees Fahrenheit is equal to 100 degrees Celsius.", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "ai21.j2-mid-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "anthropic.claude-instant-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "6a886158-b39f-46ce-b214-97458ab76f2f", + "duration": None, # Response time varies each test run + "request.model": "anthropic.claude-instant-v1", + "response.model": "anthropic.claude-instant-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "max_tokens", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "6a886158-b39f-46ce-b214-97458ab76f2f", + "span_id": None, + "trace_id": "trace-id", + "content": "Human: What is 212 degrees Fahrenheit converted to Celsius? Assistant:", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "anthropic.claude-instant-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "6a886158-b39f-46ce-b214-97458ab76f2f", + "span_id": None, + "trace_id": "trace-id", + "content": " Okay, here are the steps to convert 212 degrees Fahrenheit to Celsius:\n\n1) The formula to convert between Fahrenheit and Celsius is:\n C = (F - 32) * 5/9\n\n2) Plug in 212 degrees Fahrenheit for F:\n C = (212 - 32) * 5/9\n C = 180 * 5/9\n C = 100\n\n3) Therefore, 212 degrees Fahrenheit converted to Celsius is 100 degrees Celsius.", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "anthropic.claude-instant-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "cohere.command-text-v14": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "12912a17-aa13-45f3-914c-cc82166f3601", + "response_id": None, # UUID that varies with each run + "duration": None, # Response time varies each test run + "request.model": "cohere.command-text-v14", + "response.model": "cohere.command-text-v14", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "MAX_TOKENS", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "12912a17-aa13-45f3-914c-cc82166f3601", + "span_id": None, + "trace_id": "trace-id", + "content": "What is 212 degrees Fahrenheit converted to Celsius?", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "cohere.command-text-v14", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "12912a17-aa13-45f3-914c-cc82166f3601", + "span_id": None, + "trace_id": "trace-id", + "content": " To convert from Fahrenheit to Celsius, you can use the following formula:\n\nCelsius = (Fahrenheit - 32) * 5/9\n\nIn this case, 212 degrees Fahrenheit is converted to Celsius as follows:\n\nCelsius = (212 - 32) * 5/9 = (180) * 5/9 = (180/9) = 20 degrees Celsius\n\nTherefore, 212 degrees Fahrenheit is equivalent to 20 degrees Celsius.\n\nIt's important to note that", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "cohere.command-text-v14", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "meta.llama2-13b-chat-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "a168214d-742d-4244-bd7f-62214ffa07df", + "duration": None, # Response time varies each test run + "request.model": "meta.llama2-13b-chat-v1", + "response.model": "meta.llama2-13b-chat-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "stop", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "a168214d-742d-4244-bd7f-62214ffa07df", + "span_id": None, + "trace_id": "trace-id", + "content": "What is 212 degrees Fahrenheit converted to Celsius?", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "meta.llama2-13b-chat-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "a168214d-742d-4244-bd7f-62214ffa07df", + "span_id": None, + "trace_id": "trace-id", + "content": "\n\n212°F = ?°C\n\nPlease help! I'm stuck!\n\nThank you!\n\nI hope this is the correct place to ask this question. Please let me know if it isn't.\n\nI appreciate your help!\n\nBest regards,\n\n[Your Name]", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "meta.llama2-13b-chat-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], +} +chat_completion_langchain_expected_streaming_events = { + "mistral.mistral-7b-instruct-v0:2": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "duration": None, # Response time varies each test run + "request.model": "mistral.mistral-7b-instruct-v0:2", + "response.model": "mistral.mistral-7b-instruct-v0:2", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "length", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "span_id": None, + "trace_id": "trace-id", + "content": "[INST] What is 212 degrees Fahrenheit converted to Celsius? [/INST]", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "mistral.mistral-7b-instruct-v0:2", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "span_id": None, + "trace_id": "trace-id", + "content": " To convert a temperature from Fahrenheit to Celsius, you can use the following formula: Celsius = (Fahrenheit - 32) × 5/9.\n\nSo, to convert 212 degrees Fahrenheit to Celsius, do the following calculation:\n\nCelsius = (212°F - 32) × 5/9\n\nCelsius = (212 - 32)", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "mistral.mistral-7b-instruct-v0:2", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "amazon.titan-text-express-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "884db5c9-18ab-4f27-8892-33656176a2e6", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-text-express-v1", + "response.model": "amazon.titan-text-express-v1", + "response.choices.finish_reason": "FINISH", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "884db5c9-18ab-4f27-8892-33656176a2e6", + "span_id": None, + "trace_id": "trace-id", + "content": "\n\nUser: The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n\nCurrent conversation:\n\nHuman: Hi there!\nAI:\n\nBot:", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "884db5c9-18ab-4f27-8892-33656176a2e6", + "span_id": None, + "trace_id": "trace-id", + "content": " Hello, how can I help you today?", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "anthropic.claude-instant-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "1a72a1f6-310f-469c-af1d-2c59eb600089", + "duration": None, # Response time varies each test run + "request.model": "anthropic.claude-instant-v1", + "response.model": "anthropic.claude-instant-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "1a72a1f6-310f-469c-af1d-2c59eb600089", + "span_id": None, + "trace_id": "trace-id", + "content": "The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n\nCurrent conversation:\n\nHuman: Hi there!\nAI:", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "anthropic.claude-instant-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "1a72a1f6-310f-469c-af1d-2c59eb600089", + "span_id": None, + "trace_id": "trace-id", + "content": "Hello! My name is Claude.\n\nH: Nice to meet you Claude. Can you tell me a bit about yourself?", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "anthropic.claude-instant-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "meta.llama2-13b-chat-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "cce6b34c-812c-4f97-8885-515829aa9639", + "duration": None, # Response time varies each test run + "request.model": "meta.llama2-13b-chat-v1", + "response.model": "meta.llama2-13b-chat-v1", + "response.choices.finish_reason": "stop", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "cce6b34c-812c-4f97-8885-515829aa9639", + "span_id": None, + "trace_id": "trace-id", + "content": "[INST] The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n\nCurrent conversation:\n\nHuman: Hi there!\nAI: [/INST]", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "meta.llama2-13b-chat-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "cce6b34c-812c-4f97-8885-515829aa9639", + "span_id": None, + "trace_id": "trace-id", + "content": " Hello! It's great to talk to you! I'm an AI, and I'm here to help answer any questions you might have. What's on your mind? 🤔 Do you have a specific topic you'd like to discuss, or do you just want to chat? 💬 I'm all ears! 👂", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "meta.llama2-13b-chat-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], +} +chat_completion_langchain_expected_events = { + "mistral.mistral-7b-instruct-v0:2": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "duration": None, # Response time varies each test run + "request.model": "mistral.mistral-7b-instruct-v0:2", + "response.model": "mistral.mistral-7b-instruct-v0:2", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "length", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "span_id": None, + "trace_id": "trace-id", + "content": "[INST] What is 212 degrees Fahrenheit converted to Celsius? [/INST]", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "mistral.mistral-7b-instruct-v0:2", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "span_id": None, + "trace_id": "trace-id", + "content": " To convert a temperature from Fahrenheit to Celsius, you can use the following formula:\n\nCelsius = (Fahrenheit - 32) \u00d7 5/9\n\nSo, to convert 212 degrees Fahrenheit to Celsius:\n\nCelsius = (212 - 32) \u00d7 5/9\nCelsius = 180.56 \u00d7 5/9\nCelsius", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "mistral.mistral-7b-instruct-v0:2", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "amazon.titan-text-express-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "884db5c9-18ab-4f27-8892-33656176a2e6", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-text-express-v1", + "response.model": "amazon.titan-text-express-v1", + "response.choices.finish_reason": "FINISH", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "884db5c9-18ab-4f27-8892-33656176a2e6", + "span_id": None, + "trace_id": "trace-id", + "content": "\n\nUser: The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n\nCurrent conversation:\n\nHuman: Hi there!\nAI:\n\nBot:", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "884db5c9-18ab-4f27-8892-33656176a2e6", + "span_id": None, + "trace_id": "trace-id", + "content": " Hello, how can I help you today?", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "anthropic.claude-instant-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "1a72a1f6-310f-469c-af1d-2c59eb600089", + "duration": None, # Response time varies each test run + "request.model": "anthropic.claude-instant-v1", + "response.model": "anthropic.claude-instant-v1", + "response.choices.finish_reason": "end_turn", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "1a72a1f6-310f-469c-af1d-2c59eb600089", + "span_id": None, + "trace_id": "trace-id", + "content": "The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n\nCurrent conversation:\n\nHuman: Hi there!\nAI:", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "anthropic.claude-instant-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "1a72a1f6-310f-469c-af1d-2c59eb600089", + "span_id": None, + "trace_id": "trace-id", + "content": "{'type': 'text', 'text': \"Hello! It's nice to meet you.\"}", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "anthropic.claude-instant-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "meta.llama2-13b-chat-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "cce6b34c-812c-4f97-8885-515829aa9639", + "duration": None, # Response time varies each test run + "request.model": "meta.llama2-13b-chat-v1", + "response.model": "meta.llama2-13b-chat-v1", + "response.choices.finish_reason": "stop", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "cce6b34c-812c-4f97-8885-515829aa9639", + "span_id": None, + "trace_id": "trace-id", + "content": "[INST] The following is a friendly conversation between a human and an AI. The AI is talkative and provides lots of specific details from its context. If the AI does not know the answer to a question, it truthfully says it does not know.\n\nCurrent conversation:\n\nHuman: Hi there!\nAI: [/INST]", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "meta.llama2-13b-chat-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "cce6b34c-812c-4f97-8885-515829aa9639", + "span_id": None, + "trace_id": "trace-id", + "content": " Hello! It's great to talk to you! I'm an AI, and I'm here to help answer any questions you might have. What's on your mind? 🤔 Do you have a specific topic you'd like to discuss, or is there something you'd like to know? 🤓 I'm all ears! 👂", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "meta.llama2-13b-chat-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], +} + +chat_completion_streaming_expected_events = { + "mistral.mistral-7b-instruct-v0:2": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "duration": None, # Response time varies each test run + "request.model": "mistral.mistral-7b-instruct-v0:2", + "response.model": "mistral.mistral-7b-instruct-v0:2", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "length", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "span_id": None, + "trace_id": "trace-id", + "content": "[INST] What is 212 degrees Fahrenheit converted to Celsius? [/INST]", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "mistral.mistral-7b-instruct-v0:2", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "span_id": None, + "trace_id": "trace-id", + "content": " To convert a temperature from Fahrenheit to Celsius, you can use the following formula: Celsius = (Fahrenheit - 32) × 5/9.\n\nSo, to convert 212 degrees Fahrenheit to Celsius, do the following calculation:\n\nCelsius = (212°F - 32) × 5/9\n\nCelsius = (212 - 32)", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "mistral.mistral-7b-instruct-v0:2", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "amazon.titan-text-express-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "b427270f-371a-458d-81b6-a05aafb2704c", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-text-express-v1", + "response.model": "amazon.titan-text-express-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "FINISH", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "request_id": "b427270f-371a-458d-81b6-a05aafb2704c", + "span_id": None, + "trace_id": "trace-id", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "content": "What is 212 degrees Fahrenheit converted to Celsius?", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "request_id": "b427270f-371a-458d-81b6-a05aafb2704c", + "span_id": None, + "trace_id": "trace-id", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "content": "\n1 degree Fahrenheit is 0.56 degrees Celsius. Therefore, 212 degree Fahrenheit in Celsius would be 115.72.", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "anthropic.claude-instant-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "a645548f-0b3a-47ce-a675-f51e6e9037de", + "duration": None, # Response time varies each test run + "request.model": "anthropic.claude-instant-v1", + "response.model": "anthropic.claude-instant-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "stop_sequence", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "request_id": "a645548f-0b3a-47ce-a675-f51e6e9037de", + "span_id": None, + "trace_id": "trace-id", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "content": "Human: What is 212 degrees Fahrenheit converted to Celsius? Assistant:", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "anthropic.claude-instant-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "request_id": "a645548f-0b3a-47ce-a675-f51e6e9037de", + "span_id": None, + "trace_id": "trace-id", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "content": " Here are the steps to convert 212 degrees Fahrenheit to Celsius:\n\n1) The formula to convert between Fahrenheit and Celsius is:\n C = (F - 32) * 5/9\n\n2) Plug in 212 degrees Fahrenheit for F:\n C = (212 - 32) * 5/9\n C = 180 * 5/9\n C = 100\n\n3) Therefore, 212 degrees Fahrenheit is equal to 100 degrees Celsius.", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "anthropic.claude-instant-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "cohere.command-text-v14": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "4f8ab6c5-42d1-4e35-9573-30f9f41f821e", + "response_id": None, # UUID that varies with each run + "duration": None, # Response time varies each test run + "request.model": "cohere.command-text-v14", + "response.model": "cohere.command-text-v14", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "COMPLETE", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "request_id": "4f8ab6c5-42d1-4e35-9573-30f9f41f821e", + "span_id": None, + "trace_id": "trace-id", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "content": "What is 212 degrees Fahrenheit converted to Celsius?", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "cohere.command-text-v14", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "request_id": "4f8ab6c5-42d1-4e35-9573-30f9f41f821e", + "span_id": None, + "trace_id": "trace-id", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "content": " To convert Fahrenheit to Celsius, you can use the formula:\n\nCelsius = (Fahrenheit - 32) * 5/9\n\nIn this case, if you have 212 degrees Fahrenheit, you can use this formula to calculate the equivalent temperature in Celsius:\n\nCelsius = (212 - 32) * 5/9 = 100 * 5/9 = 50\n\nTherefore, 212 degrees Fahrenheit is equivalent to 50 degrees Celsius.", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "cohere.command-text-v14", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], + "meta.llama2-13b-chat-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "6dd99878-0919-4f92-850c-48f50f923b76", + "duration": None, # Response time varies each test run + "request.model": "meta.llama2-13b-chat-v1", + "response.model": "meta.llama2-13b-chat-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "response.choices.finish_reason": "length", + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 2, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "request_id": "6dd99878-0919-4f92-850c-48f50f923b76", + "span_id": None, + "trace_id": "trace-id", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "content": "What is 212 degrees Fahrenheit converted to Celsius?", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "meta.llama2-13b-chat-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "request_id": "6dd99878-0919-4f92-850c-48f50f923b76", + "span_id": None, + "trace_id": "trace-id", + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "content": " What is the conversion formula?\n\n212 degrees Fahrenheit is equal to 100 degrees Celsius.\n\nThe conversion formula is:\n\n°C = (°F - 32) × 5/9\n\nSo, to convert 212 degrees Fahrenheit to Celsius, we can use the formula like this:\n\n°C = (212 - 32) × 5/9\n", + "role": "assistant", + "completion_id": None, + "sequence": 1, + "response.model": "meta.llama2-13b-chat-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "is_response": True, + }, + ), + ], +} + +chat_completion_invalid_model_error_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "f4908827-3db9-4742-9103-2bbc34578b03", + "span_id": None, + "trace_id": "trace-id", + "duration": None, # Response time varies each test run + "request.model": "does-not-exist", + "response.model": "does-not-exist", + "vendor": "bedrock", + "ingest_source": "Python", + "error": True, + }, + ), +] + +chat_completion_invalid_access_key_error_events = { + "mistral.mistral-7b-instruct-v0:2": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "duration": None, # Response time varies each test run + "request.model": "mistral.mistral-7b-instruct-v0:2", + "response.model": "mistral.mistral-7b-instruct-v0:2", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 1, + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "48c7ee13-7790-461f-959f-04b0a4cf91c8", + "span_id": None, + "trace_id": "trace-id", + "content": "[INST] Invalid Token [/INST]", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "mistral.mistral-7b-instruct-v0:2", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ], + "amazon.titan-text-express-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "15b39c8b-8e85-42c9-9623-06720301bda3", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-text-express-v1", + "response.model": "amazon.titan-text-express-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 1, + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "15b39c8b-8e85-42c9-9623-06720301bda3", + "span_id": None, + "trace_id": "trace-id", + "content": "Invalid Token", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ], + "ai21.j2-mid-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "9021791d-3797-493d-9277-e33aa6f6d544", + "duration": None, # Response time varies each test run + "request.model": "ai21.j2-mid-v1", + "response.model": "ai21.j2-mid-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 1, + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "9021791d-3797-493d-9277-e33aa6f6d544", + "span_id": None, + "trace_id": "trace-id", + "content": "Invalid Token", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "ai21.j2-mid-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ], + "anthropic.claude-instant-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "37396f55-b721-4bae-9461-4c369f5a080d", + "duration": None, # Response time varies each test run + "request.model": "anthropic.claude-instant-v1", + "response.model": "anthropic.claude-instant-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 1, + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "37396f55-b721-4bae-9461-4c369f5a080d", + "span_id": None, + "trace_id": "trace-id", + "content": "Human: Invalid Token Assistant:", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "anthropic.claude-instant-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ], + "cohere.command-text-v14": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "22476490-a0d6-42db-b5ea-32d0b8a7f751", + "duration": None, # Response time varies each test run + "request.model": "cohere.command-text-v14", + "response.model": "cohere.command-text-v14", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 1, + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "22476490-a0d6-42db-b5ea-32d0b8a7f751", + "span_id": None, + "trace_id": "trace-id", + "content": "Invalid Token", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "cohere.command-text-v14", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ], + "meta.llama2-13b-chat-v1": [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "22476490-a0d6-42db-b5ea-32d0b8a7f751", + "duration": None, # Response time varies each test run + "request.model": "meta.llama2-13b-chat-v1", + "response.model": "meta.llama2-13b-chat-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 1, + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "22476490-a0d6-42db-b5ea-32d0b8a7f751", + "span_id": None, + "trace_id": "trace-id", + "content": "Invalid Token", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "meta.llama2-13b-chat-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ], +} + + +chat_completion_expected_malformed_request_body_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "e72d1b46-9f16-4bf0-8eee-f7778f32e5a5", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-text-express-v1", + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + "error": True, + }, + ), +] + +chat_completion_expected_malformed_response_body_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "81508a1c-33a8-4294-8743-f0c629af2f49", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-text-express-v1", + "response.model": "amazon.titan-text-express-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 1, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "81508a1c-33a8-4294-8743-f0c629af2f49", + "span_id": None, + "trace_id": "trace-id", + "content": "Malformed Body", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), +] + +chat_completion_expected_malformed_response_streaming_body_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "a5a8cebb-fd33-4437-8168-5667fbdfc1fb", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-text-express-v1", + "response.model": "amazon.titan-text-express-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 1, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "a5a8cebb-fd33-4437-8168-5667fbdfc1fb", + "span_id": None, + "trace_id": "trace-id", + "content": "Malformed Streaming Body", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), +] + +chat_completion_expected_malformed_response_streaming_chunk_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "request_id": "a5a8cebb-fd33-4437-8168-5667fbdfc1fb", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-text-express-v1", + "response.model": "amazon.titan-text-express-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 1, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "request_id": "a5a8cebb-fd33-4437-8168-5667fbdfc1fb", + "span_id": None, + "trace_id": "trace-id", + "content": "Malformed Streaming Chunk", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), +] + + +chat_completion_expected_streaming_error_events = [ + ( + {"type": "LlmChatCompletionSummary"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-text-express-v1", + "response.model": "amazon.titan-text-express-v1", + "request.temperature": 0.7, + "request.max_tokens": 100, + "vendor": "bedrock", + "ingest_source": "Python", + "response.number_of_messages": 1, + "error": True, + }, + ), + ( + {"type": "LlmChatCompletionMessage"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "content": "Streaming Exception", + "role": "user", + "completion_id": None, + "sequence": 0, + "response.model": "amazon.titan-text-express-v1", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), +] diff --git a/tests/external_botocore/_test_bedrock_embeddings.py b/tests/external_botocore/_test_bedrock_embeddings.py new file mode 100644 index 0000000000..66f609f7bc --- /dev/null +++ b/tests/external_botocore/_test_bedrock_embeddings.py @@ -0,0 +1,200 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +embedding_payload_templates = { + "amazon.titan-embed-text-v1": '{ "inputText": "%s" }', + "amazon.titan-embed-g1-text-02": '{ "inputText": "%s" }', + "cohere.embed-english-v3": '{"texts": ["%s"], "input_type": "search_document"}', +} + +embedding_expected_events = { + "amazon.titan-embed-text-v1": [ + ( + {"type": "LlmEmbedding"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "input": "This is an embedding test.", + "duration": None, # Response time varies each test run + "response.model": "amazon.titan-embed-text-v1", + "request.model": "amazon.titan-embed-text-v1", + "request_id": "11233989-07e8-4ecb-9ba6-79601ba6d8cc", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ], + "amazon.titan-embed-g1-text-02": [ + ( + {"type": "LlmEmbedding"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "input": "This is an embedding test.", + "duration": None, # Response time varies each test run + "response.model": "amazon.titan-embed-g1-text-02", + "request.model": "amazon.titan-embed-g1-text-02", + "request_id": "b10ac895-eae3-4f07-b926-10b2866c55ed", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ], + "cohere.embed-english-v3": [ + ( + {"type": "LlmEmbedding"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "input": "['This is an embedding test.']", + "duration": None, # Response time varies each test run + "response.model": "cohere.embed-english-v3", + "request.model": "cohere.embed-english-v3", + "request_id": "11233989-07e8-4ecb-9ba6-79601ba6d8cc", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), + ], +} + +embedding_invalid_access_key_error_events = { + "amazon.titan-embed-text-v1": [ + ( + {"type": "LlmEmbedding"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "input": "Invalid Token", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-embed-text-v1", + "response.model": "amazon.titan-embed-text-v1", + "request_id": "aece6ad7-e2ff-443b-a953-ba7d385fd0cc", + "vendor": "bedrock", + "ingest_source": "Python", + "error": True, + }, + ), + ], + "amazon.titan-embed-g1-text-02": [ + ( + {"type": "LlmEmbedding"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "input": "Invalid Token", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-embed-g1-text-02", + "response.model": "amazon.titan-embed-g1-text-02", + "request_id": "73328313-506e-4da8-af0f-51017fa6ca3f", + "vendor": "bedrock", + "ingest_source": "Python", + "error": True, + }, + ), + ], + "cohere.embed-english-v3": [ + ( + {"type": "LlmEmbedding"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "input": "['Invalid Token']", + "duration": None, # Response time varies each test run + "request.model": "cohere.embed-english-v3", + "response.model": "cohere.embed-english-v3", + "request_id": "73328313-506e-4da8-af0f-51017fa6ca3f", + "vendor": "bedrock", + "ingest_source": "Python", + "error": True, + }, + ), + ], +} + +embedding_expected_client_errors = { + "amazon.titan-embed-text-v1": { + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, + "amazon.titan-embed-g1-text-02": { + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, + "cohere.embed-english-v3": { + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, +} + +embedding_expected_malformed_request_body_events = [ + ( + {"type": "LlmEmbedding"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-embed-g1-text-02", + "response.model": "amazon.titan-embed-g1-text-02", + "request_id": "b3646569-18c5-4173-a9fa-bbe9c648f636", + "vendor": "bedrock", + "ingest_source": "Python", + "error": True, + }, + ), +] + +embedding_expected_malformed_response_body_events = [ + ( + {"type": "LlmEmbedding"}, + { + "id": None, # UUID that varies with each run + "llm.conversation_id": "my-awesome-id", + "llm.foo": "bar", + "span_id": None, + "trace_id": "trace-id", + "input": "Malformed Body", + "duration": None, # Response time varies each test run + "request.model": "amazon.titan-embed-g1-text-02", + "response.model": "amazon.titan-embed-g1-text-02", + "request_id": "b10ac895-eae3-4f07-b926-10b2866c55ed", + "vendor": "bedrock", + "ingest_source": "Python", + }, + ), +] diff --git a/tests/external_botocore/_test_file.txt b/tests/external_botocore/_test_file.txt new file mode 100644 index 0000000000..c57eff55eb --- /dev/null +++ b/tests/external_botocore/_test_file.txt @@ -0,0 +1 @@ +Hello World! \ No newline at end of file diff --git a/tests/external_botocore/conftest.py b/tests/external_botocore/conftest.py index e5cf155336..08ed863818 100644 --- a/tests/external_botocore/conftest.py +++ b/tests/external_botocore/conftest.py @@ -12,19 +12,170 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io +import json +import os +import re + import pytest +from _mock_external_bedrock_server import ( + MockExternalBedrockServer, + extract_shortened_prompt, +) +from botocore.response import StreamingBody +from testing_support.fixtures import ( # noqa: F401, pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, + override_application_settings, +) -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from newrelic.common.object_wrapper import wrap_function_wrapper +from newrelic.common.package_version_utils import ( + get_package_version, + get_package_version_tuple, +) +from newrelic.common.signature import bind_args +BOTOCORE_VERSION = get_package_version("botocore") _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "custom_insights_events.max_attribute_value": 4096, + "ai_monitoring.enabled": True, } - collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (external_botocore)', - default_settings=_default_settings) + app_name="Python Agent Test (external_botocore)", + default_settings=_default_settings, + linked_applications=["Python Agent Test (external_botocore)"], +) + + +# Bedrock Fixtures +BEDROCK_AUDIT_LOG_FILE = os.path.join(os.path.realpath(os.path.dirname(__file__)), "bedrock_audit.log") +BEDROCK_AUDIT_LOG_CONTENTS = {} + + +@pytest.fixture(scope="session") +def bedrock_server(): + """ + This fixture will either create a mocked backend for testing purposes, or will + set up an audit log file to log responses of the real Bedrock backend to a file. + The behavior can be controlled by setting NEW_RELIC_TESTING_RECORD_BEDROCK_RESPONSES=1 as + an environment variable to run using the real Bedrock backend. (Default: mocking) + """ + import boto3 + + from newrelic.core.config import _environ_as_bool + + if get_package_version_tuple("botocore") < (1, 31, 57): + pytest.skip(reason="Bedrock Runtime not available.") + + if not _environ_as_bool("NEW_RELIC_TESTING_RECORD_BEDROCK_RESPONSES", False): + # Use mocked Bedrock backend and prerecorded responses + with MockExternalBedrockServer() as server: + client = boto3.client( # nosec + "bedrock-runtime", + "us-east-1", + endpoint_url=f"http://localhost:{server.port}", + aws_access_key_id="NOT-A-REAL-SECRET", + aws_secret_access_key="NOT-A-REAL-SECRET", + ) + + yield client + else: + # Use real Bedrock backend and record responses + assert ( + os.environ["AWS_ACCESS_KEY_ID"] and os.environ["AWS_SECRET_ACCESS_KEY"] + ), "AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required." + + # Construct real client + client = boto3.client( + "bedrock-runtime", + "us-east-1", + ) + + # Apply function wrappers to record data + wrap_function_wrapper( + "botocore.endpoint", "Endpoint._do_get_response", wrap_botocore_endpoint_Endpoint__do_get_response + ) + wrap_function_wrapper( + "botocore.eventstream", + "EventStreamBuffer.add_data", + wrap_botocore_eventstream_add_data, + ) + yield client # Run tests + + # Write responses to audit log + bedrock_audit_log_contents = dict(sorted(BEDROCK_AUDIT_LOG_CONTENTS.items(), key=lambda i: (i[1][1], i[0]))) + with open(BEDROCK_AUDIT_LOG_FILE, "w") as audit_log_fp: + json.dump(bedrock_audit_log_contents, fp=audit_log_fp, indent=4) + + +# Intercept outgoing requests and log to file for mocking +RECORDED_HEADERS = set(["x-amzn-requestid", "x-amzn-errortype", "content-type"]) + + +def wrap_botocore_endpoint_Endpoint__do_get_response(wrapped, instance, args, kwargs): + request = bind__do_get_response(*args, **kwargs) + if not request: + return wrapped(*args, **kwargs) + + match = re.search(r"/model/([0-9a-zA-Z%.-]+)/", request.url) + model = match.group(1) + + # Send request + result = wrapped(*args, **kwargs) + + # Unpack response + success, exception = result + response = (success or exception)[0] + + if isinstance(request.body, io.BytesIO): + request.body.seek(0) + body = request.body.read() + else: + body = request.body + + try: + content = json.loads(body) + except Exception: + content = body.decode("utf-8") + + prompt = extract_shortened_prompt(content, model) + headers = dict(response.headers.items()) + headers = dict( + filter( + lambda k: k[0].lower() in RECORDED_HEADERS or k[0].startswith("x-ratelimit"), + headers.items(), + ) + ) + status_code = response.status_code + + # Log response + if response.raw.chunked: + # Log response + BEDROCK_AUDIT_LOG_CONTENTS[prompt] = headers, status_code, [] # Append response data to audit log + else: + # Clean up data + response_content = response.content + data = json.loads(response_content.decode("utf-8")) + result[0][1]["body"] = StreamingBody(io.BytesIO(response_content), len(response_content)) + BEDROCK_AUDIT_LOG_CONTENTS[prompt] = headers, status_code, data # Append response data to audit log + return result + + +def bind__do_get_response(request, operation_model, context): + return request + + +def wrap_botocore_eventstream_add_data(wrapped, instance, args, kwargs): + bound_args = bind_args(wrapped, args, kwargs) + data = bound_args["data"].hex() # convert bytes to hex for storage + prompt = [k for k in BEDROCK_AUDIT_LOG_CONTENTS.keys()][-1] + BEDROCK_AUDIT_LOG_CONTENTS[prompt][2].append(data) + return wrapped(*args, **kwargs) diff --git a/tests/external_botocore/test_bedrock_chat_completion.py b/tests/external_botocore/test_bedrock_chat_completion.py new file mode 100644 index 0000000000..be0226e55c --- /dev/null +++ b/tests/external_botocore/test_bedrock_chat_completion.py @@ -0,0 +1,962 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import os +from io import BytesIO + +import boto3 +import botocore.errorfactory +import botocore.eventstream +import botocore.exceptions +import pytest +from _test_bedrock_chat_completion import ( + chat_completion_expected_events, + chat_completion_expected_malformed_request_body_events, + chat_completion_expected_malformed_response_body_events, + chat_completion_expected_malformed_response_streaming_body_events, + chat_completion_expected_malformed_response_streaming_chunk_events, + chat_completion_expected_streaming_error_events, + chat_completion_invalid_access_key_error_events, + chat_completion_invalid_model_error_events, + chat_completion_payload_templates, + chat_completion_streaming_expected_events, +) +from conftest import BOTOCORE_VERSION # pylint: disable=E0611 +from testing_support.fixtures import ( + override_llm_token_callback_settings, + reset_core_stats_engine, + validate_attributes, +) +from testing_support.ml_testing_utils import ( # noqa: F401 + add_token_count_to_events, + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + disabled_ai_monitoring_streaming_settings, + events_sans_content, + events_sans_llm_metadata, + events_with_context_attrs, + llm_token_count_callback, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import ( + validate_error_trace_attributes, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.api.transaction import add_custom_attribute +from newrelic.common.object_names import callable_name +from newrelic.hooks.external_botocore import MODEL_EXTRACTORS + + +@pytest.fixture(scope="session", params=[False, True], ids=["ResponseStandard", "ResponseStreaming"]) +def response_streaming(request): + return request.param + + +@pytest.fixture(scope="session", params=[False, True], ids=["RequestStandard", "RequestStreaming"]) +def request_streaming(request): + return request.param + + +@pytest.fixture( + scope="module", + params=[ + "amazon.titan-text-express-v1", + "ai21.j2-mid-v1", + "anthropic.claude-instant-v1", + "cohere.command-text-v14", + "meta.llama2-13b-chat-v1", + "mistral.mistral-7b-instruct-v0:2", + ], +) +def model_id(request, response_streaming): + model = request.param + if response_streaming and model == "ai21.j2-mid-v1": + pytest.skip(reason="Streaming not supported.") + + return model + + +@pytest.fixture(scope="module") +def exercise_model(bedrock_server, model_id, request_streaming, response_streaming): + payload_template = chat_completion_payload_templates[model_id] + + def _exercise_model(prompt, temperature=0.7, max_tokens=100): + body = (payload_template % (prompt, temperature, max_tokens)).encode("utf-8") + if request_streaming: + body = BytesIO(body) + + response = bedrock_server.invoke_model( + body=body, + modelId=model_id, + accept="application/json", + contentType="application/json", + ) + response_body = json.loads(response.get("body").read()) + assert response_body + + return response_body + + def _exercise_streaming_model(prompt, temperature=0.7, max_tokens=100): + body = (payload_template % (prompt, temperature, max_tokens)).encode("utf-8") + if request_streaming: + body = BytesIO(body) + + response = bedrock_server.invoke_model_with_response_stream( + body=body, + modelId=model_id, + accept="application/json", + contentType="application/json", + ) + body = response.get("body") + for resp in body: + assert resp + + if response_streaming: + return _exercise_streaming_model + else: + return _exercise_model + + +@pytest.fixture(scope="module") +def expected_events(model_id, response_streaming): + if response_streaming: + return chat_completion_streaming_expected_events[model_id] + else: + return chat_completion_expected_events[model_id] + + +@pytest.fixture(scope="module") +def expected_metrics(response_streaming): + if response_streaming: + return [("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)] + else: + return [("Llm/completion/Bedrock/invoke_model", 1)] + + +@pytest.fixture(scope="module") +def expected_invalid_access_key_error_events(model_id): + return chat_completion_invalid_access_key_error_events[model_id] + + +_test_bedrock_chat_completion_prompt = "What is 212 degrees Fahrenheit converted to Celsius?" + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_in_txn_with_llm_metadata( + set_trace_info, exercise_model, expected_events, expected_metrics +): + @validate_custom_events(events_with_context_attrs(expected_events)) + # One summary event, one user message, and one response message from the assistant + @validate_custom_event_count(count=3) + @validate_transaction_metrics( + name="test_bedrock_chat_completion_in_txn_with_llm_metadata", + scoped_metrics=expected_metrics, + rollup_metrics=expected_metrics, + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @validate_attributes("agent", ["llm"]) + @background_task(name="test_bedrock_chat_completion_in_txn_with_llm_metadata") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + with WithLlmCustomAttributes({"context": "attr"}): + exercise_model(prompt=_test_bedrock_chat_completion_prompt, temperature=0.7, max_tokens=100) + + _test() + + +@disabled_ai_monitoring_record_content_settings +@reset_core_stats_engine() +def test_bedrock_chat_completion_no_content(set_trace_info, exercise_model, expected_events, expected_metrics): + @validate_custom_events(events_sans_content(expected_events)) + # One summary event, one user message, and one response message from the assistant + @validate_custom_event_count(count=3) + @validate_transaction_metrics( + name="test_bedrock_chat_completion_no_content", + scoped_metrics=expected_metrics, + rollup_metrics=expected_metrics, + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @validate_attributes("agent", ["llm"]) + @background_task(name="test_bedrock_chat_completion_no_content") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + exercise_model(prompt=_test_bedrock_chat_completion_prompt, temperature=0.7, max_tokens=100) + + _test() + + +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +def test_bedrock_chat_completion_with_token_count(set_trace_info, exercise_model, expected_events, expected_metrics): + @validate_custom_events(add_token_count_to_events(expected_events)) + # One summary event, one user message, and one response message from the assistant + @validate_custom_event_count(count=3) + @validate_transaction_metrics( + name="test_bedrock_chat_completion_with_token_count", + scoped_metrics=expected_metrics, + rollup_metrics=expected_metrics, + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @validate_attributes("agent", ["llm"]) + @background_task(name="test_bedrock_chat_completion_with_token_count") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + exercise_model(prompt=_test_bedrock_chat_completion_prompt, temperature=0.7, max_tokens=100) + + _test() + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_no_llm_metadata(set_trace_info, exercise_model, expected_events, expected_metrics): + @validate_custom_events(events_sans_llm_metadata(expected_events)) + # One summary event, one user message, and one response message from the assistant + @validate_custom_event_count(count=3) + @validate_transaction_metrics( + name="test_bedrock_chat_completion_in_txn_no_llm_metadata", + scoped_metrics=expected_metrics, + rollup_metrics=expected_metrics, + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion_in_txn_no_llm_metadata") + def _test(): + set_trace_info() + exercise_model(prompt=_test_bedrock_chat_completion_prompt, temperature=0.7, max_tokens=100) + + _test() + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_bedrock_chat_completion_outside_txn(exercise_model): + add_custom_attribute("llm.conversation_id", "my-awesome-id") + exercise_model(prompt=_test_bedrock_chat_completion_prompt, temperature=0.7, max_tokens=100) + + +@disabled_ai_monitoring_settings +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +@background_task(name="test_bedrock_chat_completion_disabled_ai_monitoring_setting") +def test_bedrock_chat_completion_disabled_ai_monitoring_settings(set_trace_info, exercise_model): + set_trace_info() + exercise_model(prompt=_test_bedrock_chat_completion_prompt, temperature=0.7, max_tokens=100) + + +@reset_core_stats_engine() +@disabled_ai_monitoring_streaming_settings +def test_bedrock_chat_completion_streaming_disabled( + bedrock_server, +): + """Streaming is disabled, but the rest of the AI settings are enabled. Custom events should not be collected.""" + + @validate_custom_event_count(count=0) + @validate_transaction_metrics( + name="test_bedrock_chat_completion", + scoped_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], + rollup_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion") + def _test(): + model = "amazon.titan-text-express-v1" + body = (chat_completion_payload_templates[model] % (_test_bedrock_chat_completion_prompt, 0.7, 100)).encode( + "utf-8" + ) + + response = bedrock_server.invoke_model_with_response_stream( + body=body, + modelId=model, + accept="application/json", + contentType="application/json", + ) + list(response["body"]) # Iterate + + _test() + + +_client_error = botocore.exceptions.ClientError +_client_error_name = callable_name(_client_error) + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_error_invalid_model( + bedrock_server, set_trace_info, response_streaming, expected_metrics +): + @validate_custom_events(events_with_context_attrs(chat_completion_invalid_model_error_events)) + @validate_error_trace_attributes( + "botocore.errorfactory:ValidationException", + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 400, + "error.message": "The provided model identifier is invalid.", + "error.code": "ValidationException", + }, + }, + ) + @validate_transaction_metrics( + name="test_bedrock_chat_completion_error_invalid_model", + scoped_metrics=expected_metrics, + rollup_metrics=expected_metrics, + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion_error_invalid_model") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + with pytest.raises(_client_error): + with WithLlmCustomAttributes({"context": "attr"}): + if response_streaming: + stream = bedrock_server.invoke_model_with_response_stream( + body=b"{}", + modelId="does-not-exist", + accept="application/json", + contentType="application/json", + ) + for _ in stream: + pass + else: + bedrock_server.invoke_model( + body=b"{}", + modelId="does-not-exist", + accept="application/json", + contentType="application/json", + ) + + _test() + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_error_incorrect_access_key( + monkeypatch, + bedrock_server, + exercise_model, + set_trace_info, + expected_invalid_access_key_error_events, + expected_metrics, +): + """ + A request is made to the server with invalid credentials. botocore will reach out to the server and receive an + UnrecognizedClientException as a response. Information from the request will be parsed and reported in customer + events. The error response can also be parsed, and will be included as attributes on the recorded exception. + """ + + @validate_custom_events(expected_invalid_access_key_error_events) + @validate_error_trace_attributes( + _client_error_name, + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, + }, + ) + @validate_transaction_metrics( + name="test_bedrock_chat_completion", + scoped_metrics=expected_metrics, + rollup_metrics=expected_metrics, + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion") + def _test(): + monkeypatch.setattr(bedrock_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") + + with pytest.raises(_client_error): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + exercise_model(prompt="Invalid Token", temperature=0.7, max_tokens=100) + + _test() + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +def test_bedrock_chat_completion_error_incorrect_access_key_no_content( + monkeypatch, + bedrock_server, + exercise_model, + set_trace_info, + expected_invalid_access_key_error_events, + expected_metrics, +): + """ + Duplicate of test_bedrock_chat_completion_error_incorrect_access_key, but with content recording disabled. + + See the original test for a description of the error case. + """ + + @validate_custom_events(events_sans_content(expected_invalid_access_key_error_events)) + @validate_error_trace_attributes( + _client_error_name, + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, + }, + ) + @validate_transaction_metrics( + name="test_bedrock_chat_completion", + scoped_metrics=expected_metrics, + rollup_metrics=expected_metrics, + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion") + def _test(): + monkeypatch.setattr(bedrock_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") + + with pytest.raises(_client_error): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + exercise_model(prompt="Invalid Token", temperature=0.7, max_tokens=100) + + _test() + + +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +def test_bedrock_chat_completion_error_incorrect_access_key_with_token( + monkeypatch, + bedrock_server, + exercise_model, + set_trace_info, + expected_invalid_access_key_error_events, + expected_metrics, +): + @validate_custom_events(add_token_count_to_events(expected_invalid_access_key_error_events)) + @validate_error_trace_attributes( + _client_error_name, + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, + }, + ) + @validate_transaction_metrics( + name="test_bedrock_chat_completion", + scoped_metrics=expected_metrics, + rollup_metrics=expected_metrics, + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion") + def _test(): + monkeypatch.setattr(bedrock_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") + + with pytest.raises(_client_error): # not sure where this exception actually comes from + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + exercise_model(prompt="Invalid Token", temperature=0.7, max_tokens=100) + + _test() + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_error_malformed_request_body( + bedrock_server, + set_trace_info, + response_streaming, + expected_metrics, +): + """ + A request was made to the server, but the request body contains invalid JSON. The library will accept the invalid + payload, and still send a request. Our instrumentation will be unable to read it. As a result, no request + information will be recorded in custom events. This includes the initial prompt message event, which cannot be read + so it cannot be captured. The server will then respond with a ValidationException response immediately due to the + bad request. The response can still be parsed, so error information from the response will be recorded as normal. + """ + + @validate_custom_events(chat_completion_expected_malformed_request_body_events) + @validate_custom_event_count(count=1) + @validate_error_trace_attributes( + "botocore.errorfactory:ValidationException", + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 400, + "error.message": "Malformed input request, please reformat your input and try again.", + "error.code": "ValidationException", + }, + }, + ) + @validate_transaction_metrics( + name="test_bedrock_chat_completion", + scoped_metrics=expected_metrics, + rollup_metrics=expected_metrics, + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion") + def _test(): + model = "amazon.titan-text-express-v1" + body = "{ Malformed Request Body".encode("utf-8") + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + with pytest.raises(_client_error): + if response_streaming: + bedrock_server.invoke_model_with_response_stream( + body=body, + modelId=model, + accept="application/json", + contentType="application/json", + ) + else: + bedrock_server.invoke_model( + body=body, + modelId=model, + accept="application/json", + contentType="application/json", + ) + + _test() + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_error_malformed_response_body( + bedrock_server, + set_trace_info, +): + """ + After a non-streaming request was made to the server, the server responded with a response body that contains + invalid JSON. Since the JSON body is not parsed by botocore and just returned to the user as bytes, no parsing + exceptions will be raised. Instrumentation will attempt to parse the invalid body, and should not raise an + exception when it fails to do so. As a result, recorded events will not contain the streamed response data but will contain the request data. + """ + + @validate_custom_events(chat_completion_expected_malformed_response_body_events) + @validate_custom_event_count(count=2) + @validate_transaction_metrics( + name="test_bedrock_chat_completion", + scoped_metrics=[("Llm/completion/Bedrock/invoke_model", 1)], + rollup_metrics=[("Llm/completion/Bedrock/invoke_model", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion") + def _test(): + model = "amazon.titan-text-express-v1" + body = (chat_completion_payload_templates[model] % ("Malformed Body", 0.7, 100)).encode("utf-8") + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + response = bedrock_server.invoke_model( + body=body, + modelId=model, + accept="application/json", + contentType="application/json", + ) + assert response + + _test() + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_error_malformed_response_streaming_body( + bedrock_server, + set_trace_info, +): + """ + A chunk in the stream returned by the server is valid, but contains a body with JSON that cannot be parsed. + Since the JSON body is not parsed by botocore and just returned to the user as bytes, no parsing exceptions will + be raised. Instrumentation will attempt to parse the invalid body, and should not raise an exception when it fails + to do so. The result should be all streamed response data missing from the recorded events, but request and summary + events are recorded as normal. + """ + + @validate_custom_events(chat_completion_expected_malformed_response_streaming_body_events) + @validate_custom_event_count(count=2) + @validate_transaction_metrics( + name="test_bedrock_chat_completion", + scoped_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], + rollup_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion") + def _test(): + model = "amazon.titan-text-express-v1" + body = (chat_completion_payload_templates[model] % ("Malformed Streaming Body", 0.7, 100)).encode("utf-8") + + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + response = bedrock_server.invoke_model_with_response_stream( + body=body, + modelId=model, + accept="application/json", + contentType="application/json", + ) + + chunks = list(response["body"]) + assert chunks, "No response chunks returned" + for chunk in chunks: + with pytest.raises(json.decoder.JSONDecodeError): + json.loads(chunk["chunk"]["bytes"]) + + _test() + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_error_malformed_response_streaming_chunk( + bedrock_server, + set_trace_info, +): + """ + A chunk in the stream returned by the server is missing the prelude which causes an InvalidHeadersLength exception + to be raised during parsing of the chunk. Since the streamed chunk is not able to be parsed, the response + attribute on the raised exception is not present. This means all streamed response data will be missing from the + recorded events. + """ + + @validate_custom_events(chat_completion_expected_malformed_response_streaming_chunk_events) + @validate_custom_event_count(count=2) + @validate_error_trace_attributes( + "botocore.eventstream:ChecksumMismatch", + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "llm.conversation_id": "my-awesome-id", + }, + }, + forgone_params={ + "agent": (), + "intrinsic": (), + "user": ("http.statusCode", "error.message", "error.code"), + }, + ) + @validate_transaction_metrics( + name="test_bedrock_chat_completion", + scoped_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], + rollup_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion") + def _test(): + model = "amazon.titan-text-express-v1" + body = (chat_completion_payload_templates[model] % ("Malformed Streaming Chunk", 0.7, 100)).encode("utf-8") + with pytest.raises(botocore.eventstream.ChecksumMismatch): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + response = bedrock_server.invoke_model_with_response_stream( + body=body, + modelId=model, + accept="application/json", + contentType="application/json", + ) + response = "".join(chunk for chunk in response["body"]) + assert response + + _test() + + +_event_stream_error = botocore.exceptions.EventStreamError +_event_stream_error_name = "botocore.exceptions:EventStreamError" + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_error_streaming_exception( + bedrock_server, + set_trace_info, +): + """ + During a streaming call, the streamed chunk's headers indicate an error. These headers are not HTTP headers, but + headers embedded in the binary format of the response from the server. The streamed chunk's response body is not + required to contain any information regarding the exception, the headers are sufficient to cause botocore's + parser to raise an actual exception based on the error code. The response attribute on the raised exception will + contain the error information. This means error data will be reported for the response, but all response message + data will be missing from the recorded events since the server returned an error instead of message data inside + the streamed response. + """ + + @validate_custom_events(chat_completion_expected_streaming_error_events) + @validate_custom_event_count(count=2) + @validate_error_trace_attributes( + _event_stream_error_name, + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "error.message": "Malformed input request, please reformat your input and try again.", + "error.code": "ValidationException", + }, + }, + forgone_params={ + "agent": (), + "intrinsic": (), + "user": ("http.statusCode"), + }, + ) + @validate_transaction_metrics( + name="test_bedrock_chat_completion", + scoped_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], + rollup_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion") + def _test(): + with pytest.raises(_event_stream_error): + model = "amazon.titan-text-express-v1" + body = (chat_completion_payload_templates[model] % ("Streaming Exception", 0.7, 100)).encode("utf-8") + + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + response = bedrock_server.invoke_model_with_response_stream( + body=body, + modelId=model, + accept="application/json", + contentType="application/json", + ) + list(response["body"]) # Iterate + + _test() + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +def test_bedrock_chat_completion_error_streaming_exception_no_content( + bedrock_server, + set_trace_info, +): + """ + Duplicate of test_bedrock_chat_completion_error_streaming_exception, but with content recording disabled. + + See the original test for a description of the error case. + """ + + @validate_custom_events(events_sans_content(chat_completion_expected_streaming_error_events)) + @validate_custom_event_count(count=2) + @validate_error_trace_attributes( + _event_stream_error_name, + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "error.message": "Malformed input request, please reformat your input and try again.", + "error.code": "ValidationException", + }, + }, + forgone_params={ + "agent": (), + "intrinsic": (), + "user": ("http.statusCode"), + }, + ) + @validate_transaction_metrics( + name="test_bedrock_chat_completion", + scoped_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], + rollup_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion") + def _test(): + with pytest.raises(_event_stream_error): + model = "amazon.titan-text-express-v1" + body = (chat_completion_payload_templates[model] % ("Streaming Exception", 0.7, 100)).encode("utf-8") + + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + response = bedrock_server.invoke_model_with_response_stream( + body=body, + modelId=model, + accept="application/json", + contentType="application/json", + ) + list(response["body"]) # Iterate + + _test() + + +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +def test_bedrock_chat_completion_error_streaming_exception_with_token_count( + bedrock_server, + set_trace_info, +): + """ + Duplicate of test_bedrock_chat_completion_error_streaming_exception, but with token callback being set. + + See the original test for a description of the error case. + """ + + @validate_custom_events(add_token_count_to_events(chat_completion_expected_streaming_error_events)) + @validate_custom_event_count(count=2) + @validate_error_trace_attributes( + _event_stream_error_name, + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "error.message": "Malformed input request, please reformat your input and try again.", + "error.code": "ValidationException", + }, + }, + forgone_params={ + "agent": (), + "intrinsic": (), + "user": ("http.statusCode"), + }, + ) + @validate_transaction_metrics( + name="test_bedrock_chat_completion", + scoped_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], + rollup_metrics=[("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_chat_completion") + def _test(): + with pytest.raises(_event_stream_error): + model = "amazon.titan-text-express-v1" + body = (chat_completion_payload_templates[model] % ("Streaming Exception", 0.7, 100)).encode("utf-8") + + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + response = bedrock_server.invoke_model_with_response_stream( + body=body, + modelId=model, + accept="application/json", + contentType="application/json", + ) + list(response["body"]) # Iterate + + _test() + + +def test_bedrock_chat_completion_functions_marked_as_wrapped_for_sdk_compatibility(bedrock_server): + assert bedrock_server._nr_wrapped + + +def test_chat_models_instrumented(): + SUPPORTED_MODELS = [model for model, _, _, _ in MODEL_EXTRACTORS if "embed" not in model] + + _id = os.environ.get("AWS_ACCESS_KEY_ID") + key = os.environ.get("AWS_SECRET_ACCESS_KEY") + if not _id or not key: + pytest.skip(reason="Credentials not available.") + + client = boto3.client( + "bedrock", + "us-east-1", + ) + response = client.list_foundation_models(byOutputModality="TEXT") + models = [model["modelId"] for model in response["modelSummaries"]] + not_supported = [] + for model in models: + is_supported = any([model.startswith(supported_model) for supported_model in SUPPORTED_MODELS]) + if not is_supported: + not_supported.append(model) + + assert not not_supported, f"The following unsupported models were found: {not_supported}" diff --git a/tests/external_botocore/test_bedrock_chat_completion_via_langchain.py b/tests/external_botocore/test_bedrock_chat_completion_via_langchain.py new file mode 100644 index 0000000000..3bd18764fa --- /dev/null +++ b/tests/external_botocore/test_bedrock_chat_completion_via_langchain.py @@ -0,0 +1,134 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from _test_bedrock_chat_completion import ( + chat_completion_langchain_expected_events, + chat_completion_langchain_expected_streaming_events, +) +from conftest import BOTOCORE_VERSION # pylint: disable=E0611 +from testing_support.fixtures import reset_core_stats_engine, validate_attributes +from testing_support.ml_testing_utils import ( # noqa: F401 + events_with_context_attrs, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.llm_custom_attributes import WithLlmCustomAttributes +from newrelic.api.transaction import add_custom_attribute + +UNSUPPORTED_LANGCHAIN_MODELS = [ + "ai21.j2-mid-v1", + "cohere.command-text-v14", +] + + +@pytest.fixture( + scope="module", + params=[ + "amazon.titan-text-express-v1", + "ai21.j2-mid-v1", + "anthropic.claude-instant-v1", + "cohere.command-text-v14", + "meta.llama2-13b-chat-v1", + ], +) +def model_id(request): + model = request.param + if model in UNSUPPORTED_LANGCHAIN_MODELS: + pytest.skip(reason="Not supported by Langchain.") + return model + + +@pytest.fixture(scope="session", params=[False, True], ids=["ResponseStandard", "ResponseStreaming"]) +def response_streaming(request): + return request.param + + +@pytest.fixture(scope="module") +def exercise_model(bedrock_server, model_id, response_streaming): + try: + # These are only available in certain botocore environments. + from langchain.chains import ConversationChain + from langchain_community.chat_models import BedrockChat + except ImportError: + pytest.skip(reason="Langchain not installed.") + + def _exercise_model(prompt): + bedrock_llm = BedrockChat( + model_id=model_id, + client=bedrock_server, + streaming=response_streaming, + ) + conversation = ConversationChain(llm=bedrock_llm) + result = conversation.predict(input=prompt) + if response_streaming: + for r in result: + assert r + else: + assert result + + return _exercise_model + + +@pytest.fixture(scope="module") +def expected_events(model_id, response_streaming): + if response_streaming: + return chat_completion_langchain_expected_streaming_events[model_id] + return chat_completion_langchain_expected_events[model_id] + + +@pytest.fixture(scope="module") +def expected_metrics(response_streaming): + if response_streaming: + return [("Llm/completion/Bedrock/invoke_model_with_response_stream", 1)] + return [("Llm/completion/Bedrock/invoke_model", 1)] + + +@reset_core_stats_engine() +def test_bedrock_chat_completion_in_txn_with_llm_metadata( + set_trace_info, + exercise_model, + expected_events, + expected_metrics, + response_streaming, +): + @validate_custom_events(events_with_context_attrs(expected_events)) + # One summary event, one user message, and one response message from the assistant + @validate_custom_event_count(count=6) + @validate_transaction_metrics( + name="test_bedrock_chat_completion_in_txn_with_llm_metadata", + scoped_metrics=expected_metrics, + rollup_metrics=expected_metrics, + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @validate_attributes("agent", ["llm"]) + @background_task(name="test_bedrock_chat_completion_in_txn_with_llm_metadata") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + with WithLlmCustomAttributes({"context": "attr"}): + exercise_model(prompt="Hi there!") + + _test() diff --git a/tests/external_botocore/test_bedrock_embeddings.py b/tests/external_botocore/test_bedrock_embeddings.py new file mode 100644 index 0000000000..8ed17fa4f7 --- /dev/null +++ b/tests/external_botocore/test_bedrock_embeddings.py @@ -0,0 +1,485 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import json +import os +from io import BytesIO + +import boto3 +import botocore.exceptions +import pytest +from _test_bedrock_embeddings import ( + embedding_expected_events, + embedding_expected_malformed_request_body_events, + embedding_expected_malformed_response_body_events, + embedding_invalid_access_key_error_events, + embedding_payload_templates, +) +from conftest import BOTOCORE_VERSION # pylint: disable=E0611 +from testing_support.fixtures import ( + override_llm_token_callback_settings, + reset_core_stats_engine, + validate_attributes, +) +from testing_support.ml_testing_utils import ( # noqa: F401 + add_token_count_to_events, + disabled_ai_monitoring_record_content_settings, + disabled_ai_monitoring_settings, + events_sans_content, + events_sans_llm_metadata, + llm_token_count_callback, + set_trace_info, +) +from testing_support.validators.validate_custom_event import validate_custom_event_count +from testing_support.validators.validate_custom_events import validate_custom_events +from testing_support.validators.validate_error_trace_attributes import ( + validate_error_trace_attributes, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.api.transaction import add_custom_attribute +from newrelic.common.object_names import callable_name +from newrelic.hooks.external_botocore import MODEL_EXTRACTORS + + +@pytest.fixture(scope="session", params=[False, True], ids=["RequestStandard", "RequestStreaming"]) +def request_streaming(request): + return request.param + + +@pytest.fixture( + scope="module", + params=[ + "amazon.titan-embed-text-v1", + "amazon.titan-embed-g1-text-02", + "cohere.embed-english-v3", + ], +) +def model_id(request): + return request.param + + +@pytest.fixture(scope="module") +def exercise_model(bedrock_server, model_id, request_streaming): + payload_template = embedding_payload_templates[model_id] + + def _exercise_model(prompt): + body = (payload_template % prompt).encode("utf-8") + if request_streaming: + body = BytesIO(body) + + response = bedrock_server.invoke_model( + body=body, + modelId=model_id, + accept="application/json", + contentType="application/json", + ) + response_body = json.loads(response.get("body").read()) + assert response_body + + return response_body + + return _exercise_model + + +@pytest.fixture(scope="module") +def expected_events(model_id): + return embedding_expected_events[model_id] + + +@pytest.fixture(scope="module") +def expected_invalid_access_key_error_events(model_id): + return embedding_invalid_access_key_error_events[model_id] + + +_test_bedrock_embedding_prompt = "This is an embedding test." + + +@reset_core_stats_engine() +def test_bedrock_embedding_with_llm_metadata(set_trace_info, exercise_model, expected_events): + @validate_custom_events(expected_events) + @validate_custom_event_count(count=1) + @validate_transaction_metrics( + name="test_bedrock_embedding", + scoped_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + rollup_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @validate_attributes("agent", ["llm"]) + @background_task(name="test_bedrock_embedding") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + exercise_model(prompt=_test_bedrock_embedding_prompt) + + _test() + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +def test_bedrock_embedding_no_content(set_trace_info, exercise_model, model_id): + @validate_custom_events(events_sans_content(embedding_expected_events[model_id])) + @validate_custom_event_count(count=1) + @validate_transaction_metrics( + name="test_bedrock_embedding", + scoped_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + rollup_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @validate_attributes("agent", ["llm"]) + @background_task(name="test_bedrock_embedding") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + exercise_model(prompt=_test_bedrock_embedding_prompt) + + _test() + + +@reset_core_stats_engine() +def test_bedrock_embedding_no_llm_metadata(set_trace_info, exercise_model, expected_events): + @validate_custom_events(events_sans_llm_metadata(expected_events)) + @validate_custom_event_count(count=1) + @validate_transaction_metrics( + name="test_bedrock_embedding_no_llm_metadata", + scoped_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + rollup_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_embedding_no_llm_metadata") + def _test(): + set_trace_info() + exercise_model(prompt=_test_bedrock_embedding_prompt) + + _test() + + +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +def test_bedrock_embedding_with_token_count(set_trace_info, exercise_model, expected_events): + @validate_custom_events(add_token_count_to_events(expected_events)) + @validate_custom_event_count(count=1) + @validate_transaction_metrics( + name="test_bedrock_embedding", + scoped_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + rollup_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @validate_attributes("agent", ["llm"]) + @background_task(name="test_bedrock_embedding") + def _test(): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + + exercise_model(prompt="This is an embedding test.") + + _test() + + +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +def test_bedrock_embedding_outside_txn(exercise_model): + add_custom_attribute("llm.conversation_id", "my-awesome-id") + exercise_model(prompt=_test_bedrock_embedding_prompt) + + +@disabled_ai_monitoring_settings +@reset_core_stats_engine() +@validate_custom_event_count(count=0) +@background_task(name="test_bedrock_embedding_disabled_ai_monitoring_setting") +def test_bedrock_embedding_disabled_ai_monitoring_settings(set_trace_info, exercise_model): + set_trace_info() + exercise_model(prompt=_test_bedrock_embedding_prompt) + + +_client_error = botocore.exceptions.ClientError +_client_error_name = callable_name(_client_error) + + +@reset_core_stats_engine() +def test_bedrock_embedding_error_incorrect_access_key( + monkeypatch, + bedrock_server, + exercise_model, + set_trace_info, + expected_invalid_access_key_error_events, +): + """ + A request is made to the server with invalid credentials. botocore will reach out to the server and receive an + UnrecognizedClientException as a response. Information from the request will be parsed and reported in customer + events. The error response can also be parsed, and will be included as attributes on the recorded exception. + """ + + @validate_custom_events(expected_invalid_access_key_error_events) + @validate_error_trace_attributes( + _client_error_name, + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, + }, + ) + @validate_transaction_metrics( + name="test_bedrock_embedding", + scoped_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + rollup_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_embedding") + def _test(): + monkeypatch.setattr(bedrock_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") + + with pytest.raises(_client_error): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + exercise_model(prompt="Invalid Token") + + _test() + + +@reset_core_stats_engine() +@disabled_ai_monitoring_record_content_settings +def test_bedrock_embedding_error_incorrect_access_key_no_content( + monkeypatch, + bedrock_server, + exercise_model, + set_trace_info, + expected_invalid_access_key_error_events, +): + @validate_custom_events(events_sans_content(expected_invalid_access_key_error_events)) + @validate_error_trace_attributes( + _client_error_name, + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, + }, + ) + @validate_transaction_metrics( + name="test_bedrock_embedding", + scoped_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + rollup_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + background_task=True, + ) + @background_task(name="test_bedrock_embedding") + def _test(): + monkeypatch.setattr(bedrock_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") + + with pytest.raises(_client_error): + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + exercise_model(prompt="Invalid Token") + + _test() + + +@reset_core_stats_engine() +@override_llm_token_callback_settings(llm_token_count_callback) +def test_bedrock_embedding_error_incorrect_access_key_with_token_count( + monkeypatch, + bedrock_server, + exercise_model, + set_trace_info, + expected_invalid_access_key_error_events, +): + @validate_custom_events(add_token_count_to_events(expected_invalid_access_key_error_events)) + @validate_error_trace_attributes( + _client_error_name, + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 403, + "error.message": "The security token included in the request is invalid.", + "error.code": "UnrecognizedClientException", + }, + }, + ) + @validate_transaction_metrics( + name="test_bedrock_embedding", + scoped_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + rollup_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + background_task=True, + ) + @background_task(name="test_bedrock_embedding") + def _test(): + monkeypatch.setattr(bedrock_server._request_signer._credentials, "access_key", "INVALID-ACCESS-KEY") + + with pytest.raises(_client_error): # not sure where this exception actually comes from + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + exercise_model(prompt="Invalid Token") + + _test() + + +@reset_core_stats_engine() +def test_bedrock_embedding_error_malformed_request_body( + bedrock_server, + set_trace_info, +): + """ + A request was made to the server, but the request body contains invalid JSON. The library will accept the invalid + payload, and still send a request. Our instrumentation will be unable to read it. As a result, no request + information will be recorded in custom events. This includes the initial prompt message event, which cannot be read + so it cannot be captured. The server will then respond with a ValidationException response immediately due to the + bad request. The response can still be parsed, so error information from the response will be recorded as normal. + """ + + @validate_custom_events(embedding_expected_malformed_request_body_events) + @validate_custom_event_count(count=1) + @validate_error_trace_attributes( + "botocore.errorfactory:ValidationException", + exact_attrs={ + "agent": {}, + "intrinsic": {}, + "user": { + "http.statusCode": 400, + "error.message": "Malformed input request, please reformat your input and try again.", + "error.code": "ValidationException", + }, + }, + ) + @validate_transaction_metrics( + name="test_bedrock_embedding", + scoped_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + rollup_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_embedding") + def _test(): + model = "amazon.titan-embed-g1-text-02" + body = "{ Malformed Request Body".encode("utf-8") + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + with pytest.raises(_client_error): + bedrock_server.invoke_model( + body=body, + modelId=model, + accept="application/json", + contentType="application/json", + ) + + _test() + + +@reset_core_stats_engine() +def test_bedrock_embedding_error_malformed_response_body( + bedrock_server, + set_trace_info, +): + """ + After a non-streaming request was made to the server, the server responded with a response body that contains + invalid JSON. Since the JSON body is not parsed by botocore and just returned to the user as bytes, no parsing + exceptions will be raised. Instrumentation will attempt to parse the invalid body, and should not raise an + exception when it fails to do so. As a result, recorded events will not contain the streamed response data but will contain the request data. + """ + + @validate_custom_events(embedding_expected_malformed_response_body_events) + @validate_custom_event_count(count=1) + @validate_transaction_metrics( + name="test_bedrock_embedding", + scoped_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + rollup_metrics=[("Llm/embedding/Bedrock/invoke_model", 1)], + custom_metrics=[ + (f"Supportability/Python/ML/Bedrock/{BOTOCORE_VERSION}", 1), + ], + background_task=True, + ) + @background_task(name="test_bedrock_embedding") + def _test(): + model = "amazon.titan-embed-g1-text-02" + body = (embedding_payload_templates[model] % "Malformed Body").encode("utf-8") + set_trace_info() + add_custom_attribute("llm.conversation_id", "my-awesome-id") + add_custom_attribute("llm.foo", "bar") + add_custom_attribute("non_llm_attr", "python-agent") + + response = bedrock_server.invoke_model( + body=body, + modelId=model, + accept="application/json", + contentType="application/json", + ) + assert response + + _test() + + +def test_embedding_models_instrumented(): + SUPPORTED_MODELS = [model for model, _, _, _ in MODEL_EXTRACTORS if "embed" in model] + + _id = os.environ.get("AWS_ACCESS_KEY_ID") + key = os.environ.get("AWS_SECRET_ACCESS_KEY") + if not _id or not key: + pytest.skip(reason="Credentials not available.") + + client = boto3.client( + "bedrock", + "us-east-1", + ) + response = client.list_foundation_models(byOutputModality="EMBEDDING") + models = [model["modelId"] for model in response["modelSummaries"]] + not_supported = [] + for model in models: + is_supported = any([model.startswith(supported_model) for supported_model in SUPPORTED_MODELS]) + if not is_supported: + not_supported.append(model) + + assert not not_supported, f"The following unsupported models were found: {not_supported}" diff --git a/tests/external_botocore/test_boto3_iam.py b/tests/external_botocore/test_boto3_iam.py new file mode 100644 index 0000000000..ae1f5e466f --- /dev/null +++ b/tests/external_botocore/test_boto3_iam.py @@ -0,0 +1,83 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid + +import boto3 +from moto import mock_aws +from testing_support.fixtures import dt_enabled +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple + +MOTO_VERSION = get_package_version_tuple("moto") + +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec (This is fine for testing purposes) + +TEST_USER = f"python-agent-test-{uuid.uuid4()}" + +_iam_scoped_metrics = [ + ("External/iam.amazonaws.com/botocore/POST", 3), +] + +_iam_rollup_metrics = [ + ("External/all", 3), + ("External/allOther", 3), + ("External/iam.amazonaws.com/all", 3), + ("External/iam.amazonaws.com/botocore/POST", 3), +] + + +@dt_enabled +@validate_span_events(exact_agents={"http.url": "https://iam.amazonaws.com/"}, count=3) +@validate_span_events(expected_agents=("aws.requestId",), count=3) +@validate_span_events(exact_agents={"aws.operation": "CreateUser"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "GetUser"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteUser"}, count=1) +@validate_tt_segment_params(present_params=("aws.requestId",)) +@validate_transaction_metrics( + "test_boto3_iam:test_iam", + scoped_metrics=_iam_scoped_metrics, + rollup_metrics=_iam_rollup_metrics, + background_task=True, +) +@background_task() +@mock_aws +def test_iam(): + iam = boto3.client( + "iam", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + ) + + # Create user + resp = iam.create_user(UserName=TEST_USER) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Get the user + resp = iam.get_user(UserName=TEST_USER) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert resp["User"]["UserName"] == TEST_USER + + # Delete the user + resp = iam.delete_user(UserName=TEST_USER) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 diff --git a/tests/external_boto3/test_boto3_s3.py b/tests/external_botocore/test_boto3_s3.py similarity index 72% rename from tests/external_boto3/test_boto3_s3.py rename to tests/external_botocore/test_boto3_s3.py index a7ecf034ab..e2c57b2eb2 100644 --- a/tests/external_boto3/test_boto3_s3.py +++ b/tests/external_botocore/test_boto3_s3.py @@ -12,68 +12,60 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import uuid import boto3 import botocore -import moto -from testing_support.fixtures import override_application_settings +from moto import mock_aws +from testing_support.fixtures import dt_enabled from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple -MOTO_VERSION = tuple(int(v) for v in moto.__version__.split(".")[:3]) - -# patch earlier versions of moto to support py37 -if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1): - import re - - moto.packages.responses.responses.re._pattern_type = re.Pattern +MOTO_VERSION = get_package_version_tuple("moto") +BOTOCORE_VERSION = get_package_version_tuple("botocore") AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec AWS_REGION_NAME = "us-west-2" -TEST_BUCKET = "python-agent-test-%s" % uuid.uuid4() - -BOTOCORE_VERSION = tuple(map(int, botocore.__version__.split("."))) - +TEST_BUCKET = f"python-agent-test-{uuid.uuid4()}" if BOTOCORE_VERSION < (1, 7, 41): S3_URL = "s3-us-west-2.amazonaws.com" - EXPECTED_BUCKET_URL = "https://%s/%s" % (S3_URL, TEST_BUCKET) - EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "/hello_world" + EXPECTED_BUCKET_URL = f"https://{S3_URL}/{TEST_BUCKET}" + EXPECTED_KEY_URL = f"{EXPECTED_BUCKET_URL}/hello_world" elif BOTOCORE_VERSION < (1, 28): S3_URL = "s3.us-west-2.amazonaws.com" - EXPECTED_BUCKET_URL = "https://%s/%s" % (S3_URL, TEST_BUCKET) - EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "/hello_world" + EXPECTED_BUCKET_URL = f"https://{S3_URL}/{TEST_BUCKET}" + EXPECTED_KEY_URL = f"{EXPECTED_BUCKET_URL}/hello_world" else: - S3_URL = "%s.s3.us-west-2.amazonaws.com" % TEST_BUCKET - EXPECTED_BUCKET_URL = "https://%s/" % S3_URL - EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "hello_world" + S3_URL = f"{TEST_BUCKET}.s3.us-west-2.amazonaws.com" + EXPECTED_BUCKET_URL = f"https://{S3_URL}/" + EXPECTED_KEY_URL = f"{EXPECTED_BUCKET_URL}hello_world" _s3_scoped_metrics = [ - ("External/%s/botocore/GET" % S3_URL, 2), - ("External/%s/botocore/PUT" % S3_URL, 2), - ("External/%s/botocore/DELETE" % S3_URL, 2), + (f"External/{S3_URL}/botocore/GET", 2), + (f"External/{S3_URL}/botocore/PUT", 2), + (f"External/{S3_URL}/botocore/DELETE", 2), ] _s3_rollup_metrics = [ ("External/all", 6), ("External/allOther", 6), - ("External/%s/all" % S3_URL, 6), - ("External/%s/botocore/GET" % S3_URL, 2), - ("External/%s/botocore/PUT" % S3_URL, 2), - ("External/%s/botocore/DELETE" % S3_URL, 2), + (f"External/{S3_URL}/all", 6), + (f"External/{S3_URL}/botocore/GET", 2), + (f"External/{S3_URL}/botocore/PUT", 2), + (f"External/{S3_URL}/botocore/DELETE", 2), ] -@override_application_settings({"distributed_tracing.enabled": True}) +@dt_enabled @validate_span_events(exact_agents={"aws.operation": "CreateBucket"}, count=1) @validate_span_events(exact_agents={"aws.operation": "PutObject"}, count=1) @validate_span_events(exact_agents={"aws.operation": "ListObjects"}, count=1) @@ -86,7 +78,7 @@ "test_boto3_s3:test_s3", scoped_metrics=_s3_scoped_metrics, rollup_metrics=_s3_rollup_metrics, background_task=True ) @background_task() -@moto.mock_s3 +@mock_aws def test_s3(): client = boto3.client( "s3", diff --git a/tests/external_botocore/test_boto3_sns.py b/tests/external_botocore/test_boto3_sns.py new file mode 100644 index 0000000000..141d675670 --- /dev/null +++ b/tests/external_botocore/test_boto3_sns.py @@ -0,0 +1,95 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import boto3 +import pytest +from moto import mock_aws +from testing_support.fixtures import dt_enabled +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple + +MOTO_VERSION = get_package_version_tuple("moto") +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec (This is fine for testing purposes) +AWS_REGION_NAME = "us-east-1" +SNS_URL = "sns-us-east-1.amazonaws.com" +TOPIC = "arn:aws:sns:us-east-1:123456789012:some-topic" +sns_metrics = [(f"MessageBroker/SNS/Topic/Produce/Named/{TOPIC}", 1)] +sns_metrics_phone = [("MessageBroker/SNS/Topic" "/Produce/Named/PhoneNumber", 1)] + + +@dt_enabled +@validate_span_events(expected_agents=("aws.requestId",), count=2) +@validate_span_events(exact_agents={"aws.operation": "CreateTopic"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Publish"}, count=1) +@validate_tt_segment_params(present_params=("aws.requestId",)) +@pytest.mark.parametrize("topic_argument", ("TopicArn", "TargetArn")) +@validate_transaction_metrics( + "test_boto3_sns:test_publish_to_sns_topic", + scoped_metrics=sns_metrics, + rollup_metrics=sns_metrics, + background_task=True, +) +@background_task() +@mock_aws +def test_publish_to_sns_topic(topic_argument): + conn = boto3.client( + "sns", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + region_name=AWS_REGION_NAME, + ) + + topic_arn = conn.create_topic(Name="some-topic")["TopicArn"] + + kwargs = {topic_argument: topic_arn} + published_message = conn.publish(Message="my msg", **kwargs) + assert "MessageId" in published_message + + +@dt_enabled +@validate_span_events(expected_agents=("aws.requestId",), count=3) +@validate_span_events(exact_agents={"aws.operation": "CreateTopic"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Subscribe"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Publish"}, count=1) +@validate_tt_segment_params(present_params=("aws.requestId",)) +@validate_transaction_metrics( + "test_boto3_sns:test_publish_to_sns_phone", + scoped_metrics=sns_metrics_phone, + rollup_metrics=sns_metrics_phone, + background_task=True, +) +@background_task() +@mock_aws +def test_publish_to_sns_phone(): + conn = boto3.client( + "sns", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + region_name=AWS_REGION_NAME, + ) + + topic_arn = conn.create_topic(Name="some-topic")["TopicArn"] + conn.subscribe(TopicArn=topic_arn, Protocol="sms", Endpoint="5555555555") + + published_message = conn.publish(PhoneNumber="5555555555", Message="my msg") + assert "MessageId" in published_message diff --git a/tests/external_botocore/test_botocore_dynamodb.py b/tests/external_botocore/test_botocore_dynamodb.py index 44862d827d..c031f543f6 100644 --- a/tests/external_botocore/test_botocore_dynamodb.py +++ b/tests/external_botocore/test_botocore_dynamodb.py @@ -12,96 +12,93 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import uuid import botocore.session -import moto +from moto import mock_aws +from testing_support.fixtures import dt_enabled +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.background_task import background_task -from testing_support.fixtures import ( - validate_tt_segment_params, override_application_settings) -from testing_support.validators.validate_span_events import ( - validate_span_events) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from newrelic.common.package_version_utils import get_package_version_tuple -MOTO_VERSION = tuple(int(v) for v in moto.__version__.split('.')[:3]) +MOTO_VERSION = get_package_version_tuple("moto") +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec (This is fine for testing purposes) +AWS_REGION = "us-east-1" -# patch earlier versions of moto to support py37 -if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1): - import re - moto.packages.responses.responses.re._pattern_type = re.Pattern - - -AWS_ACCESS_KEY_ID = 'AAAAAAAAAAAACCESSKEY' -AWS_SECRET_ACCESS_KEY = 'AAAAAASECRETKEY' -AWS_REGION = 'us-east-1' - -TEST_TABLE = 'python-agent-test-%s' % uuid.uuid4() +TEST_TABLE = f"python-agent-test-{uuid.uuid4()}" _dynamodb_scoped_metrics = [ - ('Datastore/statement/DynamoDB/%s/create_table' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/put_item' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/get_item' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/update_item' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/query' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/scan' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/delete_item' % TEST_TABLE, 1), - ('Datastore/statement/DynamoDB/%s/delete_table' % TEST_TABLE, 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/create_table", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/put_item", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/get_item", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/update_item", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/query", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/scan", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/delete_item", 1), + (f"Datastore/statement/DynamoDB/{TEST_TABLE}/delete_table", 1), ] _dynamodb_rollup_metrics = [ - ('Datastore/all', 8), - ('Datastore/allOther', 8), - ('Datastore/DynamoDB/all', 8), - ('Datastore/DynamoDB/allOther', 8), + ("Datastore/all", 8), + ("Datastore/allOther", 8), + ("Datastore/DynamoDB/all", 8), + ("Datastore/DynamoDB/allOther", 8), ] -@override_application_settings({'distributed_tracing.enabled': True}) -@validate_span_events(expected_agents=('aws.requestId',), count=8) -@validate_span_events(exact_agents={'aws.operation': 'PutItem'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'GetItem'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'DeleteItem'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'CreateTable'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'DeleteTable'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'Query'}, count=1) -@validate_span_events(exact_agents={'aws.operation': 'Scan'}, count=1) -@validate_tt_segment_params(present_params=('aws.requestId',)) +@dt_enabled +@validate_span_events(expected_agents=("aws.requestId",), count=8) +@validate_span_events(exact_agents={"aws.operation": "PutItem"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "GetItem"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteItem"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "CreateTable"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DeleteTable"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Query"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "Scan"}, count=1) +@validate_tt_segment_params(present_params=("aws.requestId",)) @validate_transaction_metrics( - 'test_botocore_dynamodb:test_dynamodb', - scoped_metrics=_dynamodb_scoped_metrics, - rollup_metrics=_dynamodb_rollup_metrics, - background_task=True) + "test_botocore_dynamodb:test_dynamodb", + scoped_metrics=_dynamodb_scoped_metrics, + rollup_metrics=_dynamodb_rollup_metrics, + background_task=True, +) @background_task() -@moto.mock_dynamodb2 +@mock_aws def test_dynamodb(): session = botocore.session.get_session() client = session.create_client( - 'dynamodb', - region_name=AWS_REGION, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY + "dynamodb", + region_name=AWS_REGION, + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, ) # Create table resp = client.create_table( - TableName=TEST_TABLE, - AttributeDefinitions=[ - {'AttributeName': 'Id', 'AttributeType': 'N'}, - {'AttributeName': 'Foo', 'AttributeType': 'S'}, - ], - KeySchema=[ - {'AttributeName': 'Id', 'KeyType': 'HASH'}, - {'AttributeName': 'Foo', 'KeyType': 'RANGE'}, - ], - ProvisionedThroughput={ - 'ReadCapacityUnits': 5, - 'WriteCapacityUnits': 5, - }, + TableName=TEST_TABLE, + AttributeDefinitions=[ + {"AttributeName": "Id", "AttributeType": "N"}, + {"AttributeName": "Foo", "AttributeType": "S"}, + ], + KeySchema=[ + {"AttributeName": "Id", "KeyType": "HASH"}, + {"AttributeName": "Foo", "KeyType": "RANGE"}, + ], + ProvisionedThroughput={ + "ReadCapacityUnits": 5, + "WriteCapacityUnits": 5, + }, ) - assert resp['TableDescription']['TableName'] == TEST_TABLE + assert resp["TableDescription"]["TableName"] == TEST_TABLE # moto response is ACTIVE, AWS response is CREATING # assert resp['TableDescription']['TableStatus'] == 'ACTIVE' @@ -111,73 +108,68 @@ def test_dynamodb(): # Put item resp = client.put_item( - TableName=TEST_TABLE, - Item={ - 'Id': {'N': '101'}, - 'Foo': {'S': 'hello_world'}, - 'SomeValue': {'S': 'some_random_attribute'}, - } + TableName=TEST_TABLE, + Item={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + "SomeValue": {"S": "some_random_attribute"}, + }, ) # No checking response, due to inconsistent return values. # moto returns resp['Attributes']. AWS returns resp['ResponseMetadata'] # Get item resp = client.get_item( - TableName=TEST_TABLE, - Key={ - 'Id': {'N': '101'}, - 'Foo': {'S': 'hello_world'}, - 'SomeValue': {'S': 'some_random_attribute'}, - } + TableName=TEST_TABLE, + Key={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + }, ) - assert resp['Item']['SomeValue']['S'] == 'some_random_attribute' + assert resp["Item"]["SomeValue"]["S"] == "some_random_attribute" # Update item resp = client.update_item( - TableName=TEST_TABLE, - Key={ - 'Id': {'N': '101'}, - 'Foo': {'S': 'hello_world'}, - 'SomeValue': {'S': 'some_random_attribute'}, - }, - AttributeUpdates={ - 'Foo2': { - 'Value': {'S': 'hello_world2'}, - 'Action': 'PUT' - }, - }, - ReturnValues='ALL_NEW', + TableName=TEST_TABLE, + Key={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + }, + AttributeUpdates={ + "Foo2": {"Value": {"S": "hello_world2"}, "Action": "PUT"}, + }, + ReturnValues="ALL_NEW", ) - assert resp['Attributes']['Foo2'] + assert resp["Attributes"]["Foo2"] # Query for item resp = client.query( - TableName=TEST_TABLE, - Select='ALL_ATTRIBUTES', - KeyConditionExpression='#Id = :v_id', - ExpressionAttributeNames={'#Id': 'Id'}, - ExpressionAttributeValues={':v_id': {'N': '101'}}, + TableName=TEST_TABLE, + Select="ALL_ATTRIBUTES", + KeyConditionExpression="#Id = :v_id", + ExpressionAttributeNames={"#Id": "Id"}, + ExpressionAttributeValues={":v_id": {"N": "101"}}, ) - assert len(resp['Items']) == 1 - assert resp['Items'][0]['SomeValue']['S'] == 'some_random_attribute' + assert len(resp["Items"]) == 1 + assert resp["Items"][0]["SomeValue"]["S"] == "some_random_attribute" # Scan resp = client.scan(TableName=TEST_TABLE) - assert len(resp['Items']) == 1 + assert len(resp["Items"]) == 1 # Delete item resp = client.delete_item( - TableName=TEST_TABLE, - Key={ - 'Id': {'N': '101'}, - 'Foo': {'S': 'hello_world'}, - }, + TableName=TEST_TABLE, + Key={ + "Id": {"N": "101"}, + "Foo": {"S": "hello_world"}, + }, ) # No checking response, due to inconsistent return values. # moto returns resp['Attributes']. AWS returns resp['ResponseMetadata'] # Delete table resp = client.delete_table(TableName=TEST_TABLE) - assert resp['TableDescription']['TableName'] == TEST_TABLE + assert resp["TableDescription"]["TableName"] == TEST_TABLE # moto response is ACTIVE, AWS response is DELETING # assert resp['TableDescription']['TableStatus'] == 'DELETING' diff --git a/tests/external_botocore/test_botocore_ec2.py b/tests/external_botocore/test_botocore_ec2.py index 0cfd09b6fd..4154f61a26 100644 --- a/tests/external_botocore/test_botocore_ec2.py +++ b/tests/external_botocore/test_botocore_ec2.py @@ -12,86 +12,79 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import uuid import botocore.session -import moto +from moto import mock_aws +from testing_support.fixtures import dt_enabled +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.background_task import background_task -from testing_support.fixtures import ( - validate_tt_segment_params, override_application_settings) -from testing_support.validators.validate_span_events import ( - validate_span_events) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from newrelic.common.package_version_utils import get_package_version_tuple -MOTO_VERSION = tuple(int(v) for v in moto.__version__.split('.')[:3]) +MOTO_VERSION = get_package_version_tuple("moto") +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec (This is fine for testing purposes) +AWS_REGION = "us-east-1" +UBUNTU_14_04_PARAVIRTUAL_AMI = "ami-c65be9ae" -# patch earlier versions of moto to support py37 -if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1): - import re - moto.packages.responses.responses.re._pattern_type = re.Pattern - -AWS_ACCESS_KEY_ID = 'AAAAAAAAAAAACCESSKEY' -AWS_SECRET_ACCESS_KEY = 'AAAAAASECRETKEY' -AWS_REGION = 'us-east-1' -UBUNTU_14_04_PARAVIRTUAL_AMI = 'ami-c65be9ae' - -TEST_INSTANCE = 'python-agent-test-%s' % uuid.uuid4() +TEST_INSTANCE = f"python-agent-test-{uuid.uuid4()}" _ec2_scoped_metrics = [ - ('External/ec2.us-east-1.amazonaws.com/botocore/POST', 3), + ("External/ec2.us-east-1.amazonaws.com/botocore/POST", 3), ] _ec2_rollup_metrics = [ - ('External/all', 3), - ('External/allOther', 3), - ('External/ec2.us-east-1.amazonaws.com/all', 3), - ('External/ec2.us-east-1.amazonaws.com/botocore/POST', 3), + ("External/all", 3), + ("External/allOther", 3), + ("External/ec2.us-east-1.amazonaws.com/all", 3), + ("External/ec2.us-east-1.amazonaws.com/botocore/POST", 3), ] -@override_application_settings({'distributed_tracing.enabled': True}) -@validate_span_events(expected_agents=('aws.requestId',), count=3) -@validate_span_events(exact_agents={'aws.operation': 'RunInstances'}, count=1) -@validate_span_events( - exact_agents={'aws.operation': 'DescribeInstances'}, count=1) -@validate_span_events( - exact_agents={'aws.operation': 'TerminateInstances'}, count=1) -@validate_tt_segment_params(present_params=('aws.requestId',)) +@dt_enabled +@validate_span_events(expected_agents=("aws.requestId",), count=3) +@validate_span_events(exact_agents={"aws.operation": "RunInstances"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "DescribeInstances"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "TerminateInstances"}, count=1) +@validate_tt_segment_params(present_params=("aws.requestId",)) @validate_transaction_metrics( - 'test_botocore_ec2:test_ec2', - scoped_metrics=_ec2_scoped_metrics, - rollup_metrics=_ec2_rollup_metrics, - background_task=True) + "test_botocore_ec2:test_ec2", + scoped_metrics=_ec2_scoped_metrics, + rollup_metrics=_ec2_rollup_metrics, + background_task=True, +) @background_task() -@moto.mock_ec2 +@mock_aws def test_ec2(): session = botocore.session.get_session() client = session.create_client( - 'ec2', - region_name=AWS_REGION, - aws_access_key_id=AWS_ACCESS_KEY_ID, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY + "ec2", region_name=AWS_REGION, aws_access_key_id=AWS_ACCESS_KEY_ID, aws_secret_access_key=AWS_SECRET_ACCESS_KEY ) # Create instance resp = client.run_instances( - ImageId=UBUNTU_14_04_PARAVIRTUAL_AMI, - InstanceType='m1.small', - MinCount=1, - MaxCount=1, + ImageId=UBUNTU_14_04_PARAVIRTUAL_AMI, + InstanceType="m1.small", + MinCount=1, + MaxCount=1, ) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 - assert len(resp['Instances']) == 1 - instance_id = resp['Instances'][0]['InstanceId'] + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert len(resp["Instances"]) == 1 + instance_id = resp["Instances"][0]["InstanceId"] # Describe instance resp = client.describe_instances(InstanceIds=[instance_id]) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 - assert resp['Reservations'][0]['Instances'][0]['InstanceId'] == instance_id + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert resp["Reservations"][0]["Instances"][0]["InstanceId"] == instance_id # Delete instance resp = client.terminate_instances(InstanceIds=[instance_id]) - assert resp['ResponseMetadata']['HTTPStatusCode'] == 200 - assert resp['TerminatingInstances'][0]['InstanceId'] == instance_id + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + assert resp["TerminatingInstances"][0]["InstanceId"] == instance_id diff --git a/tests/external_botocore/test_botocore_s3.py b/tests/external_botocore/test_botocore_s3.py index 1984d8103e..2805c343bd 100644 --- a/tests/external_botocore/test_botocore_s3.py +++ b/tests/external_botocore/test_botocore_s3.py @@ -12,62 +12,55 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import uuid import botocore import botocore.session -import moto -from testing_support.fixtures import override_application_settings +from moto import mock_aws +from testing_support.fixtures import dt_enabled from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple -MOTO_VERSION = tuple(int(v) for v in moto.__version__.split(".")[:3]) -BOTOCORE_VERSION = tuple(int(v) for v in botocore.__version__.split(".")[:3]) - - -# patch earlier versions of moto to support py37 -if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1): - import re - - moto.packages.responses.responses.re._pattern_type = re.Pattern +MOTO_VERSION = MOTO_VERSION = get_package_version_tuple("moto") +BOTOCORE_VERSION = get_package_version_tuple("botocore") AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec AWS_REGION = "us-east-1" -TEST_BUCKET = "python-agent-test-%s" % uuid.uuid4() +TEST_BUCKET = f"python-agent-test-{uuid.uuid4()}" if BOTOCORE_VERSION >= (1, 28): - S3_URL = "%s.s3.amazonaws.com" % TEST_BUCKET - EXPECTED_BUCKET_URL = "https://%s/" % S3_URL - EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "hello_world" + S3_URL = f"{TEST_BUCKET}.s3.amazonaws.com" + EXPECTED_BUCKET_URL = f"https://{S3_URL}/" + EXPECTED_KEY_URL = f"{EXPECTED_BUCKET_URL}hello_world" else: S3_URL = "s3.amazonaws.com" - EXPECTED_BUCKET_URL = "https://%s/%s" % (S3_URL, TEST_BUCKET) - EXPECTED_KEY_URL = EXPECTED_BUCKET_URL + "/hello_world" + EXPECTED_BUCKET_URL = f"https://{S3_URL}/{TEST_BUCKET}" + EXPECTED_KEY_URL = f"{EXPECTED_BUCKET_URL}/hello_world" _s3_scoped_metrics = [ - ("External/%s/botocore/GET" % S3_URL, 2), - ("External/%s/botocore/PUT" % S3_URL, 2), - ("External/%s/botocore/DELETE" % S3_URL, 2), + (f"External/{S3_URL}/botocore/GET", 2), + (f"External/{S3_URL}/botocore/PUT", 2), + (f"External/{S3_URL}/botocore/DELETE", 2), ] _s3_rollup_metrics = [ ("External/all", 6), ("External/allOther", 6), - ("External/%s/all" % S3_URL, 6), - ("External/%s/botocore/GET" % S3_URL, 2), - ("External/%s/botocore/PUT" % S3_URL, 2), - ("External/%s/botocore/DELETE" % S3_URL, 2), + (f"External/{S3_URL}/all", 6), + (f"External/{S3_URL}/botocore/GET", 2), + (f"External/{S3_URL}/botocore/PUT", 2), + (f"External/{S3_URL}/botocore/DELETE", 2), ] -@override_application_settings({"distributed_tracing.enabled": True}) +@dt_enabled @validate_span_events(exact_agents={"aws.operation": "CreateBucket"}, count=1) @validate_span_events(exact_agents={"aws.operation": "PutObject"}, count=1) @validate_span_events(exact_agents={"aws.operation": "ListObjects"}, count=1) @@ -83,7 +76,7 @@ background_task=True, ) @background_task() -@moto.mock_s3 +@mock_aws def test_s3(): session = botocore.session.get_session() client = session.create_client( diff --git a/tests/external_botocore/test_botocore_sqs.py b/tests/external_botocore/test_botocore_sqs.py index 3f7d8c0220..b3a6e17578 100644 --- a/tests/external_botocore/test_botocore_sqs.py +++ b/tests/external_botocore/test_botocore_sqs.py @@ -12,53 +12,89 @@ # See the License for the specific language governing permissions and # limitations under the License. -import sys import uuid import botocore.session -import moto import pytest -from testing_support.fixtures import override_application_settings +from moto import mock_aws +from testing_support.fixtures import dt_enabled from testing_support.validators.validate_span_events import validate_span_events from testing_support.validators.validate_transaction_metrics import ( validate_transaction_metrics, ) from newrelic.api.background_task import background_task -from newrelic.common.package_version_utils import get_package_version +from newrelic.common.package_version_utils import get_package_version_tuple -MOTO_VERSION = tuple(int(v) for v in moto.__version__.split(".")[:3]) - -# patch earlier versions of moto to support py37 -if sys.version_info >= (3, 7) and MOTO_VERSION <= (1, 3, 1): - import re - - moto.packages.responses.responses.re._pattern_type = re.Pattern +MOTO_VERSION = get_package_version_tuple("moto") +BOTOCORE_VERSION = get_package_version_tuple("botocore") url = "sqs.us-east-1.amazonaws.com" -botocore_version = tuple([int(n) for n in get_package_version("botocore").split(".")]) -if botocore_version < (1, 29, 0): +EXPECTED_SEND_MESSAGE_AGENT_ATTRS = { + "expected_agents": ["messaging.destination.name"], + "exact_agents": { + "aws.operation": "SendMessage", + "cloud.account.id": "123456789012", + "cloud.region": "us-east-1", + "messaging.system": "aws_sqs", + }, +} +EXPECTED_RECIEVE_MESSAGE_AGENT_ATTRS = { + "expected_agents": ["messaging.destination.name"], + "exact_agents": { + "aws.operation": "ReceiveMessage", + "cloud.account.id": "123456789012", + "cloud.region": "us-east-1", + "messaging.system": "aws_sqs", + }, +} +EXPECTED_SEND_MESSAGE_BATCH_AGENT_ATTRS = required = { + "expected_agents": ["messaging.destination.name"], + "exact_agents": { + "aws.operation": "SendMessageBatch", + "cloud.account.id": "123456789012", + "cloud.region": "us-east-1", + "messaging.system": "aws_sqs", + }, +} +if BOTOCORE_VERSION < (1, 29, 0): url = "queue.amazonaws.com" + # The old style url does not contain the necessary AWS info. + EXPECTED_SEND_MESSAGE_AGENT_ATTRS = { + "exact_agents": { + "aws.operation": "SendMessage", + }, + } + EXPECTED_RECIEVE_MESSAGE_AGENT_ATTRS = { + "exact_agents": { + "aws.operation": "ReceiveMessage", + }, + } + EXPECTED_SEND_MESSAGE_BATCH_AGENT_ATTRS = { + "exact_agents": { + "aws.operation": "SendMessageBatch", + }, + } AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec AWS_REGION = "us-east-1" -TEST_QUEUE = "python-agent-test-%s" % uuid.uuid4() +TEST_QUEUE = f"python-agent-test-{uuid.uuid4()}" _sqs_scoped_metrics = [ - ("MessageBroker/SQS/Queue/Produce/Named/%s" % TEST_QUEUE, 2), - ("External/%s/botocore/POST" % url, 3), + (f"MessageBroker/SQS/Queue/Produce/Named/{TEST_QUEUE}", 2), + (f"External/{url}/botocore/POST", 3), ] _sqs_rollup_metrics = [ - ("MessageBroker/SQS/Queue/Produce/Named/%s" % TEST_QUEUE, 2), - ("MessageBroker/SQS/Queue/Consume/Named/%s" % TEST_QUEUE, 1), + (f"MessageBroker/SQS/Queue/Produce/Named/{TEST_QUEUE}", 2), + (f"MessageBroker/SQS/Queue/Consume/Named/{TEST_QUEUE}", 1), ("External/all", 3), ("External/allOther", 3), - ("External/%s/all" % url, 3), - ("External/%s/botocore/POST" % url, 3), + (f"External/{url}/all", 3), + (f"External/{url}/botocore/POST", 3), ] _sqs_scoped_metrics_malformed = [ @@ -70,11 +106,20 @@ ] -@override_application_settings({"distributed_tracing.enabled": True}) +@dt_enabled @validate_span_events(exact_agents={"aws.operation": "CreateQueue"}, count=1) -@validate_span_events(exact_agents={"aws.operation": "SendMessage"}, count=1) -@validate_span_events(exact_agents={"aws.operation": "ReceiveMessage"}, count=1) -@validate_span_events(exact_agents={"aws.operation": "SendMessageBatch"}, count=1) +@validate_span_events( + **EXPECTED_SEND_MESSAGE_AGENT_ATTRS, + count=1, +) +@validate_span_events( + **EXPECTED_RECIEVE_MESSAGE_AGENT_ATTRS, + count=1, +) +@validate_span_events( + **EXPECTED_SEND_MESSAGE_BATCH_AGENT_ATTRS, + count=1, +) @validate_span_events(exact_agents={"aws.operation": "PurgeQueue"}, count=1) @validate_span_events(exact_agents={"aws.operation": "DeleteQueue"}, count=1) @validate_transaction_metrics( @@ -84,7 +129,7 @@ background_task=True, ) @background_task() -@moto.mock_sqs +@mock_aws def test_sqs(): session = botocore.session.get_session() client = session.create_client( @@ -124,7 +169,7 @@ def test_sqs(): assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 -@override_application_settings({"distributed_tracing.enabled": True}) +@dt_enabled @validate_transaction_metrics( "test_botocore_sqs:test_sqs_malformed", scoped_metrics=_sqs_scoped_metrics_malformed, @@ -132,7 +177,7 @@ def test_sqs(): background_task=True, ) @background_task() -@moto.mock_sqs +@mock_aws def test_sqs_malformed(): session = botocore.session.get_session() client = session.create_client( diff --git a/tests/external_botocore/test_s3transfer.py b/tests/external_botocore/test_s3transfer.py new file mode 100644 index 0000000000..2503d0ba1c --- /dev/null +++ b/tests/external_botocore/test_s3transfer.py @@ -0,0 +1,84 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import uuid + +import boto3 +import botocore +from moto import mock_aws +from testing_support.fixtures import dt_enabled +from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple + +MOTO_VERSION = get_package_version_tuple("moto") +BOTOCORE_VERSION = get_package_version_tuple("botocore") + +AWS_ACCESS_KEY_ID = "AAAAAAAAAAAACCESSKEY" +AWS_SECRET_ACCESS_KEY = "AAAAAASECRETKEY" # nosec +AWS_REGION_NAME = "us-west-2" + +TEST_BUCKET = f"python-agent-test-{uuid.uuid4()}" + +if BOTOCORE_VERSION < (1, 7, 41): + S3_URL = "s3-us-west-2.amazonaws.com" + EXPECTED_BUCKET_URL = f"https://{S3_URL}/{TEST_BUCKET}" + EXPECTED_KEY_URL = f"{EXPECTED_BUCKET_URL}/hello_world" +elif BOTOCORE_VERSION < (1, 28): + S3_URL = "s3.us-west-2.amazonaws.com" + EXPECTED_BUCKET_URL = f"https://{S3_URL}/{TEST_BUCKET}" + EXPECTED_KEY_URL = f"{EXPECTED_BUCKET_URL}/hello_world" +else: + S3_URL = f"{TEST_BUCKET}.s3.us-west-2.amazonaws.com" + EXPECTED_BUCKET_URL = f"https://{S3_URL}/" + EXPECTED_KEY_URL = f"{EXPECTED_BUCKET_URL}hello_world" + + +@dt_enabled +@validate_span_events(exact_agents={"aws.operation": "CreateBucket"}, count=1) +@validate_span_events(exact_agents={"aws.operation": "PutObject"}, count=1) +@validate_transaction_metrics( + "test_s3transfer:test_s3_context_propagation", + scoped_metrics=[ + (f"External/{S3_URL}/botocore/PUT", 2), + ], + rollup_metrics=[ + ("External/all", 2), + ("External/allOther", 2), + (f"External/{S3_URL}/all", 2), + (f"External/{S3_URL}/botocore/PUT", 2), + ], + background_task=True, +) +@background_task() +@mock_aws +def test_s3_context_propagation(): + client = boto3.client( + "s3", + aws_access_key_id=AWS_ACCESS_KEY_ID, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + region_name=AWS_REGION_NAME, + ) + + # Create bucket + resp = client.create_bucket(Bucket=TEST_BUCKET, CreateBucketConfiguration={"LocationConstraint": AWS_REGION_NAME}) + assert resp["ResponseMetadata"]["HTTPStatusCode"] == 200 + + # Upload file + client.upload_file(Filename="_test_file.txt", Bucket=TEST_BUCKET, Key="_test_file.txt") + # No return value to check for this function currently diff --git a/tests/external_feedparser/conftest.py b/tests/external_feedparser/conftest.py index 11d19f1cd5..136cfa8787 100644 --- a/tests/external_feedparser/conftest.py +++ b/tests/external_feedparser/conftest.py @@ -13,21 +13,24 @@ # limitations under the License. import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) from testing_support.mock_external_http_server import MockExternalHTTPServer _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (external_feedparser)', - default_settings=_default_settings) + app_name="Python Agent Test (external_feedparser)", default_settings=_default_settings +) def create_handler(response): @@ -35,8 +38,10 @@ def handler(self): self.send_response(200) self.end_headers() self.wfile.write(response) + return handler + @pytest.fixture(scope="session") def server(): with open("packages.xml", "rb") as f: diff --git a/tests/external_feedparser/test_feedparser.py b/tests/external_feedparser/test_feedparser.py index 5e515cfc30..36833dff41 100644 --- a/tests/external_feedparser/test_feedparser.py +++ b/tests/external_feedparser/test_feedparser.py @@ -29,17 +29,17 @@ def feedparser(): "feed://localhost", )) def test_feedparser_external(feedparser, server, url): - url = url + ':' + str(server.port) + url = f"{url}:{str(server.port)}" @validate_transaction_metrics( "test_feedparser_external", background_task=True, - scoped_metrics=(("External/localhost:%d/feedparser/GET" % server.port, 1),), + scoped_metrics=((f"External/localhost:{server.port}/feedparser/GET", 1),), ) @background_task(name="test_feedparser_external") def _test(): feed = feedparser.parse(url) - assert feed["feed"]["link"] == u"https://pypi.org/" + assert feed["feed"]["link"] == "https://pypi.org/" _test() @@ -50,7 +50,7 @@ def test_feedparser_file(feedparser, stream, server): @validate_transaction_metrics( "test_feedparser_file", background_task=True, - scoped_metrics=(("External/localhost:%d/feedparser/GET" % server.port, None),), + scoped_metrics=((f"External/localhost:{server.port}/feedparser/GET", None),), ) @background_task(name="test_feedparser_file") def _test(): @@ -59,7 +59,7 @@ def _test(): feed = feedparser.parse(f) else: feed = feedparser.parse("packages.xml") - assert feed["feed"]["link"] == u"https://pypi.org/" + assert feed["feed"]["link"] == "https://pypi.org/" _test() @@ -70,6 +70,6 @@ def _test(): )) def test_feedparser_no_transaction(feedparser, server, url): if url.startswith('http://'): - url = url + ':' + str(server.port) + url = f"{url}:{str(server.port)}" feed = feedparser.parse(url) - assert feed["feed"]["link"] == u"https://pypi.org/" + assert feed["feed"]["link"] == "https://pypi.org/" diff --git a/tests/external_http/conftest.py b/tests/external_http/conftest.py index f8afb49f3b..5d18222499 100644 --- a/tests/external_http/conftest.py +++ b/tests/external_http/conftest.py @@ -13,25 +13,29 @@ # limitations under the License. import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) from testing_support.mock_external_http_server import ( - MockExternalHTTPHResponseHeadersServer) - + MockExternalHTTPHResponseHeadersServer, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (external_http)', - default_settings=_default_settings) + app_name="Python Agent Test (external_http)", default_settings=_default_settings +) + -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def server(): with MockExternalHTTPHResponseHeadersServer() as _server: yield _server diff --git a/tests/external_http/test_http.py b/tests/external_http/test_http.py index e08518f5ff..d3ff94c1ea 100644 --- a/tests/external_http/test_http.py +++ b/tests/external_http/test_http.py @@ -13,44 +13,36 @@ # limitations under the License. import pytest -import six from testing_support.external_fixtures import ( cache_outgoing_headers, insert_incoming_headers, ) -from testing_support.fixtures import ( - cat_enabled, - override_application_settings, -) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.fixtures import cat_enabled, override_application_settings from testing_support.validators.validate_cross_process_headers import ( validate_cross_process_headers, ) from testing_support.validators.validate_external_node_params import ( validate_external_node_params, ) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from newrelic.api.background_task import background_task -if six.PY2: - import httplib -else: - import http.client as httplib +import http.client @pytest.fixture(scope="session") def metrics(server): - if six.PY2: - _external_metric = "External/localhost:%s/httplib/" % server.port - else: - _external_metric = "External/localhost:%s/http/" % server.port + _external_metric = f"External/localhost:{server.port}/http/" scoped = [(_external_metric, 1)] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%s/all" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), (_external_metric, 1), ] @@ -63,7 +55,7 @@ def test_http_http_request(server, metrics): ) @background_task(name="test_http:test_http_http_request") def _test(): - connection = httplib.HTTPConnection("localhost", server.port) + connection = http.client.HTTPConnection("localhost", server.port) connection.request("GET", "/") response = connection.getresponse() response.read() @@ -78,7 +70,7 @@ def test_http_https_request(server, metrics): ) @background_task(name="test_http:test_http_https_request") def _test(): - connection = httplib.HTTPSConnection("localhost", server.port) + connection = http.client.HTTPSConnection("localhost", server.port) try: connection.request("GET", "/") except Exception: @@ -101,7 +93,7 @@ def test_http_cross_process_request(distributed_tracing, span_events, server): @cache_outgoing_headers @validate_cross_process_headers def _test(): - connection = httplib.HTTPConnection("localhost", server.port) + connection = http.client.HTTPConnection("localhost", server.port) connection.request("GET", "/") response = connection.getresponse() response.read() @@ -120,14 +112,14 @@ def _test(): @cat_enabled def test_http_cross_process_response(server): - _test_http_cross_process_response_scoped_metrics = [("ExternalTransaction/localhost:%s/1#2/test" % server.port, 1)] + _test_http_cross_process_response_scoped_metrics = [(f"ExternalTransaction/localhost:{server.port}/1#2/test", 1)] _test_http_cross_process_response_rollup_metrics = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%s/all" % server.port, 1), - ("ExternalApp/localhost:%s/1#2/all" % server.port, 1), - ("ExternalTransaction/localhost:%s/1#2/test" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"ExternalApp/localhost:{server.port}/1#2/all", 1), + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1), ] _test_http_cross_process_response_external_node_params = [ @@ -146,7 +138,7 @@ def test_http_cross_process_response(server): @validate_external_node_params(params=_test_http_cross_process_response_external_node_params) @background_task(name="test_http:test_http_cross_process_response") def _test(): - connection = httplib.HTTPConnection("localhost", server.port) + connection = http.client.HTTPConnection("localhost", server.port) connection.request("GET", "/") response = connection.getresponse() response.read() diff --git a/tests/external_httplib/conftest.py b/tests/external_httplib/conftest.py index 2edbeab911..729523599a 100644 --- a/tests/external_httplib/conftest.py +++ b/tests/external_httplib/conftest.py @@ -13,26 +13,29 @@ # limitations under the License. import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) from testing_support.mock_external_http_server import ( - MockExternalHTTPHResponseHeadersServer) - + MockExternalHTTPHResponseHeadersServer, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (external_httplib)', - default_settings=_default_settings) + app_name="Python Agent Test (external_httplib)", default_settings=_default_settings +) -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def server(): with MockExternalHTTPHResponseHeadersServer() as _server: yield _server diff --git a/tests/external_httplib/test_httplib.py b/tests/external_httplib/test_httplib.py index c7747f8ff7..634bb61731 100644 --- a/tests/external_httplib/test_httplib.py +++ b/tests/external_httplib/test_httplib.py @@ -14,21 +14,13 @@ import pytest -try: - import http.client as httplib -except ImportError: - import httplib +import http.client as httplib from testing_support.external_fixtures import ( cache_outgoing_headers, insert_incoming_headers, ) -from testing_support.fixtures import ( - cat_enabled, - override_application_settings, - validate_tt_segment_params, -) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.fixtures import cat_enabled, override_application_settings from testing_support.validators.validate_cross_process_headers import ( validate_cross_process_headers, ) @@ -36,32 +28,27 @@ validate_external_node_params, ) from testing_support.validators.validate_span_events import validate_span_events +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.background_task import background_task from newrelic.common.encoding_utils import DistributedTracePayload -from newrelic.packages import six - - -def select_python_version(py2, py3): - return six.PY3 and py3 or py2 def test_httplib_http_request(server): scoped = [ - select_python_version( - py2=("External/localhost:%d/httplib/" % server.port, 1), - py3=("External/localhost:%d/http/" % server.port, 1), - ) + (f"External/localhost:{server.port}/http/", 1), ] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - select_python_version( - py2=("External/localhost:%d/httplib/" % server.port, 1), - py3=("External/localhost:%d/http/" % server.port, 1), - ), + (f"External/localhost:{server.port}/all", 1), + (f"External/localhost:{server.port}/http/", 1), ] @validate_transaction_metrics( @@ -80,20 +67,14 @@ def _test(): def test_httplib_https_request(server): _test_httplib_https_request_scoped_metrics = [ - select_python_version( - py2=("External/localhost:%d/httplib/" % server.port, 1), - py3=("External/localhost:%d/http/" % server.port, 1), - ) + (f"External/localhost:{server.port}/http/", 1), ] _test_httplib_https_request_rollup_metrics = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - select_python_version( - py2=("External/localhost:%d/httplib/" % server.port, 1), - py3=("External/localhost:%d/http/" % server.port, 1), - ), + (f"External/localhost:{server.port}/all", 1), + (f"External/localhost:{server.port}/http/", 1), ] @validate_transaction_metrics( @@ -104,7 +85,8 @@ def test_httplib_https_request(server): ) @background_task(name="test_httplib:test_httplib_https_request") def _test(): - connection = httplib.HTTPSConnection("localhost", server.port) + # fix HTTPSConnection: https://wiki.openstack.org/wiki/OSSN/OSSN-0033 + connection = httplib.HTTPSConnection("localhost", server.port) # nosec # It doesn't matter that a SSL exception is raised here because the # agent still records this as an external request try: @@ -119,20 +101,14 @@ def _test(): def test_httplib_http_with_port_request(server): scoped = [ - select_python_version( - py2=("External/localhost:%d/httplib/" % server.port, 1), - py3=("External/localhost:%d/http/" % server.port, 1), - ) + (f"External/localhost:{server.port}/http/", 1), ] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - select_python_version( - py2=("External/localhost:%d/httplib/" % server.port, 1), - py3=("External/localhost:%d/http/" % server.port, 1), - ), + (f"External/localhost:{server.port}/all", 1), + (f"External/localhost:{server.port}/http/", 1), ] @validate_transaction_metrics( @@ -192,14 +168,14 @@ def _test(): @cat_enabled @insert_incoming_headers def test_httplib_cross_process_response(server): - scoped = [("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1)] + scoped = [(f"ExternalTransaction/localhost:{server.port}/1#2/test", 1)] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("ExternalApp/localhost:%d/1#2/all" % server.port, 1), - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"ExternalApp/localhost:{server.port}/1#2/all", 1), + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1), ] @validate_transaction_metrics( @@ -224,14 +200,14 @@ def _test(): def test_httplib_multiple_requests_cross_process_response(server): connection = httplib.HTTPConnection("localhost", server.port) - scoped = [("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1)] + scoped = [(f"ExternalTransaction/localhost:{server.port}/1#2/test", 1)] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("ExternalApp/localhost:%d/1#2/all" % server.port, 1), - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"ExternalApp/localhost:{server.port}/1#2/all", 1), + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1), ] @validate_transaction_metrics( @@ -324,17 +300,15 @@ def test_span_events(server): "span_events.enabled": True, } - uri = "http://localhost:%d" % server.port + uri = f"http://localhost:{server.port}" exact_intrinsics = { - "name": select_python_version( - py2="External/localhost:%d/httplib/" % server.port, py3="External/localhost:%d/http/" % server.port - ), + "name": f"External/localhost:{server.port}/http/", "type": "Span", "sampled": True, "category": "http", "span.kind": "client", - "component": select_python_version(py2="httplib", py3="http"), + "component": "http", } exact_agents = { "http.url": uri, diff --git a/tests/external_httplib/test_urllib.py b/tests/external_httplib/test_urllib.py index cea88a8dde..90ee87d079 100644 --- a/tests/external_httplib/test_urllib.py +++ b/tests/external_httplib/test_urllib.py @@ -39,13 +39,13 @@ @pytest.fixture(scope="session") def metrics(server): - scoped = [("External/localhost:%d/urllib/" % server.port, 1)] + scoped = [(f"External/localhost:{server.port}/urllib/", 1)] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("External/localhost:%d/urllib/" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"External/localhost:{server.port}/urllib/", 1), ] return scoped, rollup @@ -61,7 +61,7 @@ def test_urlopener_http_request(server, metrics): @background_task(name="test_urllib:test_urlopener_http_request") def _test(): opener = urllib.URLopener() - opener.open("http://localhost:%d/" % server.port) + opener.open(f"http://localhost:{server.port}/") _test() @@ -77,7 +77,7 @@ def test_urlopener_https_request(server, metrics): def _test(): opener = urllib.URLopener() try: - opener.open("https://localhost:%d/" % server.port) + opener.open(f"https://localhost:{server.port}/") except Exception: pass @@ -85,13 +85,13 @@ def _test(): def test_urlopener_http_request_with_port(server): - scoped = [("External/localhost:%d/urllib/" % server.port, 1)] + scoped = [(f"External/localhost:{server.port}/urllib/", 1)] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("External/localhost:%d/urllib/" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"External/localhost:{server.port}/urllib/", 1), ] @validate_transaction_metrics( @@ -103,7 +103,7 @@ def test_urlopener_http_request_with_port(server): @background_task(name="test_urllib:test_urlopener_http_request_with_port") def _test(): opener = urllib.URLopener() - opener.open("http://localhost:%d/" % server.port) + opener.open(f"http://localhost:{server.port}/") _test() @@ -135,21 +135,21 @@ def test_urlopener_file_request(): @validate_cross_process_headers def test_urlopener_cross_process_request(server): opener = urllib.URLopener() - opener.open("http://localhost:%d/" % server.port) + opener.open(f"http://localhost:{server.port}/") @cat_enabled def test_urlopener_cross_process_response(server): _test_urlopener_cross_process_response_scoped_metrics = [ - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1) + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1) ] _test_urlopener_cross_process_response_rollup_metrics = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("ExternalApp/localhost:%d/1#2/all" % server.port, 1), - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"ExternalApp/localhost:{server.port}/1#2/all", 1), + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1), ] _test_urlopener_cross_process_response_external_node_params = [ @@ -169,7 +169,7 @@ def test_urlopener_cross_process_response(server): @background_task(name="test_urllib:test_urlopener_cross_process_response") def _test(): opener = urllib.URLopener() - opener.open("http://localhost:%d/" % server.port) + opener.open(f"http://localhost:{server.port}/") _test() @@ -183,7 +183,7 @@ def test_urlretrieve_http_request(server, metrics): ) @background_task(name="test_urllib:test_urlretrieve_http_request") def _test(): - urllib.urlretrieve("http://localhost:%d/" % server.port) + urllib.urlretrieve(f"http://localhost:{server.port}/") _test() @@ -198,7 +198,7 @@ def test_urlretrieve_https_request(server, metrics): @background_task(name="test_urllib:test_urlretrieve_https_request") def _test(): try: - urllib.urlretrieve("https://localhost:%d/" % server.port) + urllib.urlretrieve(f"https://localhost:{server.port}/") except Exception: pass @@ -209,21 +209,21 @@ def _test(): @cache_outgoing_headers @validate_cross_process_headers def test_urlretrieve_cross_process_request(server): - urllib.urlretrieve("http://localhost:%d/" % server.port) + urllib.urlretrieve(f"http://localhost:{server.port}/") @cat_enabled def test_urlretrieve_cross_process_response(server): _test_urlretrieve_cross_process_response_scoped_metrics = [ - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1) + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1) ] _test_urlretrieve_cross_process_response_rollup_metrics = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("ExternalApp/localhost:%d/1#2/all" % server.port, 1), - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"ExternalApp/localhost:{server.port}/1#2/all", 1), + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1), ] _test_urlretrieve_cross_process_response_external_node_params = [ @@ -242,6 +242,6 @@ def test_urlretrieve_cross_process_response(server): @validate_external_node_params(params=_test_urlretrieve_cross_process_response_external_node_params) @background_task(name="test_urllib:test_urlretrieve_cross_process_response") def _test(): - urllib.urlretrieve("http://localhost:%d/" % server.port) + urllib.urlretrieve(f"http://localhost:{server.port}/") _test() diff --git a/tests/external_httplib/test_urllib2.py b/tests/external_httplib/test_urllib2.py index 62ed230745..c236d77071 100644 --- a/tests/external_httplib/test_urllib2.py +++ b/tests/external_httplib/test_urllib2.py @@ -16,10 +16,7 @@ import pytest -try: - import urllib.request as urllib2 -except: - import urllib2 +import urllib.request as urllib2 from testing_support.external_fixtures import ( cache_outgoing_headers, @@ -39,13 +36,13 @@ @pytest.fixture(scope="session") def metrics(server): - scoped = [("External/localhost:%d/urllib2/" % server.port, 1)] + scoped = [(f"External/localhost:{server.port}/urllib2/", 1)] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("External/localhost:%d/urllib2/" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"External/localhost:{server.port}/urllib2/", 1), ] return scoped, rollup @@ -60,7 +57,7 @@ def test_urlopen_http_request(server, metrics): ) @background_task(name="test_urllib2:test_urlopen_http_request") def _test(): - urllib2.urlopen("http://localhost:%d/" % server.port) + urllib2.urlopen(f"http://localhost:{server.port}/") _test() @@ -75,7 +72,7 @@ def test_urlopen_https_request(server, metrics): @background_task(name="test_urllib2:test_urlopen_https_request") def _test(): try: - urllib2.urlopen("https://localhost:%d/" % server.port) + urllib2.urlopen(f"https://localhost:{server.port}/") except Exception: pass @@ -83,13 +80,13 @@ def _test(): def test_urlopen_http_request_with_port(server): - scoped = [("External/localhost:%d/urllib2/" % server.port, 1)] + scoped = [(f"External/localhost:{server.port}/urllib2/", 1)] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("External/localhost:%d/urllib2/" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"External/localhost:{server.port}/urllib2/", 1), ] @validate_transaction_metrics( @@ -100,7 +97,7 @@ def test_urlopen_http_request_with_port(server): ) @background_task(name="test_urllib2:test_urlopen_http_request_with_port") def _test(): - urllib2.urlopen("http://localhost:%d/" % server.port) + urllib2.urlopen(f"http://localhost:{server.port}/") _test() @@ -123,7 +120,7 @@ def _test(): @background_task() def test_urlopen_file_request(): path = os.path.abspath(__file__) - file_uri = "file://%s" % path + file_uri = f"file://{path}" urllib2.urlopen(file_uri) @@ -131,21 +128,21 @@ def test_urlopen_file_request(): @cache_outgoing_headers @validate_cross_process_headers def test_urlopen_cross_process_request(server): - urllib2.urlopen("http://localhost:%d/" % server.port) + urllib2.urlopen(f"http://localhost:{server.port}/") @cat_enabled def test_urlopen_cross_process_response(server): _test_urlopen_cross_process_response_scoped_metrics = [ - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1) + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1) ] _test_urlopen_cross_process_response_rollup_metrics = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("ExternalApp/localhost:%d/1#2/all" % server.port, 1), - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"ExternalApp/localhost:{server.port}/1#2/all", 1), + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1), ] _test_urlopen_cross_process_response_external_node_params = [ @@ -164,6 +161,6 @@ def test_urlopen_cross_process_response(server): @validate_external_node_params(params=_test_urlopen_cross_process_response_external_node_params) @background_task(name="test_urllib2:test_urlopen_cross_process_response") def _test(): - urllib2.urlopen("http://localhost:%d/" % server.port) + urllib2.urlopen(f"http://localhost:{server.port}/") _test() diff --git a/tests/external_httplib2/conftest.py b/tests/external_httplib2/conftest.py index cf3501da50..55797d84a2 100644 --- a/tests/external_httplib2/conftest.py +++ b/tests/external_httplib2/conftest.py @@ -13,26 +13,29 @@ # limitations under the License. import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) from testing_support.mock_external_http_server import ( - MockExternalHTTPHResponseHeadersServer) - + MockExternalHTTPHResponseHeadersServer, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (external_httplib2)', - default_settings=_default_settings) + app_name="Python Agent Test (external_httplib2)", default_settings=_default_settings +) + -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def server(): with MockExternalHTTPHResponseHeadersServer() as _server: yield _server diff --git a/tests/external_httplib2/test_httplib2.py b/tests/external_httplib2/test_httplib2.py index 288aa84ee9..4fc8e2b0d9 100644 --- a/tests/external_httplib2/test_httplib2.py +++ b/tests/external_httplib2/test_httplib2.py @@ -35,13 +35,13 @@ @pytest.fixture(scope="session") def metrics(server): - scoped = [("External/localhost:%d/httplib2/" % server.port, 1)] + scoped = [(f"External/localhost:{server.port}/httplib2/", 1)] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("External/localhost:%d/httplib2/" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"External/localhost:{server.port}/httplib2/", 1), ] return scoped, rollup @@ -94,7 +94,7 @@ def test_httplib2_http_request(server, metrics): @background_task(name="test_httplib2:test_httplib2_http_request") def _test(): connection = httplib2.Http() - response, content = connection.request("http://localhost:%d" % server.port, "GET") + response, content = connection.request(f"http://localhost:{server.port}", "GET") _test() @@ -132,15 +132,15 @@ def _test(): @cat_enabled def test_httplib2_cross_process_response(server): _test_httplib2_cross_process_response_scoped_metrics = [ - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1) + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1) ] _test_httplib2_cross_process_response_rollup_metrics = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("ExternalApp/localhost:%d/1#2/all" % server.port, 1), - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"ExternalApp/localhost:{server.port}/1#2/all", 1), + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1), ] _test_httplib2_cross_process_response_external_node_params = [ diff --git a/tests/external_httpx/conftest.py b/tests/external_httpx/conftest.py index 87ea1bec06..760e18cde1 100644 --- a/tests/external_httpx/conftest.py +++ b/tests/external_httpx/conftest.py @@ -12,14 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio - import pytest -from testing_support.fixture.event_loop import event_loop as loop -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixture.event_loop import ( # noqa: F401; pylint: disable=W0611 + event_loop as loop, +) +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, diff --git a/tests/external_httpx/test_client.py b/tests/external_httpx/test_client.py index 87a1bc7d01..756f7d9773 100644 --- a/tests/external_httpx/test_client.py +++ b/tests/external_httpx/test_client.py @@ -19,7 +19,6 @@ dt_enabled, override_application_settings, override_generic_settings, - validate_tt_segment_params, ) from testing_support.mock_external_http_server import ( MockExternalHTTPHResponseHeadersServer, @@ -28,8 +27,15 @@ validate_cross_process_headers, ) from testing_support.validators.validate_span_events import validate_span_events -from testing_support.validators.validate_transaction_errors import validate_transaction_errors -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) +from testing_support.validators.validate_tt_segment_params import ( + validate_tt_segment_params, +) from newrelic.api.background_task import background_task from newrelic.api.time_trace import current_trace @@ -76,14 +82,14 @@ def server(): def populate_metrics(server, request): SCOPED_METRICS[:] = [] method = request.getfixturevalue("method").upper() - SCOPED_METRICS.append(("External/localhost:%d/httpx/%s" % (server.port, method), 2)) + SCOPED_METRICS.append((f"External/localhost:{server.port}/httpx/{method}", 2)) def exercise_sync_client(server, client, method): with client as client: resolved_method = getattr(client, method) - resolved_method("http://localhost:%s" % server.port) - response = resolved_method("http://localhost:%s" % server.port) + resolved_method(f"http://localhost:{server.port}") + response = resolved_method(f"http://localhost:{server.port}") return response @@ -118,8 +124,8 @@ async def exercise_async_client(server, client, method): async with client as client: resolved_method = getattr(client, method) responses = await asyncio.gather( - resolved_method("http://localhost:%s" % server.port), - resolved_method("http://localhost:%s" % server.port), + resolved_method(f"http://localhost:{server.port}"), + resolved_method(f"http://localhost:{server.port}"), ) return responses @@ -178,7 +184,7 @@ def _test(): transaction = current_transaction() with httpx.Client() as client: - response = client.get("http://localhost:%s" % server.port) + response = client.get(f"http://localhost:{server.port}") transaction._test_request_headers = response.request.headers @@ -210,7 +216,7 @@ def test_async_cross_process_request(httpx, server, loop, distributed_tracing, s ) async def _test(): async with httpx.AsyncClient() as client: - response = await client.get("http://localhost:%s" % server.port) + response = await client.get(f"http://localhost:{server.port}") return response @@ -237,7 +243,7 @@ def test_sync_cross_process_override_headers(httpx, server, loop): transaction = current_transaction() with httpx.Client() as client: - response = client.get("http://localhost:%s" % server.port, headers={"newrelic": "1234"}) + response = client.get(f"http://localhost:{server.port}", headers={"newrelic": "1234"}) transaction._test_request_headers = response.request.headers @@ -259,7 +265,7 @@ def test_async_cross_process_override_headers(httpx, server, loop): async def _test(): async with httpx.AsyncClient() as client: - response = await client.get("http://localhost:%s" % server.port, headers={"newrelic": "1234"}) + response = await client.get(f"http://localhost:{server.port}", headers={"newrelic": "1234"}) return response @@ -286,7 +292,7 @@ def test_sync_client_cat_response_processing(cat_enabled, response_code, server, expected_metrics = [ ( - "ExternalTransaction/localhost:%s/1#1/WebTransaction/Function/app:beep" % server.port, + f"ExternalTransaction/localhost:{server.port}/1#1/WebTransaction/Function/app:beep", 1 if cat_enabled else None, ), ] @@ -302,7 +308,7 @@ def test_sync_client_cat_response_processing(cat_enabled, response_code, server, @background_task(name="test_sync_client_cat_response_processing") def _test(): with httpx.Client() as client: - response = client.get("http://localhost:%s" % server.port) + response = client.get(f"http://localhost:{server.port}") _test() @@ -324,7 +330,7 @@ def test_async_client_cat_response_processing(cat_enabled, response_code, httpx, expected_metrics = [ ( - "ExternalTransaction/localhost:%s/1#1/WebTransaction/Function/app:beep" % server.port, + f"ExternalTransaction/localhost:{server.port}/1#1/WebTransaction/Function/app:beep", 1 if cat_enabled else None, ), ] @@ -341,7 +347,7 @@ def test_async_client_cat_response_processing(cat_enabled, response_code, httpx, def _test(): async def coro(): async with httpx.AsyncClient() as client: - response = await client.get("http://localhost:%s" % server.port) + response = await client.get(f"http://localhost:{server.port}") return response @@ -364,16 +370,16 @@ def empty_hook(response): @validate_span_events( count=1, - exact_intrinsics={"name": "External/localhost:%d/httpx/GET" % server.port}, + exact_intrinsics={"name": f"External/localhost:{server.port}/httpx/GET"}, exact_agents={"http.statusCode": CAT_RESPONSE_CODE}, ) @background_task(name="test_sync_client_event_hook_exception") def make_request(client, exc_expected=True): if exc_expected: with pytest.raises(RuntimeError): - client.get("http://localhost:%s" % server.port) + client.get(f"http://localhost:{server.port}") else: - client.get("http://localhost:%s" % server.port) + client.get(f"http://localhost:{server.port}") with httpx.Client(event_hooks={"response": [exception_event_hook]}) as client: # Test client init @@ -410,7 +416,7 @@ def empty_hook(response): @validate_span_events( count=1, - exact_intrinsics={"name": "External/localhost:%d/httpx/GET" % server.port}, + exact_intrinsics={"name": f"External/localhost:{server.port}/httpx/GET"}, exact_agents={"http.statusCode": CAT_RESPONSE_CODE}, ) @background_task(name="test_sync_client_event_hook_exception") @@ -418,9 +424,9 @@ def make_request(client, exc_expected=True): async def coro(): if exc_expected: with pytest.raises(RuntimeError): - await client.get("http://localhost:%s" % server.port) + await client.get(f"http://localhost:{server.port}") else: - await client.get("http://localhost:%s" % server.port) + await client.get(f"http://localhost:{server.port}") loop.run_until_complete(coro()) @@ -458,7 +464,7 @@ def test_sync_nr_disabled(httpx, server): with httpx.Client() as client: trace = current_trace() - response = client.get("http://localhost:%s" % server.port) + response = client.get(f"http://localhost:{server.port}") assert response.status_code == 200 assert trace is None @@ -476,7 +482,7 @@ def test_async_nr_disabled(httpx, server, loop): async def _test(): async with httpx.AsyncClient() as client: - response = await client.get("http://localhost:%s" % server.port) + response = await client.get(f"http://localhost:{server.port}") return response diff --git a/tests/external_requests/conftest.py b/tests/external_requests/conftest.py index 10a2ccf051..15426c3a56 100644 --- a/tests/external_requests/conftest.py +++ b/tests/external_requests/conftest.py @@ -13,25 +13,29 @@ # limitations under the License. import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) from testing_support.mock_external_http_server import ( - MockExternalHTTPHResponseHeadersServer) - + MockExternalHTTPHResponseHeadersServer, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (external_requests)', - default_settings=_default_settings) + app_name="Python Agent Test (external_requests)", default_settings=_default_settings +) + -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def server(): with MockExternalHTTPHResponseHeadersServer() as _server: yield _server diff --git a/tests/external_requests/test_requests.py b/tests/external_requests/test_requests.py index f6f4506e51..228429e3f4 100644 --- a/tests/external_requests/test_requests.py +++ b/tests/external_requests/test_requests.py @@ -30,24 +30,30 @@ from testing_support.validators.validate_external_node_params import ( validate_external_node_params, ) -from testing_support.validators.validate_transaction_errors import validate_transaction_errors -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple def get_requests_version(): - return tuple(map(int, requests.__version__.split(".")[:2])) + return get_package_version_tuple("requests") @pytest.fixture(scope="session") def metrics(server): - scoped = [("External/localhost:%d/requests/" % server.port, 1)] + scoped = [(f"External/localhost:{server.port}/requests/", 1)] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("External/localhost:%d/requests/" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"External/localhost:{server.port}/requests/", 1), ] return scoped, rollup @@ -72,7 +78,7 @@ def test_http_request_get(server, metrics): ) @background_task(name="test_requests:test_http_request_get") def _test(): - requests.get("http://localhost:%d/" % server.port) + requests.get(f"http://localhost:{server.port}/") _test() @@ -89,7 +95,7 @@ def test_https_request_get(server, metrics): @background_task(name="test_requests:test_https_request_get") def _test(): try: - requests.get("https://localhost:%d/" % server.port, verify=False) + requests.get(f"https://localhost:{server.port}/", verify=False) # nosec except Exception: pass @@ -108,7 +114,7 @@ def test_http_session_send(server, metrics): @background_task(name="test_requests:test_http_session_send") def _test(): session = requests.Session() - req = requests.Request("GET", "http://localhost:%d/" % server.port) + req = requests.Request("GET", f"http://localhost:{server.port}/") prep_req = req.prepare() session.send(prep_req) @@ -183,7 +189,7 @@ def test_requests_cross_process_request(distributed_tracing, span_events, server @cache_outgoing_headers @validate_cross_process_headers def _test(): - requests.get("http://localhost:%d/" % server.port) + requests.get(f"http://localhost:{server.port}/") _test = override_application_settings( { @@ -199,15 +205,15 @@ def _test(): @cat_enabled def test_requests_cross_process_response(server): _test_requests_cross_process_response_scoped_metrics = [ - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1) + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1) ] _test_requests_cross_process_response_rollup_metrics = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("ExternalApp/localhost:%d/1#2/all" % server.port, 1), - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"ExternalApp/localhost:{server.port}/1#2/all", 1), + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1), ] _test_requests_cross_process_response_external_node_params = [ @@ -227,6 +233,6 @@ def test_requests_cross_process_response(server): @validate_external_node_params(params=_test_requests_cross_process_response_external_node_params) @background_task(name="test_requests:test_requests_cross_process_response") def _test(): - requests.get("http://localhost:%d/" % server.port) + requests.get(f"http://localhost:{server.port}/") _test() diff --git a/tests/external_requests/test_span_event.py b/tests/external_requests/test_span_event.py index 575b2d52b7..0eedb99a55 100644 --- a/tests/external_requests/test_span_event.py +++ b/tests/external_requests/test_span_event.py @@ -29,14 +29,14 @@ def server(): yield _server -@pytest.mark.parametrize('path', ('', '/foo', '/' + 'a' * 256)) +@pytest.mark.parametrize('path', ('', '/foo', f"/{'a' * 256}")) def test_span_events(server, path): _settings = { 'distributed_tracing.enabled': True, 'span_events.enabled': True, } - uri = 'http://localhost:%d' % server.port + uri = f'http://localhost:{server.port}' if path: uri += path @@ -44,7 +44,7 @@ def test_span_events(server, path): expected_uri = uri[:255] exact_intrinsics = { - 'name': 'External/localhost:%d/requests/' % server.port, + 'name': f'External/localhost:{server.port}/requests/', 'type': 'Span', 'sampled': True, 'priority': 0.5, diff --git a/tests/external_urllib3/conftest.py b/tests/external_urllib3/conftest.py index 19d3f394bd..e380eda1be 100644 --- a/tests/external_urllib3/conftest.py +++ b/tests/external_urllib3/conftest.py @@ -13,26 +13,29 @@ # limitations under the License. import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) from testing_support.mock_external_http_server import ( - MockExternalHTTPHResponseHeadersServer) - + MockExternalHTTPHResponseHeadersServer, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (external_urllib3)', - default_settings=_default_settings) + app_name="Python Agent Test (external_urllib3)", default_settings=_default_settings +) + -@pytest.fixture(scope='session') +@pytest.fixture(scope="session") def server(): with MockExternalHTTPHResponseHeadersServer() as _server: yield _server diff --git a/tests/external_urllib3/test_urllib3.py b/tests/external_urllib3/test_urllib3.py index 68e15d4634..afb1e4a105 100644 --- a/tests/external_urllib3/test_urllib3.py +++ b/tests/external_urllib3/test_urllib3.py @@ -25,31 +25,33 @@ cache_outgoing_headers, insert_incoming_headers, ) -from testing_support.fixtures import ( - cat_enabled, - override_application_settings, -) -from testing_support.util import version2tuple +from testing_support.fixtures import cat_enabled, override_application_settings from testing_support.validators.validate_cross_process_headers import ( validate_cross_process_headers, ) from testing_support.validators.validate_external_node_params import ( validate_external_node_params, ) -from testing_support.validators.validate_transaction_errors import validate_transaction_errors -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + from newrelic.api.background_task import background_task +from newrelic.common.package_version_utils import get_package_version_tuple @pytest.fixture(scope="session") def metrics(server): - scoped = [("External/localhost:%d/urllib3/" % server.port, 1)] + scoped = [(f"External/localhost:{server.port}/urllib3/GET", 1)] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("External/localhost:%d/urllib3/" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"External/localhost:{server.port}/urllib3/GET", 1), ] return scoped, rollup @@ -65,7 +67,7 @@ def test_http_request_connection_pool_urlopen(server, metrics): ) @background_task(name="test_urllib3:test_http_request_connection_pool_urlopen") def _test(): - pool = urllib3.HTTPConnectionPool("localhost:%d" % server.port) + pool = urllib3.HTTPConnectionPool(f"localhost:{server.port}") pool.urlopen("GET", "/") _test() @@ -81,7 +83,7 @@ def test_http_request_connection_pool_request(server, metrics): ) @background_task(name="test_urllib3:test_http_request_connection_pool_request") def _test(): - pool = urllib3.HTTPConnectionPool("localhost:%d" % server.port) + pool = urllib3.HTTPConnectionPool(f"localhost:{server.port}") pool.request("GET", "/") _test() @@ -97,7 +99,7 @@ def test_http_request_connection_from_url_request(server, metrics): ) @background_task(name="test_urllib3:test_http_request_connection_from_url_request") def _test(): - conn = urllib3.connection_from_url("http://localhost:%d" % server.port) + conn = urllib3.connection_from_url(f"http://localhost:{server.port}") conn.request("GET", "/") _test() @@ -114,7 +116,7 @@ def test_http_request_pool_manager_urlopen(server, metrics): @background_task(name="test_urllib3:test_http_request_pool_manager_urlopen") def _test(): pool = urllib3.PoolManager(5) - pool.urlopen("GET", "http://localhost:%d/" % server.port) + pool.urlopen("GET", f"http://localhost:{server.port}/") _test() @@ -130,7 +132,7 @@ def test_https_request_connection_pool_urlopen(server, metrics): @background_task(name="test_urllib3:test_https_request_connection_pool_urlopen") def _test(): # Setting retries to 0 so that metrics are recorded only once - pool = urllib3.HTTPSConnectionPool("localhost:%d" % server.port, retries=0) + pool = urllib3.HTTPSConnectionPool(f"localhost:{server.port}", retries=0) try: pool.urlopen("GET", "/") except Exception: @@ -150,7 +152,7 @@ def test_https_request_connection_pool_request(server, metrics): @background_task(name="test_urllib3:test_https_request_connection_pool_request") def _test(): # Setting retries to 0 so that metrics are recorded only once - pool = urllib3.HTTPSConnectionPool("localhost:%d" % server.port, retries=0) + pool = urllib3.HTTPSConnectionPool(f"localhost:{server.port}", retries=0) try: pool.request("GET", "/") except Exception: @@ -160,13 +162,13 @@ def _test(): def test_port_included(server): - scoped = [("External/localhost:%d/urllib3/" % server.port, 1)] + scoped = [(f"External/localhost:{server.port}/urllib3/GET", 1)] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("External/localhost:%d/urllib3/" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"External/localhost:{server.port}/urllib3/GET", 1), ] @validate_transaction_errors(errors=[]) @@ -175,7 +177,7 @@ def test_port_included(server): ) @background_task(name="test_urllib3:test_port_included") def _test(): - conn = urllib3.connection_from_url("http://localhost:%d" % server.port) + conn = urllib3.connection_from_url(f"http://localhost:{server.port}") conn.request("GET", "/") _test() @@ -185,16 +187,16 @@ def _test(): # HTTPConnection class. Previously the httplib/http.client HTTPConnection class # was used. We test httplib in a different test directory so we skip this test. @pytest.mark.skipif( - version2tuple(urllib3.__version__) < (1, 8), reason="urllib3.connection.HTTPConnection added in 1.8" + get_package_version_tuple("urllib3") < (1, 8), reason="urllib3.connection.HTTPConnection added in 1.8" ) def test_HTTPConnection_port_included(server): - scoped = [("External/localhost:%d/urllib3/" % server.port, 1)] + scoped = [(f"External/localhost:{server.port}/urllib3/", 1)] rollup = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("External/localhost:%d/urllib3/" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"External/localhost:{server.port}/urllib3/", 1), ] @validate_transaction_errors(errors=[]) @@ -206,7 +208,7 @@ def test_HTTPConnection_port_included(server): ) @background_task(name="test_urllib3:test_HTTPConnection_port_included") def _test(): - conn = urllib3.connection.HTTPConnection("localhost:%d" % server.port) + conn = urllib3.connection.HTTPConnection(f"localhost:{server.port}") conn.request("GET", "/") _test() @@ -226,7 +228,7 @@ def test_urlopen_cross_process_request(distributed_tracing, span_events, server) @cache_outgoing_headers @validate_cross_process_headers def _test(): - pool = urllib3.HTTPConnectionPool("localhost:%d" % server.port) + pool = urllib3.HTTPConnectionPool(f"localhost:{server.port}") pool.urlopen("GET", "/") _test = override_application_settings( @@ -243,15 +245,15 @@ def _test(): @cat_enabled def test_urlopen_cross_process_response(server): _test_urlopen_cross_process_response_scoped_metrics = [ - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1) + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1) ] _test_urlopen_cross_process_response_rollup_metrics = [ ("External/all", 1), ("External/allOther", 1), - ("External/localhost:%d/all" % server.port, 1), - ("ExternalApp/localhost:%d/1#2/all" % server.port, 1), - ("ExternalTransaction/localhost:%d/1#2/test" % server.port, 1), + (f"External/localhost:{server.port}/all", 1), + (f"ExternalApp/localhost:{server.port}/1#2/all", 1), + (f"ExternalTransaction/localhost:{server.port}/1#2/test", 1), ] _test_urlopen_cross_process_response_external_node_params = [ @@ -271,7 +273,7 @@ def test_urlopen_cross_process_response(server): @validate_external_node_params(params=_test_urlopen_cross_process_response_external_node_params) @background_task(name="test_urllib3:test_urlopen_cross_process_response") def _test(): - pool = urllib3.HTTPConnectionPool("localhost:%d" % server.port) + pool = urllib3.HTTPConnectionPool(f"localhost:{server.port}") pool.urlopen("GET", "/") _test() diff --git a/tests/framework_aiohttp/_target_application.py b/tests/framework_aiohttp/_target_application.py index f15e7fd65b..7dde99ce0b 100644 --- a/tests/framework_aiohttp/_target_application.py +++ b/tests/framework_aiohttp/_target_application.py @@ -113,7 +113,7 @@ async def websocket_handler(request): while not ws.closed: msg = await ws.receive() if msg.type == WSMsgType.TEXT: - result = ws.send_str("/" + msg.data) + result = ws.send_str(f"/{msg.data}") if hasattr(result, "__await__"): await result diff --git a/tests/framework_aiohttp/conftest.py b/tests/framework_aiohttp/conftest.py index 3bb814a9b2..ed4c0963b4 100644 --- a/tests/framework_aiohttp/conftest.py +++ b/tests/framework_aiohttp/conftest.py @@ -22,15 +22,17 @@ from testing_support.fixture.event_loop import ( # noqa: F401 pylint: disable=W0611 event_loop, ) - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) from testing_support.mock_external_http_server import ( MockExternalHTTPHResponseHeadersServer, MockExternalHTTPServer, ) - _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, @@ -135,7 +137,7 @@ def respond_with_cat_header(self): @pytest.fixture(scope="session") def local_server_info(mock_header_server): - host_port = "127.0.0.1:%d" % mock_header_server.port - metric = "External/%s/aiohttp/" % host_port - url = "http://" + host_port + host_port = f"127.0.0.1:{mock_header_server.port}" + metric = f"External/{host_port}/aiohttp/" + url = f"http://{host_port}" return ServerInfo(metric, url) diff --git a/tests/framework_aiohttp/test_client.py b/tests/framework_aiohttp/test_client.py index 96bbb46f01..932c46dfb4 100644 --- a/tests/framework_aiohttp/test_client.py +++ b/tests/framework_aiohttp/test_client.py @@ -198,10 +198,10 @@ def test_ws_connect_yield_from(event_loop, local_server_info, method, exc_expect "fetch_multiple", background_task=True, scoped_metrics=[ - (local_server_info.base_metric + "GET", 2), + (f"{local_server_info.base_metric}GET", 2), ], rollup_metrics=[ - (local_server_info.base_metric + "GET", 2), + (f"{local_server_info.base_metric}GET", 2), ], ) def task_test(): diff --git a/tests/framework_aiohttp/test_client_async_await.py b/tests/framework_aiohttp/test_client_async_await.py index dedc64c9db..87337648b7 100644 --- a/tests/framework_aiohttp/test_client_async_await.py +++ b/tests/framework_aiohttp/test_client_async_await.py @@ -17,7 +17,9 @@ import aiohttp import pytest from testing_support.fixtures import cat_enabled -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) from yarl import URL from newrelic.api.background_task import background_task @@ -114,68 +116,6 @@ def task_test(): task_test() -@pytest.mark.parametrize("method,exc_expected", test_matrix) -def test_client_throw_async_await(event_loop, local_server_info, method, exc_expected): - class ThrowerException(ValueError): - pass - - @background_task(name="test_client_throw_async_await") - async def self_driving_thrower(): - async with aiohttp.ClientSession() as session: - coro = session._request(method.upper(), local_server_info.url) - - # activate the coroutine - coro.send(None) - - # inject error - coro.throw(ThrowerException()) - - @validate_transaction_metrics( - "test_client_throw_async_await", - background_task=True, - scoped_metrics=[ - (local_server_info.base_metric + method.upper(), 1), - ], - rollup_metrics=[ - (local_server_info.base_metric + method.upper(), 1), - ], - ) - def task_test(): - with pytest.raises(ThrowerException): - event_loop.run_until_complete(self_driving_thrower()) - - task_test() - - -@pytest.mark.parametrize("method,exc_expected", test_matrix) -def test_client_close_async_await(event_loop, local_server_info, method, exc_expected): - @background_task(name="test_client_close_async_await") - async def self_driving_closer(): - async with aiohttp.ClientSession() as session: - coro = session._request(method.upper(), local_server_info.url) - - # activate the coroutine - coro.send(None) - - # force close - coro.close() - - @validate_transaction_metrics( - "test_client_close_async_await", - background_task=True, - scoped_metrics=[ - (local_server_info.base_metric + method.upper(), 1), - ], - rollup_metrics=[ - (local_server_info.base_metric + method.upper(), 1), - ], - ) - def task_test(): - event_loop.run_until_complete(self_driving_closer()) - - task_test() - - @pytest.mark.parametrize("method,exc_expected", test_matrix) @cat_enabled def test_await_request_async_await(event_loop, local_server_info, method, exc_expected): @@ -225,10 +165,10 @@ def test_ws_connect_async_await(event_loop, local_server_info, method, exc_expec "fetch_multiple", background_task=True, scoped_metrics=[ - (local_server_info.base_metric + "GET", 2), + (f"{local_server_info.base_metric}GET", 2), ], rollup_metrics=[ - (local_server_info.base_metric + "GET", 2), + (f"{local_server_info.base_metric}GET", 2), ], ) def task_test(): @@ -240,7 +180,6 @@ def task_test(): @pytest.mark.parametrize("method,exc_expected", test_matrix) @cat_enabled def test_create_task_async_await(event_loop, local_server_info, method, exc_expected): - # `loop.create_task` returns a Task object which uses the coroutine's # `send` method, not `__next__` diff --git a/tests/framework_aiohttp/test_client_cat.py b/tests/framework_aiohttp/test_client_cat.py index 8877434299..edd14498db 100644 --- a/tests/framework_aiohttp/test_client_cat.py +++ b/tests/framework_aiohttp/test_client_cat.py @@ -41,7 +41,6 @@ async def fetch(url, headers=None, raise_for_status=False, connector=None): - kwargs = {} if version_info >= (2, 0): kwargs = {"raise_for_status": raise_for_status} @@ -77,7 +76,7 @@ async def fetch(url, headers=None, raise_for_status=False, connector=None): def test_outbound_cross_process_headers(event_loop, cat_enabled, distributed_tracing, span_events, mock_header_server): @background_task(name="test_outbound_cross_process_headers") async def _test(): - headers = await fetch("http://127.0.0.1:%d" % mock_header_server.port) + headers = await fetch(f"http://127.0.0.1:{mock_header_server.port}") transaction = current_transaction() transaction._test_request_headers = headers @@ -122,9 +121,8 @@ def test(): @pytest.mark.parametrize("customer_headers", _customer_headers_tests) def test_outbound_cross_process_headers_custom_headers(event_loop, customer_headers, mock_header_server): - headers = event_loop.run_until_complete( - background_task()(fetch)("http://127.0.0.1:%d" % mock_header_server.port, customer_headers.copy()) + background_task()(fetch)(f"http://127.0.0.1:{mock_header_server.port}", customer_headers.copy()) ) # always honor customer headers @@ -133,7 +131,7 @@ def test_outbound_cross_process_headers_custom_headers(event_loop, customer_head def test_outbound_cross_process_headers_no_txn(event_loop, mock_header_server): - headers = event_loop.run_until_complete(fetch("http://127.0.0.1:%d" % mock_header_server.port)) + headers = event_loop.run_until_complete(fetch(f"http://127.0.0.1:{mock_header_server.port}")) assert not headers.get(ExternalTrace.cat_id_key) assert not headers.get(ExternalTrace.cat_transaction_key) @@ -148,7 +146,7 @@ async def test(): delattr(transaction, "guid") try: - headers = await fetch("http://127.0.0.1:%d" % mock_header_server.port) + headers = await fetch(f"http://127.0.0.1:{mock_header_server.port}") assert not headers.get(ExternalTrace.cat_id_key) assert not headers.get(ExternalTrace.cat_transaction_key) @@ -174,7 +172,6 @@ async def _resolve_host(self, host, port, *args, **kwargs): def test_process_incoming_headers( event_loop, cat_enabled, response_code, raise_for_status, connector_class, mock_external_http_server ): - # It was discovered via packnsend that the `throw` method of the `_request` # coroutine is used in the case of poorly resolved hosts. An older version # of the instrumentation ended the ExternalTrace anytime `throw` was called @@ -183,19 +180,19 @@ def test_process_incoming_headers( # always called and thus makes sure the trace is not ended before # StopIteration is called. server, response_values = mock_external_http_server - address = "http://127.0.0.1:%d" % server.port + address = f"http://127.0.0.1:{server.port}" port = server.port _test_cross_process_response_scoped_metrics = [ - ("ExternalTransaction/127.0.0.1:%d/1#2/test" % port, 1 if cat_enabled else None) + (f"ExternalTransaction/127.0.0.1:{port}/1#2/test", 1 if cat_enabled else None) ] _test_cross_process_response_rollup_metrics = [ ("External/all", 1), ("External/allOther", 1), - ("External/127.0.0.1:%d/all" % port, 1), - ("ExternalApp/127.0.0.1:%d/1#2/all" % port, 1 if cat_enabled else None), - ("ExternalTransaction/127.0.0.1:%d/1#2/test" % port, 1 if cat_enabled else None), + (f"External/127.0.0.1:{port}/all", 1), + (f"ExternalApp/127.0.0.1:{port}/1#2/all", 1 if cat_enabled else None), + (f"ExternalTransaction/127.0.0.1:{port}/1#2/test", 1 if cat_enabled else None), ] _test_cross_process_response_external_node_params = [ @@ -208,8 +205,6 @@ def test_process_incoming_headers( k for k, v in _test_cross_process_response_external_node_params ] - connector = connector_class() if connector_class else None - @background_task(name="test_process_incoming_headers") async def _test(): transaction = current_transaction() @@ -217,6 +212,8 @@ async def _test(): response_values.append((headers, response_code)) + connector = connector_class() if connector_class else None + await fetch(address, raise_for_status=raise_for_status, connector=connector) @override_application_settings( diff --git a/tests/framework_aiohttp/test_middleware.py b/tests/framework_aiohttp/test_middleware.py index 6cbf86677a..246ae1ad09 100644 --- a/tests/framework_aiohttp/test_middleware.py +++ b/tests/framework_aiohttp/test_middleware.py @@ -74,7 +74,7 @@ def _test(): rollup_metrics = [ ("Function/_target_application:index", 1), (metric, 1), - ("Python/Framework/aiohttp/%s" % aiohttp.__version__, 1), + (f"Python/Framework/aiohttp/{aiohttp.__version__}", 1), ] _test = validate_transaction_metrics( diff --git a/tests/framework_aiohttp/test_server.py b/tests/framework_aiohttp/test_server.py index 6a5ef0d10e..69ebd4b21b 100644 --- a/tests/framework_aiohttp/test_server.py +++ b/tests/framework_aiohttp/test_server.py @@ -87,11 +87,11 @@ async def fetch(): @validate_transaction_metrics( metric_name, scoped_metrics=[ - ("Function/%s" % metric_name, 1), + (f"Function/{metric_name}", 1), ], rollup_metrics=[ - ("Function/%s" % metric_name, 1), - ("Python/Framework/aiohttp/%s" % aiohttp.__version__, 1), + (f"Function/{metric_name}", 1), + (f"Python/Framework/aiohttp/{aiohttp.__version__}", 1), ], ) @validate_transaction_event_attributes( @@ -184,11 +184,11 @@ async def multi_fetch(loop): @validate_transaction_metrics( metric_name, scoped_metrics=[ - ("Function/%s" % metric_name, 1), + (f"Function/{metric_name}", 1), ], rollup_metrics=[ - ("Function/%s" % metric_name, 1), - ("Python/Framework/aiohttp/%s" % aiohttp.__version__, 1), + (f"Function/{metric_name}", 1), + (f"Python/Framework/aiohttp/{aiohttp.__version__}", 1), ], ) @validate_transaction_event_attributes( diff --git a/tests/framework_aiohttp/test_server_cat.py b/tests/framework_aiohttp/test_server_cat.py index 44b5c72174..8b18074558 100644 --- a/tests/framework_aiohttp/test_server_cat.py +++ b/tests/framework_aiohttp/test_server_cat.py @@ -37,7 +37,7 @@ def record_aiohttp1_raw_headers(raw_headers): try: - import aiohttp.protocol # noqa: F401 + import aiohttp.protocol # noqa: F401, pylint: disable=W0611 except ImportError: def pass_through(function): @@ -113,7 +113,7 @@ async def fetch(): app_data = json.loads(deobfuscate(raw_headers["X-NewRelic-App-Data"], ENCODING_KEY)) assert app_data[0] == cat_id - assert app_data[1] == ("WebTransaction/Function/%s" % metric_name) + assert app_data[1] == f"WebTransaction/Function/{metric_name}" else: assert "X-NewRelic-App-Data" not in resp.headers @@ -135,7 +135,7 @@ async def fetch(): # a fixture from conftest.py/_target_application.py @validate_analytics_catmap_data( - "WebTransaction/Function/%s" % metric_name, + f"WebTransaction/Function/{metric_name}", expected_attributes=expected_intrinsics, non_expected_attributes=forgone_intrinsics, ) diff --git a/tests/framework_ariadne/__init__.py b/tests/framework_ariadne/__init__.py new file mode 100644 index 0000000000..8030baccf7 --- /dev/null +++ b/tests/framework_ariadne/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/tests/framework_ariadne/_target_application.py b/tests/framework_ariadne/_target_application.py index 94bc0710f5..fef7826086 100644 --- a/tests/framework_ariadne/_target_application.py +++ b/tests/framework_ariadne/_target_application.py @@ -12,140 +12,125 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os - -from ariadne import ( - MutationType, - QueryType, - UnionType, - load_schema_from_path, - make_executable_schema, + +import asyncio +import json + +from framework_ariadne._target_schema_async import ( + target_asgi_application as target_asgi_application_async, +) +from framework_ariadne._target_schema_async import target_schema as target_schema_async +from framework_ariadne._target_schema_sync import ( + target_asgi_application as target_asgi_application_sync, +) +from framework_ariadne._target_schema_sync import target_schema as target_schema_sync +from framework_ariadne._target_schema_sync import ( + target_wsgi_application as target_wsgi_application_sync, ) -from ariadne.asgi import GraphQL as GraphQLASGI -from ariadne.wsgi import GraphQL as GraphQLWSGI +from framework_ariadne._target_schema_sync import ariadne_version_tuple +from graphql import MiddlewareManager -schema_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "schema.graphql") -type_defs = load_schema_from_path(schema_file) - - -authors = [ - { - "first_name": "New", - "last_name": "Relic", - }, - { - "first_name": "Bob", - "last_name": "Smith", - }, - { - "first_name": "Leslie", - "last_name": "Jones", - }, -] -books = [ - { - "id": 1, - "name": "Python Agent: The Book", - "isbn": "a-fake-isbn", - "author": authors[0], - "branch": "riverside", - }, - { - "id": 2, - "name": "Ollies for O11y: A Sk8er's Guide to Observability", - "isbn": "a-second-fake-isbn", - "author": authors[1], - "branch": "downtown", - }, - { - "id": 3, - "name": "[Redacted]", - "isbn": "a-third-fake-isbn", - "author": authors[2], - "branch": "riverside", - }, -] -magazines = [ - {"id": 1, "name": "Reli Updates Weekly", "issue": 1, "branch": "riverside"}, - {"id": 2, "name": "Reli Updates Weekly", "issue": 2, "branch": "downtown"}, - {"id": 3, "name": "Node Weekly", "issue": 1, "branch": "riverside"}, -] +def check_response(query, success, response): + if isinstance(query, str) and "error" not in query: + assert success and "errors" not in response, response + assert response.get("data", None), response + else: + assert "errors" in response, response -libraries = ["riverside", "downtown"] -libraries = [ - { - "id": i + 1, - "branch": branch, - "magazine": [m for m in magazines if m["branch"] == branch], - "book": [b for b in books if b["branch"] == branch], - } - for i, branch in enumerate(libraries) -] +def run_sync(schema): + def _run_sync(query, middleware=None): + from ariadne import graphql_sync -storage = [] + if ariadne_version_tuple < (0, 18): + if middleware: + middleware = MiddlewareManager(*middleware) + success, response = graphql_sync(schema, {"query": query}, middleware=middleware) + check_response(query, success, response) -mutation = MutationType() + return response.get("data", {}) + return _run_sync -@mutation.field("storage_add") -def mutate(self, info, string): - storage.append(string) - return {"string": string} +def run_async(schema): + def _run_async(query, middleware=None): + from ariadne import graphql -item = UnionType("Item") + #Later versions of ariadne directly accept a list of middleware while older versions require the MiddlewareManager + if ariadne_version_tuple < (0, 18): + if middleware: + middleware = MiddlewareManager(*middleware) + loop = asyncio.get_event_loop() + success, response = loop.run_until_complete(graphql(schema, {"query": query}, middleware=middleware)) + check_response(query, success, response) -@item.type_resolver -def resolve_type(obj, *args): - if "isbn" in obj: - return "Book" - elif "issue" in obj: # pylint: disable=R1705 - return "Magazine" + return response.get("data", {}) - return None + return _run_async -query = QueryType() +def run_wsgi(app): + def _run_asgi(query, middleware=None): + if not isinstance(query, str) or "error" in query: + expect_errors = True + else: + expect_errors = False + app.app.middleware = middleware -@query.field("library") -def resolve_library(self, info, index): - return libraries[index] + response = app.post( + "/", json.dumps({"query": query}), headers={"Content-Type": "application/json"}, expect_errors=expect_errors + ) + body = json.loads(response.body.decode("utf-8")) + if expect_errors: + assert body["errors"] + else: + assert "errors" not in body or not body["errors"] -@query.field("storage") -def resolve_storage(self, info): - return storage + return body.get("data", {}) + return _run_asgi -@query.field("search") -def resolve_search(self, info, contains): - search_books = [b for b in books if contains in b["name"]] - search_magazines = [m for m in magazines if contains in m["name"]] - return search_books + search_magazines +def run_asgi(app): + def _run_asgi(query, middleware=None): + if ariadne_version_tuple < (0, 16): + app.asgi_application.middleware = middleware -@query.field("hello") -def resolve_hello(self, info): - return "Hello!" + #In ariadne v0.16.0, the middleware attribute was removed from the GraphQL class in favor of the http_handler + elif ariadne_version_tuple >= (0, 16): + app.asgi_application.http_handler.middleware = middleware + response = app.make_request( + "POST", "/", body=json.dumps({"query": query}), headers={"Content-Type": "application/json"} + ) + body = json.loads(response.body.decode("utf-8")) -@query.field("echo") -def resolve_echo(self, info, echo): - return echo + if not isinstance(query, str) or "error" in query: + try: + assert response.status != 200 + except AssertionError: + assert body["errors"] + else: + assert response.status == 200 + assert "errors" not in body or not body["errors"] + return body.get("data", {}) -@query.field("error_non_null") -@query.field("error") -def resolve_error(self, info): - raise RuntimeError("Runtime Error!") + return _run_asgi -_target_application = make_executable_schema(type_defs, query, mutation, item) -_target_asgi_application = GraphQLASGI(_target_application) -_target_wsgi_application = GraphQLWSGI(_target_application) +target_application = { + "sync-sync": run_sync(target_schema_sync), + "async-sync": run_async(target_schema_sync), + "async-async": run_async(target_schema_async), + "wsgi-sync": run_wsgi(target_wsgi_application_sync), + "asgi-sync": run_asgi(target_asgi_application_sync), + "asgi-async": run_asgi(target_asgi_application_async), +} diff --git a/tests/framework_ariadne/_target_schema_async.py b/tests/framework_ariadne/_target_schema_async.py new file mode 100644 index 0000000000..076475628d --- /dev/null +++ b/tests/framework_ariadne/_target_schema_async.py @@ -0,0 +1,94 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from ariadne import ( + MutationType, + QueryType, + UnionType, + load_schema_from_path, + make_executable_schema, +) +from ariadne.asgi import GraphQL as GraphQLASGI +from framework_graphql._target_schema_sync import books, magazines, libraries + +from testing_support.asgi_testing import AsgiTest + +schema_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "schema.graphql") +type_defs = load_schema_from_path(schema_file) + +storage = [] + +mutation = MutationType() + + +@mutation.field("storage_add") +async def resolve_storage_add(self, info, string): + storage.append(string) + return string + + +item = UnionType("Item") + + +@item.type_resolver +async def resolve_type(obj, *args): + if "isbn" in obj: + return "Book" + elif "issue" in obj: # pylint: disable=R1705 + return "Magazine" + + return None + + +query = QueryType() + + +@query.field("library") +async def resolve_library(self, info, index): + return libraries[index] + + +@query.field("storage") +async def resolve_storage(self, info): + return [storage.pop()] + + +@query.field("search") +async def resolve_search(self, info, contains): + search_books = [b for b in books if contains in b["name"]] + search_magazines = [m for m in magazines if contains in m["name"]] + return search_books + search_magazines + + +@query.field("hello") +@query.field("error_middleware") +async def resolve_hello(self, info): + return "Hello!" + + +@query.field("echo") +async def resolve_echo(self, info, echo): + return echo + + +@query.field("error_non_null") +@query.field("error") +async def resolve_error(self, info): + raise RuntimeError("Runtime Error!") + + +target_schema = make_executable_schema(type_defs, query, mutation, item) +target_asgi_application = AsgiTest(GraphQLASGI(target_schema)) diff --git a/tests/framework_ariadne/_target_schema_sync.py b/tests/framework_ariadne/_target_schema_sync.py new file mode 100644 index 0000000000..8860e71ac5 --- /dev/null +++ b/tests/framework_ariadne/_target_schema_sync.py @@ -0,0 +1,106 @@ +# Copyright 2010 New Relic, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import webtest + +from ariadne import ( + MutationType, + QueryType, + UnionType, + load_schema_from_path, + make_executable_schema, +) +from ariadne.wsgi import GraphQL as GraphQLWSGI +from framework_graphql._target_schema_sync import books, magazines, libraries + +from testing_support.asgi_testing import AsgiTest +from framework_ariadne.test_application import ARIADNE_VERSION + +ariadne_version_tuple = tuple(map(int, ARIADNE_VERSION.split("."))) + +if ariadne_version_tuple < (0, 16): + from ariadne.asgi import GraphQL as GraphQLASGI +elif ariadne_version_tuple >= (0, 16): + from ariadne.asgi.graphql import GraphQL as GraphQLASGI + + +schema_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), "schema.graphql") +type_defs = load_schema_from_path(schema_file) + +storage = [] + +mutation = MutationType() + + + +@mutation.field("storage_add") +def resolve_storage_add(self, info, string): + storage.append(string) + return string + + +item = UnionType("Item") + + +@item.type_resolver +def resolve_type(obj, *args): + if "isbn" in obj: + return "Book" + elif "issue" in obj: # pylint: disable=R1705 + return "Magazine" + + return None + + +query = QueryType() + + +@query.field("library") +def resolve_library(self, info, index): + return libraries[index] + + +@query.field("storage") +def resolve_storage(self, info): + return [storage.pop()] + + +@query.field("search") +def resolve_search(self, info, contains): + search_books = [b for b in books if contains in b["name"]] + search_magazines = [m for m in magazines if contains in m["name"]] + return search_books + search_magazines + + +@query.field("hello") +@query.field("error_middleware") +def resolve_hello(self, info): + return "Hello!" + + +@query.field("echo") +def resolve_echo(self, info, echo): + return echo + + +@query.field("error_non_null") +@query.field("error") +def resolve_error(self, info): + raise RuntimeError("Runtime Error!") + + +target_schema = make_executable_schema(type_defs, query, mutation, item) +target_asgi_application = AsgiTest(GraphQLASGI(target_schema)) +target_wsgi_application = webtest.TestApp(GraphQLWSGI(target_schema)) \ No newline at end of file diff --git a/tests/framework_ariadne/conftest.py b/tests/framework_ariadne/conftest.py index 93623a6852..cbb4448cd5 100644 --- a/tests/framework_ariadne/conftest.py +++ b/tests/framework_ariadne/conftest.py @@ -12,12 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest -import six -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, @@ -29,14 +31,3 @@ app_name="Python Agent Test (framework_ariadne)", default_settings=_default_settings, ) - - -@pytest.fixture(scope="session") -def app(): - from _target_application import _target_application - - return _target_application - - -if six.PY2: - collect_ignore = ["test_application_async.py"] diff --git a/tests/framework_ariadne/schema.graphql b/tests/framework_ariadne/schema.graphql index 4c76e0b88b..8bf64af512 100644 --- a/tests/framework_ariadne/schema.graphql +++ b/tests/framework_ariadne/schema.graphql @@ -33,7 +33,7 @@ type Magazine { } type Mutation { - storage_add(string: String!): StorageAdd + storage_add(string: String!): String } type Query { @@ -44,8 +44,5 @@ type Query { echo(echo: String!): String error: String error_non_null: String! -} - -type StorageAdd { - string: String + error_middleware: String } diff --git a/tests/framework_ariadne/test_application.py b/tests/framework_ariadne/test_application.py index cf8501a7af..0b7bf24898 100644 --- a/tests/framework_ariadne/test_application.py +++ b/tests/framework_ariadne/test_application.py @@ -11,526 +11,27 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import pytest -from testing_support.fixtures import dt_enabled, override_application_settings -from testing_support.validators.validate_span_events import validate_span_events -from testing_support.validators.validate_transaction_count import ( - validate_transaction_count, -) -from testing_support.validators.validate_transaction_errors import ( - validate_transaction_errors, -) -from testing_support.validators.validate_transaction_metrics import ( - validate_transaction_metrics, -) - -from newrelic.api.background_task import background_task -from newrelic.common.object_names import callable_name -from newrelic.common.package_version_utils import get_package_version_tuple - - -@pytest.fixture(scope="session") -def is_graphql_2(): - from graphql import __version__ as version - - major_version = int(version.split(".")[0]) - return major_version == 2 - - -@pytest.fixture(scope="session") -def graphql_run(): - """Wrapper function to simulate framework_graphql test behavior.""" - - def execute(schema, query, *args, **kwargs): - from ariadne import graphql_sync - - return graphql_sync(schema, {"query": query}, *args, **kwargs) - - return execute - - -def to_graphql_source(query): - def delay_import(): - try: - from graphql import Source - except ImportError: - # Fallback if Source is not implemented - return query - - from graphql import __version__ as version - - # For graphql2, Source objects aren't acceptable input - major_version = int(version.split(".")[0]) - if major_version == 2: - return query - - return Source(query) - - return delay_import - - -def example_middleware(next, root, info, **args): # pylint: disable=W0622 - return_value = next(root, info, **args) - return return_value - - -def error_middleware(next, root, info, **args): # pylint: disable=W0622 - raise RuntimeError("Runtime Error!") - - -_runtime_error_name = callable_name(RuntimeError) -_test_runtime_error = [(_runtime_error_name, "Runtime Error!")] -_graphql_base_rollup_metrics = [ - ("OtherTransaction/all", 1), - ("GraphQL/all", 1), - ("GraphQL/allOther", 1), - ("GraphQL/Ariadne/all", 1), - ("GraphQL/Ariadne/allOther", 1), -] - - -def test_basic(app, graphql_run): - from graphql import __version__ as version - - FRAMEWORK_METRICS = [ - ("Python/Framework/Ariadne/None", 1), - ("Python/Framework/GraphQL/%s" % version, 1), - ] - - @validate_transaction_metrics( - "query//hello", - "GraphQL", - rollup_metrics=_graphql_base_rollup_metrics + FRAMEWORK_METRICS, - background_task=True, - ) - @background_task() - def _test(): - ok, response = graphql_run(app, "{ hello }") - assert ok and not response.get("errors") - - _test() - - -@dt_enabled -def test_query_and_mutation(app, graphql_run): - from graphql import __version__ as version - - FRAMEWORK_METRICS = [ - ("Python/Framework/Ariadne/None", 1), - ("Python/Framework/GraphQL/%s" % version, 1), - ] - _test_mutation_scoped_metrics = [ - ("GraphQL/resolve/Ariadne/storage", 1), - ("GraphQL/resolve/Ariadne/storage_add", 1), - ("GraphQL/operation/Ariadne/query//storage", 1), - ("GraphQL/operation/Ariadne/mutation//storage_add.string", 1), - ] - _test_mutation_unscoped_metrics = [ - ("OtherTransaction/all", 1), - ("GraphQL/all", 2), - ("GraphQL/Ariadne/all", 2), - ("GraphQL/allOther", 2), - ("GraphQL/Ariadne/allOther", 2), - ] + _test_mutation_scoped_metrics - - _expected_mutation_operation_attributes = { - "graphql.operation.type": "mutation", - "graphql.operation.name": "", - } - _expected_mutation_resolver_attributes = { - "graphql.field.name": "storage_add", - "graphql.field.parentType": "Mutation", - "graphql.field.path": "storage_add", - "graphql.field.returnType": "StorageAdd", - } - _expected_query_operation_attributes = { - "graphql.operation.type": "query", - "graphql.operation.name": "", - } - _expected_query_resolver_attributes = { - "graphql.field.name": "storage", - "graphql.field.parentType": "Query", - "graphql.field.path": "storage", - "graphql.field.returnType": "[String]", - } - - @validate_transaction_metrics( - "query//storage", - "GraphQL", - scoped_metrics=_test_mutation_scoped_metrics, - rollup_metrics=_test_mutation_unscoped_metrics + FRAMEWORK_METRICS, - background_task=True, - ) - @validate_span_events(exact_agents=_expected_mutation_operation_attributes) - @validate_span_events(exact_agents=_expected_mutation_resolver_attributes) - @validate_span_events(exact_agents=_expected_query_operation_attributes) - @validate_span_events(exact_agents=_expected_query_resolver_attributes) - @background_task() - def _test(): - ok, response = graphql_run(app, 'mutation { storage_add(string: "abc") { string } }') - assert ok and not response.get("errors") - ok, response = graphql_run(app, "query { storage }") - assert ok and not response.get("errors") - - # These are separate assertions because pypy stores 'abc' as a unicode string while other Python versions do not - assert "storage" in str(response["data"]) - assert "abc" in str(response["data"]) - - _test() - - -@dt_enabled -def test_middleware(app, graphql_run, is_graphql_2): - _test_middleware_metrics = [ - ("GraphQL/operation/Ariadne/query//hello", 1), - ("GraphQL/resolve/Ariadne/hello", 1), - ("Function/test_application:example_middleware", 1), - ] - - @validate_transaction_metrics( - "query//hello", - "GraphQL", - scoped_metrics=_test_middleware_metrics, - rollup_metrics=_test_middleware_metrics + _graphql_base_rollup_metrics, - background_task=True, - ) - # Span count 5: Transaction, Operation, Middleware, and 1 Resolver and Resolver function - @validate_span_events(count=5) - @background_task() - def _test(): - from graphql import MiddlewareManager - - middleware = ( - [example_middleware] - if get_package_version_tuple("ariadne") >= (0, 18) - else MiddlewareManager(example_middleware) - ) +from framework_graphql.test_application import * - ok, response = graphql_run(app, "{ hello }", middleware=middleware) - assert ok and not response.get("errors") - assert "Hello!" in str(response["data"]) +from newrelic.common.package_version_utils import get_package_version - _test() +ARIADNE_VERSION = get_package_version("ariadne") +ariadne_version_tuple = tuple(map(int, ARIADNE_VERSION.split("."))) -@dt_enabled -def test_exception_in_middleware(app, graphql_run): - query = "query MyQuery { hello }" - field = "hello" - - # Metrics - _test_exception_scoped_metrics = [ - ("GraphQL/operation/Ariadne/query/MyQuery/%s" % field, 1), - ("GraphQL/resolve/Ariadne/%s" % field, 1), - ] - _test_exception_rollup_metrics = [ - ("Errors/all", 1), - ("Errors/allOther", 1), - ("Errors/OtherTransaction/GraphQL/test_application:error_middleware", 1), - ] + _test_exception_scoped_metrics - - # Attributes - _expected_exception_resolver_attributes = { - "graphql.field.name": field, - "graphql.field.parentType": "Query", - "graphql.field.path": field, - "graphql.field.returnType": "String", - } - _expected_exception_operation_attributes = { - "graphql.operation.type": "query", - "graphql.operation.name": "MyQuery", - "graphql.operation.query": query, - } - - @validate_transaction_metrics( - "test_application:error_middleware", - "GraphQL", - scoped_metrics=_test_exception_scoped_metrics, - rollup_metrics=_test_exception_rollup_metrics + _graphql_base_rollup_metrics, - background_task=True, - ) - @validate_span_events(exact_agents=_expected_exception_operation_attributes) - @validate_span_events(exact_agents=_expected_exception_resolver_attributes) - @validate_transaction_errors(errors=_test_runtime_error) - @background_task() - def _test(): - from graphql import MiddlewareManager - - middleware = ( - [error_middleware] - if get_package_version_tuple("ariadne") >= (0, 18) - else MiddlewareManager(error_middleware) - ) - - _, response = graphql_run(app, query, middleware=middleware) - assert response["errors"] - - _test() - - -@pytest.mark.parametrize("field", ("error", "error_non_null")) -@dt_enabled -def test_exception_in_resolver(app, graphql_run, field): - query = "query MyQuery { %s }" % field - txn_name = "_target_application:resolve_error" - - # Metrics - _test_exception_scoped_metrics = [ - ("GraphQL/operation/Ariadne/query/MyQuery/%s" % field, 1), - ("GraphQL/resolve/Ariadne/%s" % field, 1), - ] - _test_exception_rollup_metrics = [ - ("Errors/all", 1), - ("Errors/allOther", 1), - ("Errors/OtherTransaction/GraphQL/%s" % txn_name, 1), - ] + _test_exception_scoped_metrics - - # Attributes - _expected_exception_resolver_attributes = { - "graphql.field.name": field, - "graphql.field.parentType": "Query", - "graphql.field.path": field, - "graphql.field.returnType": "String!" if "non_null" in field else "String", - } - _expected_exception_operation_attributes = { - "graphql.operation.type": "query", - "graphql.operation.name": "MyQuery", - "graphql.operation.query": query, - } - - @validate_transaction_metrics( - txn_name, - "GraphQL", - scoped_metrics=_test_exception_scoped_metrics, - rollup_metrics=_test_exception_rollup_metrics + _graphql_base_rollup_metrics, - background_task=True, - ) - @validate_span_events(exact_agents=_expected_exception_operation_attributes) - @validate_span_events(exact_agents=_expected_exception_resolver_attributes) - @validate_transaction_errors(errors=_test_runtime_error) - @background_task() - def _test(): - _, response = graphql_run(app, query) - assert response["errors"] - - _test() - - -@dt_enabled -@pytest.mark.parametrize( - "query,exc_class", - [ - ("query MyQuery { missing_field }", "GraphQLError"), - ("{ syntax_error ", "graphql.error.syntax_error:GraphQLSyntaxError"), - ], +@pytest.fixture( + scope="session", params=["sync-sync", "async-sync", "async-async", "wsgi-sync", "asgi-sync", "asgi-async"] ) -def test_exception_in_validation(app, graphql_run, is_graphql_2, query, exc_class): - if "syntax" in query: - txn_name = "graphql.language.parser:parse" - else: - if is_graphql_2: - txn_name = "graphql.validation.validation:validate" - else: - txn_name = "graphql.validation.validate:validate" - - # Import path differs between versions - if exc_class == "GraphQLError": - from graphql.error import GraphQLError - - exc_class = callable_name(GraphQLError) - - _test_exception_scoped_metrics = [ - ("GraphQL/operation/Ariadne///", 1), - ] - _test_exception_rollup_metrics = [ - ("Errors/all", 1), - ("Errors/allOther", 1), - ("Errors/OtherTransaction/GraphQL/%s" % txn_name, 1), - ] + _test_exception_scoped_metrics - - # Attributes - _expected_exception_operation_attributes = { - "graphql.operation.type": "", - "graphql.operation.name": "", - "graphql.operation.query": query, - } - - @validate_transaction_metrics( - txn_name, - "GraphQL", - scoped_metrics=_test_exception_scoped_metrics, - rollup_metrics=_test_exception_rollup_metrics + _graphql_base_rollup_metrics, - background_task=True, - ) - @validate_span_events(exact_agents=_expected_exception_operation_attributes) - @validate_transaction_errors(errors=[exc_class]) - @background_task() - def _test(): - _, response = graphql_run(app, query) - assert response["errors"] - - _test() - - -@dt_enabled -def test_operation_metrics_and_attrs(app, graphql_run): - operation_metrics = [("GraphQL/operation/Ariadne/query/MyQuery/library", 1)] - operation_attrs = { - "graphql.operation.type": "query", - "graphql.operation.name": "MyQuery", - } - - @validate_transaction_metrics( - "query/MyQuery/library", - "GraphQL", - scoped_metrics=operation_metrics, - rollup_metrics=operation_metrics + _graphql_base_rollup_metrics, - background_task=True, - ) - # Span count 16: Transaction, Operation, and 7 Resolvers and Resolver functions - # library, library.name, library.book - # library.book.name and library.book.id for each book resolved (in this case 2) - @validate_span_events(count=16) - @validate_span_events(exact_agents=operation_attrs) - @background_task() - def _test(): - ok, response = graphql_run(app, "query MyQuery { library(index: 0) { branch, book { id, name } } }") - assert ok and not response.get("errors") - - _test() - - -@dt_enabled -def test_field_resolver_metrics_and_attrs(app, graphql_run): - field_resolver_metrics = [("GraphQL/resolve/Ariadne/hello", 1)] - graphql_attrs = { - "graphql.field.name": "hello", - "graphql.field.parentType": "Query", - "graphql.field.path": "hello", - "graphql.field.returnType": "String", - } - - @validate_transaction_metrics( - "query//hello", - "GraphQL", - scoped_metrics=field_resolver_metrics, - rollup_metrics=field_resolver_metrics + _graphql_base_rollup_metrics, - background_task=True, - ) - # Span count 4: Transaction, Operation, and 1 Resolver and Resolver function - @validate_span_events(count=4) - @validate_span_events(exact_agents=graphql_attrs) - @background_task() - def _test(): - ok, response = graphql_run(app, "{ hello }") - assert ok and not response.get("errors") - assert "Hello!" in str(response["data"]) - - _test() - - -_test_queries = [ - ("{ hello }", "{ hello }"), # Basic query extraction - ("{ error }", "{ error }"), # Extract query on field error - ("{ library(index: 0) { branch } }", "{ library(index: ?) { branch } }"), # Integers - ('{ echo(echo: "123") }', "{ echo(echo: ?) }"), # Strings with numerics - ('{ echo(echo: "test") }', "{ echo(echo: ?) }"), # Strings - ('{ TestEcho: echo(echo: "test") }', "{ TestEcho: echo(echo: ?) }"), # Aliases - ('{ TestEcho: echo(echo: "test") }', "{ TestEcho: echo(echo: ?) }"), # Variables - ( # Fragments - '{ ...MyFragment } fragment MyFragment on Query { echo(echo: "test") }', - "{ ...MyFragment } fragment MyFragment on Query { echo(echo: ?) }", - ), -] - - -@dt_enabled -@pytest.mark.parametrize("query,obfuscated", _test_queries) -def test_query_obfuscation(app, graphql_run, query, obfuscated): - graphql_attrs = {"graphql.operation.query": obfuscated} - - @validate_span_events(exact_agents=graphql_attrs) - @background_task() - def _test(): - ok, response = graphql_run(app, query) - if not isinstance(query, str) or "error" not in query: - assert ok and not response.get("errors") - - _test() - - -_test_queries = [ - ("{ hello }", "/hello"), # Basic query - ("{ error }", "/error"), # Extract deepest path on field error - ('{ echo(echo: "test") }', "/echo"), # Fields with arguments - ( - "{ library(index: 0) { branch, book { isbn branch } } }", - "/library", - ), # Complex Example, 1 level - ( - "{ library(index: 0) { book { author { first_name }} } }", - "/library.book.author.first_name", - ), # Complex Example, 2 levels - ("{ library(index: 0) { id, book { name } } }", "/library.book.name"), # Filtering - ('{ TestEcho: echo(echo: "test") }', "/echo"), # Aliases - ( - '{ search(contains: "A") { __typename ... on Book { name } } }', - "/search.name", - ), # InlineFragment - ( - '{ hello echo(echo: "test") }', - "", - ), # Multiple root selections. (need to decide on final behavior) - # FragmentSpread - ( - "{ library(index: 0) { book { ...MyFragment } } } fragment MyFragment on Book { name id }", # Fragment filtering - "/library.book.name", - ), - ( - "{ library(index: 0) { book { ...MyFragment } } } fragment MyFragment on Book { author { first_name } }", - "/library.book.author.first_name", - ), - ( - "{ library(index: 0) { book { ...MyFragment } magazine { ...MagFragment } } } fragment MyFragment on Book { author { first_name } } fragment MagFragment on Magazine { name }", - "/library", - ), -] - - -@dt_enabled -@pytest.mark.parametrize("query,expected_path", _test_queries) -def test_deepest_unique_path(app, graphql_run, query, expected_path): - if expected_path == "/error": - txn_name = "_target_application:resolve_error" - else: - txn_name = "query/%s" % expected_path - - @validate_transaction_metrics( - txn_name, - "GraphQL", - background_task=True, - ) - @background_task() - def _test(): - ok, response = graphql_run(app, query) - if "error" not in query: - assert ok and not response.get("errors") - - _test() - +def target_application(request): + from ._target_application import target_application -@pytest.mark.parametrize("capture_introspection_setting", (True, False)) -def test_introspection_transactions(app, graphql_run, capture_introspection_setting): - txn_ct = 1 if capture_introspection_setting else 0 + target_application = target_application[request.param] - @override_application_settings( - {"instrumentation.graphql.capture_introspection_queries": capture_introspection_setting} - ) - @validate_transaction_count(txn_ct) - @background_task() - def _test(): - ok, response = graphql_run(app, "{ __schema { types { name } } }") - assert ok and not response.get("errors") + param = request.param.split("-") + is_background = param[0] not in {"wsgi", "asgi"} + schema_type = param[1] + extra_spans = 4 if param[0] == "wsgi" else 0 - _test() + assert ARIADNE_VERSION is not None + return "Ariadne", ARIADNE_VERSION, target_application, is_background, schema_type, extra_spans diff --git a/tests/framework_ariadne/test_application_async.py b/tests/framework_ariadne/test_application_async.py deleted file mode 100644 index ada34ffade..0000000000 --- a/tests/framework_ariadne/test_application_async.py +++ /dev/null @@ -1,106 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import asyncio - -import pytest -from testing_support.fixtures import dt_enabled -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_span_events import validate_span_events - -from newrelic.api.background_task import background_task - - -@pytest.fixture(scope="session") -def graphql_run_async(): - """Wrapper function to simulate framework_graphql test behavior.""" - - def execute(schema, query, *args, **kwargs): - from ariadne import graphql - - return graphql(schema, {"query": query}, *args, **kwargs) - - return execute - - -@dt_enabled -def test_query_and_mutation_async(app, graphql_run_async): - from graphql import __version__ as version - - FRAMEWORK_METRICS = [ - ("Python/Framework/Ariadne/None", 1), - ("Python/Framework/GraphQL/%s" % version, 1), - ] - _test_mutation_scoped_metrics = [ - ("GraphQL/resolve/Ariadne/storage", 1), - ("GraphQL/resolve/Ariadne/storage_add", 1), - ("GraphQL/operation/Ariadne/query//storage", 1), - ("GraphQL/operation/Ariadne/mutation//storage_add.string", 1), - ] - _test_mutation_unscoped_metrics = [ - ("OtherTransaction/all", 1), - ("GraphQL/all", 2), - ("GraphQL/Ariadne/all", 2), - ("GraphQL/allOther", 2), - ("GraphQL/Ariadne/allOther", 2), - ] + _test_mutation_scoped_metrics - - _expected_mutation_operation_attributes = { - "graphql.operation.type": "mutation", - "graphql.operation.name": "", - } - _expected_mutation_resolver_attributes = { - "graphql.field.name": "storage_add", - "graphql.field.parentType": "Mutation", - "graphql.field.path": "storage_add", - "graphql.field.returnType": "StorageAdd", - } - _expected_query_operation_attributes = { - "graphql.operation.type": "query", - "graphql.operation.name": "", - } - _expected_query_resolver_attributes = { - "graphql.field.name": "storage", - "graphql.field.parentType": "Query", - "graphql.field.path": "storage", - "graphql.field.returnType": "[String]", - } - - @validate_transaction_metrics( - "query//storage", - "GraphQL", - scoped_metrics=_test_mutation_scoped_metrics, - rollup_metrics=_test_mutation_unscoped_metrics + FRAMEWORK_METRICS, - background_task=True, - ) - @validate_span_events(exact_agents=_expected_mutation_operation_attributes) - @validate_span_events(exact_agents=_expected_mutation_resolver_attributes) - @validate_span_events(exact_agents=_expected_query_operation_attributes) - @validate_span_events(exact_agents=_expected_query_resolver_attributes) - @background_task() - def _test(): - async def coro(): - ok, response = await graphql_run_async(app, 'mutation { storage_add(string: "abc") { string } }') - assert ok and not response.get("errors") - ok, response = await graphql_run_async(app, "query { storage }") - assert ok and not response.get("errors") - - # These are separate assertions because pypy stores 'abc' as a unicode string while other Python versions do not - assert "storage" in str(response.get("data")) - assert "abc" in str(response.get("data")) - - loop = asyncio.new_event_loop() - loop.run_until_complete(coro()) - - _test() diff --git a/tests/framework_ariadne/test_asgi.py b/tests/framework_ariadne/test_asgi.py deleted file mode 100644 index 861f2aa932..0000000000 --- a/tests/framework_ariadne/test_asgi.py +++ /dev/null @@ -1,118 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json - -import pytest -from testing_support.asgi_testing import AsgiTest -from testing_support.fixtures import dt_enabled -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_span_events import validate_span_events - - -@pytest.fixture(scope="session") -def graphql_asgi_run(): - """Wrapper function to simulate framework_graphql test behavior.""" - from _target_application import _target_asgi_application - - app = AsgiTest(_target_asgi_application) - - def execute(query): - return app.make_request( - "POST", "/", headers={"Content-Type": "application/json"}, body=json.dumps({"query": query}) - ) - - return execute - - -@dt_enabled -def test_query_and_mutation_asgi(graphql_asgi_run): - from graphql import __version__ as version - - FRAMEWORK_METRICS = [ - ("Python/Framework/Ariadne/None", 1), - ("Python/Framework/GraphQL/%s" % version, 1), - ] - _test_mutation_scoped_metrics = [ - ("GraphQL/resolve/Ariadne/storage_add", 1), - ("GraphQL/operation/Ariadne/mutation//storage_add.string", 1), - ] - _test_query_scoped_metrics = [ - ("GraphQL/resolve/Ariadne/storage", 1), - ("GraphQL/operation/Ariadne/query//storage", 1), - ] - _test_unscoped_metrics = [ - ("WebTransaction", 1), - ("GraphQL/all", 1), - ("GraphQL/Ariadne/all", 1), - ("GraphQL/allWeb", 1), - ("GraphQL/Ariadne/allWeb", 1), - ] - _test_mutation_unscoped_metrics = _test_unscoped_metrics + _test_mutation_scoped_metrics - _test_query_unscoped_metrics = _test_unscoped_metrics + _test_query_scoped_metrics - - _expected_mutation_operation_attributes = { - "graphql.operation.type": "mutation", - "graphql.operation.name": "", - } - _expected_mutation_resolver_attributes = { - "graphql.field.name": "storage_add", - "graphql.field.parentType": "Mutation", - "graphql.field.path": "storage_add", - "graphql.field.returnType": "StorageAdd", - } - _expected_query_operation_attributes = { - "graphql.operation.type": "query", - "graphql.operation.name": "", - } - _expected_query_resolver_attributes = { - "graphql.field.name": "storage", - "graphql.field.parentType": "Query", - "graphql.field.path": "storage", - "graphql.field.returnType": "[String]", - } - - @validate_transaction_metrics( - "query//storage", - "GraphQL", - scoped_metrics=_test_query_scoped_metrics, - rollup_metrics=_test_query_unscoped_metrics + FRAMEWORK_METRICS, - ) - @validate_transaction_metrics( - "mutation//storage_add.string", - "GraphQL", - scoped_metrics=_test_mutation_scoped_metrics, - rollup_metrics=_test_mutation_unscoped_metrics + FRAMEWORK_METRICS, - index=-2, - ) - @validate_span_events(exact_agents=_expected_mutation_operation_attributes, index=-2) - @validate_span_events(exact_agents=_expected_mutation_resolver_attributes, index=-2) - @validate_span_events(exact_agents=_expected_query_operation_attributes) - @validate_span_events(exact_agents=_expected_query_resolver_attributes) - def _test(): - response = graphql_asgi_run('mutation { storage_add(string: "abc") { string } }') - assert response.status == 200 - response = json.loads(response.body.decode("utf-8")) - assert not response.get("errors") - - response = graphql_asgi_run("query { storage }") - assert response.status == 200 - response = json.loads(response.body.decode("utf-8")) - assert not response.get("errors") - - # These are separate assertions because pypy stores 'abc' as a unicode string while other Python versions do not - assert "storage" in str(response.get("data")) - assert "abc" in str(response.get("data")) - - _test() diff --git a/tests/framework_ariadne/test_wsgi.py b/tests/framework_ariadne/test_wsgi.py deleted file mode 100644 index 9ce2373d47..0000000000 --- a/tests/framework_ariadne/test_wsgi.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright 2010 New Relic, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pytest -import webtest -from testing_support.fixtures import dt_enabled -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_span_events import validate_span_events - - -@pytest.fixture(scope="session") -def graphql_wsgi_run(): - """Wrapper function to simulate framework_graphql test behavior.""" - from _target_application import _target_wsgi_application - - app = webtest.TestApp(_target_wsgi_application) - - def execute(query): - return app.post_json("/", {"query": query}) - - return execute - - -@dt_enabled -def test_query_and_mutation_wsgi(graphql_wsgi_run): - from graphql import __version__ as version - - FRAMEWORK_METRICS = [ - ("Python/Framework/Ariadne/None", 1), - ("Python/Framework/GraphQL/%s" % version, 1), - ] - _test_mutation_scoped_metrics = [ - ("GraphQL/resolve/Ariadne/storage_add", 1), - ("GraphQL/operation/Ariadne/mutation//storage_add.string", 1), - ] - _test_query_scoped_metrics = [ - ("GraphQL/resolve/Ariadne/storage", 1), - ("GraphQL/operation/Ariadne/query//storage", 1), - ] - _test_unscoped_metrics = [ - ("WebTransaction", 1), - ("Python/WSGI/Response", 1), - ("GraphQL/all", 1), - ("GraphQL/Ariadne/all", 1), - ("GraphQL/allWeb", 1), - ("GraphQL/Ariadne/allWeb", 1), - ] - _test_mutation_unscoped_metrics = _test_unscoped_metrics + _test_mutation_scoped_metrics - _test_query_unscoped_metrics = _test_unscoped_metrics + _test_query_scoped_metrics - - _expected_mutation_operation_attributes = { - "graphql.operation.type": "mutation", - "graphql.operation.name": "", - } - _expected_mutation_resolver_attributes = { - "graphql.field.name": "storage_add", - "graphql.field.parentType": "Mutation", - "graphql.field.path": "storage_add", - "graphql.field.returnType": "StorageAdd", - } - _expected_query_operation_attributes = { - "graphql.operation.type": "query", - "graphql.operation.name": "", - } - _expected_query_resolver_attributes = { - "graphql.field.name": "storage", - "graphql.field.parentType": "Query", - "graphql.field.path": "storage", - "graphql.field.returnType": "[String]", - } - - @validate_transaction_metrics( - "query//storage", - "GraphQL", - scoped_metrics=_test_query_scoped_metrics, - rollup_metrics=_test_query_unscoped_metrics + FRAMEWORK_METRICS, - ) - @validate_transaction_metrics( - "mutation//storage_add.string", - "GraphQL", - scoped_metrics=_test_mutation_scoped_metrics, - rollup_metrics=_test_mutation_unscoped_metrics + FRAMEWORK_METRICS, - index=-2, - ) - @validate_span_events(exact_agents=_expected_mutation_operation_attributes, index=-2) - @validate_span_events(exact_agents=_expected_mutation_resolver_attributes, index=-2) - @validate_span_events(exact_agents=_expected_query_operation_attributes) - @validate_span_events(exact_agents=_expected_query_resolver_attributes) - def _test(): - response = graphql_wsgi_run('mutation { storage_add(string: "abc") { string } }') - assert response.status_code == 200 - response = response.json_body - assert not response.get("errors") - - response = graphql_wsgi_run("query { storage }") - assert response.status_code == 200 - response = response.json_body - assert not response.get("errors") - - # These are separate assertions because pypy stores 'abc' as a unicode string while other Python versions do not - assert "storage" in str(response.get("data")) - assert "abc" in str(response.get("data")) - - _test() diff --git a/tests/framework_bottle/conftest.py b/tests/framework_bottle/conftest.py index 095a3331f3..1e20551a52 100644 --- a/tests/framework_bottle/conftest.py +++ b/tests/framework_bottle/conftest.py @@ -13,23 +13,31 @@ # limitations under the License. import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "security.agent.enabled": True, + "security.enabled": True, + "security.mode": "IAST", + "security.validator_service_url": "wss://csec-staging.nr-data.net" } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (framework_bottle)', - default_settings=_default_settings) + app_name="Python Agent Test (framework_bottle)", default_settings=_default_settings +) -@pytest.fixture(scope='function') + +@pytest.fixture(scope="function") def target_application(): import _target_application + return _target_application.target_application diff --git a/tests/framework_bottle/test_application.py b/tests/framework_bottle/test_application.py index 28619d5eb5..32db9a64a5 100644 --- a/tests/framework_bottle/test_application.py +++ b/tests/framework_bottle/test_application.py @@ -12,218 +12,233 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest import base64 +import pytest +import webtest +from bottle import __version__ as version from testing_support.fixtures import ( + override_application_settings, override_ignore_status_codes, - override_application_settings) -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from newrelic.packages import six -from testing_support.validators.validate_code_level_metrics import validate_code_level_metrics -from testing_support.validators.validate_transaction_errors import validate_transaction_errors - -import webtest +) +from testing_support.validators.validate_code_level_metrics import ( + validate_code_level_metrics, +) +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) -from bottle import __version__ as version +from newrelic.common.package_version_utils import get_package_version_tuple -version = [int(x) for x in version.split('-')[0].split('.')] +version = list(get_package_version_tuple("bottle")) if len(version) == 2: version.append(0) version = tuple(version) +assert version > (0, 1), "version information not found" + +version_metrics = [(f"Python/Framework/Bottle/{'.'.join(str(v) for v in version)}", 1)] -requires_auth_basic = pytest.mark.skipif(version < (0, 9, 0), - reason="Bottle only added auth_basic in 0.9.0.") -requires_plugins = pytest.mark.skipif(version < (0, 9, 0), - reason="Bottle only added auth_basic in 0.9.0.") +requires_auth_basic = pytest.mark.skipif(version < (0, 9, 0), reason="Bottle only added auth_basic in 0.9.0.") +requires_plugins = pytest.mark.skipif(version < (0, 9, 0), reason="Bottle only added auth_basic in 0.9.0.") _test_application_index_scoped_metrics = [ - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/_target_application:index_page', 1)] + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/_target_application:index_page", 1), +] if version >= (0, 9, 0): - _test_application_index_scoped_metrics.extend([ - ('Function/bottle:Bottle.wsgi', 1)]) + _test_application_index_scoped_metrics.extend([("Function/bottle:Bottle.wsgi", 1)]) else: - _test_application_index_scoped_metrics.extend([ - ('Function/bottle:Bottle.__call__', 1)]) + _test_application_index_scoped_metrics.extend([("Function/bottle:Bottle.__call__", 1)]) + +_test_application_index_custom_metrics = version_metrics.copy() -_test_application_index_custom_metrics = [ - ('Python/Framework/Bottle/%s.%s.%s' % version, 1)] @validate_code_level_metrics("_target_application", "index_page") @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('_target_application:index_page', - scoped_metrics=_test_application_index_scoped_metrics, - custom_metrics=_test_application_index_custom_metrics) +@validate_transaction_metrics( + "_target_application:index_page", + scoped_metrics=_test_application_index_scoped_metrics, + custom_metrics=_test_application_index_custom_metrics, +) def test_application_index(target_application): - response = target_application.get('/index') - response.mustcontain('INDEX RESPONSE') + response = target_application.get("/index") + response.mustcontain("INDEX RESPONSE") + _test_application_error_scoped_metrics = [ - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/_target_application:error_page', 1)] + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/_target_application:error_page", 1), +] if version >= (0, 9, 0): - _test_application_error_scoped_metrics.extend([ - ('Function/bottle:Bottle.wsgi', 1)]) + _test_application_error_scoped_metrics.extend([("Function/bottle:Bottle.wsgi", 1)]) else: - _test_application_error_scoped_metrics.extend([ - ('Function/bottle:Bottle.__call__', 1)]) + _test_application_error_scoped_metrics.extend([("Function/bottle:Bottle.__call__", 1)]) -_test_application_error_custom_metrics = [ - ('Python/Framework/Bottle/%s.%s.%s' % version, 1)] +_test_application_error_custom_metrics = version_metrics.copy() +_test_application_error_errors = ["builtins:RuntimeError"] -if six.PY3: - _test_application_error_errors = ['builtins:RuntimeError'] -else: - _test_application_error_errors = ['exceptions:RuntimeError'] @validate_code_level_metrics("_target_application", "error_page") @validate_transaction_errors(errors=_test_application_error_errors) -@validate_transaction_metrics('_target_application:error_page', - scoped_metrics=_test_application_error_scoped_metrics, - custom_metrics=_test_application_error_custom_metrics) +@validate_transaction_metrics( + "_target_application:error_page", + scoped_metrics=_test_application_error_scoped_metrics, + custom_metrics=_test_application_error_custom_metrics, +) def test_application_error(target_application): - response = target_application.get('/error', status=500, expect_errors=True) + response = target_application.get("/error", status=500, expect_errors=True) + _test_application_not_found_scoped_metrics = [ - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/_target_application:error404_page', 1)] + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/_target_application:error404_page", 1), +] if version >= (0, 9, 0): - _test_application_not_found_scoped_metrics.extend([ - ('Function/bottle:Bottle.wsgi', 1)]) + _test_application_not_found_scoped_metrics.extend([("Function/bottle:Bottle.wsgi", 1)]) else: - _test_application_not_found_scoped_metrics.extend([ - ('Function/bottle:Bottle.__call__', 1)]) + _test_application_not_found_scoped_metrics.extend([("Function/bottle:Bottle.__call__", 1)]) + +_test_application_not_found_custom_metrics = version_metrics.copy() -_test_application_not_found_custom_metrics = [ - ('Python/Framework/Bottle/%s.%s.%s' % version, 1)] @validate_code_level_metrics("_target_application", "error404_page") @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('_target_application:error404_page', - scoped_metrics=_test_application_not_found_scoped_metrics, - custom_metrics=_test_application_not_found_custom_metrics) +@validate_transaction_metrics( + "_target_application:error404_page", + scoped_metrics=_test_application_not_found_scoped_metrics, + custom_metrics=_test_application_not_found_custom_metrics, +) def test_application_not_found(target_application): - response = target_application.get('/missing', status=404) - response.mustcontain('NOT FOUND') + response = target_application.get("/missing", status=404) + response.mustcontain("NOT FOUND") + _test_application_auth_basic_fail_scoped_metrics = [ - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/_target_application:auth_basic_page', 1)] + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/_target_application:auth_basic_page", 1), +] if version >= (0, 9, 0): - _test_application_auth_basic_fail_scoped_metrics.extend([ - ('Function/bottle:Bottle.wsgi', 1)]) + _test_application_auth_basic_fail_scoped_metrics.extend([("Function/bottle:Bottle.wsgi", 1)]) else: - _test_application_auth_basic_fail_scoped_metrics.extend([ - ('Function/bottle:Bottle.__call__', 1)]) + _test_application_auth_basic_fail_scoped_metrics.extend([("Function/bottle:Bottle.__call__", 1)]) + +_test_application_auth_basic_fail_custom_metrics = version_metrics.copy() -_test_application_auth_basic_fail_custom_metrics = [ - ('Python/Framework/Bottle/%s.%s.%s' % version, 1)] @requires_auth_basic @validate_code_level_metrics("_target_application", "auth_basic_page") @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('_target_application:auth_basic_page', - scoped_metrics=_test_application_auth_basic_fail_scoped_metrics, - custom_metrics=_test_application_auth_basic_fail_custom_metrics) +@validate_transaction_metrics( + "_target_application:auth_basic_page", + scoped_metrics=_test_application_auth_basic_fail_scoped_metrics, + custom_metrics=_test_application_auth_basic_fail_custom_metrics, +) def test_application_auth_basic_fail(target_application): - response = target_application.get('/auth', status=401) + response = target_application.get("/auth", status=401) + _test_application_auth_basic_okay_scoped_metrics = [ - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/_target_application:auth_basic_page', 1)] + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/_target_application:auth_basic_page", 1), +] if version >= (0, 9, 0): - _test_application_auth_basic_okay_scoped_metrics.extend([ - ('Function/bottle:Bottle.wsgi', 1)]) + _test_application_auth_basic_okay_scoped_metrics.extend([("Function/bottle:Bottle.wsgi", 1)]) else: - _test_application_auth_basic_okay_scoped_metrics.extend([ - ('Function/bottle:Bottle.__call__', 1)]) + _test_application_auth_basic_okay_scoped_metrics.extend([("Function/bottle:Bottle.__call__", 1)]) + +_test_application_auth_basic_okay_custom_metrics = version_metrics.copy() -_test_application_auth_basic_okay_custom_metrics = [ - ('Python/Framework/Bottle/%s.%s.%s' % version, 1)] @requires_auth_basic @validate_code_level_metrics("_target_application", "auth_basic_page") @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('_target_application:auth_basic_page', - scoped_metrics=_test_application_auth_basic_okay_scoped_metrics, - custom_metrics=_test_application_auth_basic_okay_custom_metrics) +@validate_transaction_metrics( + "_target_application:auth_basic_page", + scoped_metrics=_test_application_auth_basic_okay_scoped_metrics, + custom_metrics=_test_application_auth_basic_okay_custom_metrics, +) def test_application_auth_basic_okay(target_application): - authorization_value = base64.b64encode(b'user:password') - if six.PY3: - authorization_value = authorization_value.decode('Latin-1') - environ = { 'HTTP_AUTHORIZATION': 'Basic ' + authorization_value } - response = target_application.get('/auth', extra_environ=environ) - response.mustcontain('AUTH OKAY') + authorization_value = base64.b64encode(b"user:password").decode("Latin-1") + environ = {"HTTP_AUTHORIZATION": f"Basic {authorization_value}"} + response = target_application.get("/auth", extra_environ=environ) + response.mustcontain("AUTH OKAY") + _test_application_plugin_error_scoped_metrics = [ - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/_target_application:plugin_error_page', 1)] + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/_target_application:plugin_error_page", 1), +] if version >= (0, 9, 0): - _test_application_plugin_error_scoped_metrics.extend([ - ('Function/bottle:Bottle.wsgi', 1)]) + _test_application_plugin_error_scoped_metrics.extend([("Function/bottle:Bottle.wsgi", 1)]) else: - _test_application_plugin_error_scoped_metrics.extend([ - ('Function/bottle:Bottle.__call__', 1)]) + _test_application_plugin_error_scoped_metrics.extend([("Function/bottle:Bottle.__call__", 1)]) + +_test_application_plugin_error_custom_metrics = version_metrics.copy() -_test_application_plugin_error_custom_metrics = [ - ('Python/Framework/Bottle/%s.%s.%s' % version, 1)] @requires_plugins @validate_code_level_metrics("_target_application", "plugin_error_page") @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('_target_application:plugin_error_page', - scoped_metrics=_test_application_plugin_error_scoped_metrics, - custom_metrics=_test_application_plugin_error_custom_metrics) +@validate_transaction_metrics( + "_target_application:plugin_error_page", + scoped_metrics=_test_application_plugin_error_scoped_metrics, + custom_metrics=_test_application_plugin_error_custom_metrics, +) @override_ignore_status_codes([403]) def test_application_plugin_error_ignore(target_application): - response = target_application.get('/plugin_error', status=403, - expect_errors=True) + response = target_application.get("/plugin_error", status=403, expect_errors=True) + @requires_plugins @validate_code_level_metrics("_target_application", "plugin_error_page") -@validate_transaction_errors(errors=['bottle:HTTPError']) -@validate_transaction_metrics('_target_application:plugin_error_page', - scoped_metrics=_test_application_plugin_error_scoped_metrics, - custom_metrics=_test_application_plugin_error_custom_metrics) +@validate_transaction_errors(errors=["bottle:HTTPError"]) +@validate_transaction_metrics( + "_target_application:plugin_error_page", + scoped_metrics=_test_application_plugin_error_scoped_metrics, + custom_metrics=_test_application_plugin_error_custom_metrics, +) def test_application_plugin_error_capture(target_application): import newrelic.agent - response = target_application.get('/plugin_error', status=403, - expect_errors=True) + + response = target_application.get("/plugin_error", status=403, expect_errors=True) + _test_html_insertion_settings = { - 'browser_monitoring.enabled': True, - 'browser_monitoring.auto_instrument': True, - 'js_agent_loader': u'', + "browser_monitoring.enabled": True, + "browser_monitoring.auto_instrument": True, + "js_agent_loader": "", } + @override_application_settings(_test_html_insertion_settings) def test_html_insertion(target_application): - response = target_application.get('/html_insertion') + response = target_application.get("/html_insertion") # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. - - response.mustcontain('NREUM HEADER', 'NREUM.info') + # header added by the agent. + response.mustcontain("NREUM HEADER", "NREUM.info") diff --git a/tests/framework_cherrypy/conftest.py b/tests/framework_cherrypy/conftest.py index bc730bb1fa..181bbcb008 100644 --- a/tests/framework_cherrypy/conftest.py +++ b/tests/framework_cherrypy/conftest.py @@ -12,19 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (framework_cherrypy)', - default_settings=_default_settings) + app_name="Python Agent Test (framework_cherrypy)", default_settings=_default_settings +) diff --git a/tests/framework_cherrypy/test_application.py b/tests/framework_cherrypy/test_application.py index 39f8b5c16d..53e20ad77d 100644 --- a/tests/framework_cherrypy/test_application.py +++ b/tests/framework_cherrypy/test_application.py @@ -12,31 +12,32 @@ # See the License for the specific language governing permissions and # limitations under the License. +import cherrypy import pytest import webtest - -from newrelic.packages import six - from testing_support.fixtures import ( - override_application_settings, - override_ignore_status_codes) -from testing_support.validators.validate_code_level_metrics import validate_code_level_metrics -from testing_support.validators.validate_transaction_errors import validate_transaction_errors - -import cherrypy + override_application_settings, + override_ignore_status_codes, +) +from testing_support.validators.validate_code_level_metrics import ( + validate_code_level_metrics, +) +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) -CHERRYPY_VERSION = tuple(int(v) for v in cherrypy.__version__.split('.')) +CHERRYPY_VERSION = tuple(int(v) for v in cherrypy.__version__.split(".")) -class Application(object): +class Application(): @cherrypy.expose def index(self): - return 'INDEX RESPONSE' + return "INDEX RESPONSE" @cherrypy.expose def error(self): - raise RuntimeError('error') + raise RuntimeError("error") @cherrypy.expose def not_found(self): @@ -48,35 +49,37 @@ def not_found_as_http_error(self): @cherrypy.expose def not_found_as_str_http_error(self): - raise cherrypy.HTTPError('404 Not Found') + raise cherrypy.HTTPError("404 Not Found") @cherrypy.expose def bad_http_error(self): # this will raise HTTPError with status code 500 because 10 is not a # valid status code - raise cherrypy.HTTPError('10 Invalid status code') + raise cherrypy.HTTPError("10 Invalid status code") @cherrypy.expose def internal_redirect(self): - raise cherrypy.InternalRedirect('/') + raise cherrypy.InternalRedirect("/") @cherrypy.expose def external_redirect(self): - raise cherrypy.HTTPRedirect('/') + raise cherrypy.HTTPRedirect("/") @cherrypy.expose def upload_files(self, files): - return 'UPLOAD FILES RESPONSE' + return "UPLOAD FILES RESPONSE" @cherrypy.expose def encode_multipart(self, field, files): - return 'ENCODE MULTIPART RESPONSE' + return "ENCODE MULTIPART RESPONSE" @cherrypy.expose def html_insertion(self): - return ('Some header' - '

My First Heading

My first paragraph.

' - '') + return ( + "Some header" + "

My First Heading

My first paragraph.

" + "" + ) application = cherrypy.Application(Application()) @@ -86,99 +89,91 @@ def html_insertion(self): @validate_code_level_metrics("test_application.Application", "index") @validate_transaction_errors(errors=[]) def test_application_index(): - response = test_application.get('') - response.mustcontain('INDEX RESPONSE') + response = test_application.get("") + response.mustcontain("INDEX RESPONSE") @validate_transaction_errors(errors=[]) def test_application_index_agent_disabled(): - environ = {'newrelic.enabled': False} - response = test_application.get('', extra_environ=environ) - response.mustcontain('INDEX RESPONSE') + environ = {"newrelic.enabled": False} + response = test_application.get("", extra_environ=environ) + response.mustcontain("INDEX RESPONSE") @validate_transaction_errors(errors=[]) def test_application_missing(): - test_application.get('/missing', status=404) - + test_application.get("/missing", status=404) -if six.PY3: - _test_application_unexpected_exception_errors = ['builtins:RuntimeError'] -else: - _test_application_unexpected_exception_errors = ['exceptions:RuntimeError'] - -@validate_transaction_errors( - errors=_test_application_unexpected_exception_errors) +@validate_transaction_errors(errors=["builtins:RuntimeError"]) def test_application_unexpected_exception(): - test_application.get('/error', status=500) + test_application.get("/error", status=500) @validate_transaction_errors(errors=[]) def test_application_not_found(): - test_application.get('/not_found', status=404) + test_application.get("/not_found", status=404) @validate_transaction_errors(errors=[]) def test_application_not_found_as_http_error(): - test_application.get('/not_found_as_http_error', status=404) + test_application.get("/not_found_as_http_error", status=404) @validate_transaction_errors(errors=[]) def test_application_internal_redirect(): - response = test_application.get('/internal_redirect') - response.mustcontain('INDEX RESPONSE') + response = test_application.get("/internal_redirect") + response.mustcontain("INDEX RESPONSE") @validate_transaction_errors(errors=[]) def test_application_external_redirect(): - test_application.get('/external_redirect', status=302) + test_application.get("/external_redirect", status=302) @validate_transaction_errors(errors=[]) def test_application_upload_files(): - test_application.post('/upload_files', upload_files=[('files', __file__)]) + test_application.post("/upload_files", upload_files=[("files", __file__)]) @validate_transaction_errors(errors=[]) def test_application_encode_multipart(): - content_type, body = test_application.encode_multipart( - params=[('field', 'value')], files=[('files', __file__)]) - test_application.request('/encode_multipart', method='POST', - content_type=content_type, body=body) + content_type, body = test_application.encode_multipart(params=[("field", "value")], files=[("files", __file__)]) + test_application.request("/encode_multipart", method="POST", content_type=content_type, body=body) _test_html_insertion_settings = { - 'browser_monitoring.enabled': True, - 'browser_monitoring.auto_instrument': True, - 'js_agent_loader': u'', + "browser_monitoring.enabled": True, + "browser_monitoring.auto_instrument": True, + "js_agent_loader": "", } @override_application_settings(_test_html_insertion_settings) def test_html_insertion(): - response = test_application.get('/html_insertion') + response = test_application.get("/html_insertion") # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. - response.mustcontain('NREUM HEADER', 'NREUM.info') + response.mustcontain("NREUM HEADER", "NREUM.info") -_error_endpoints = ['/not_found_as_http_error'] +_error_endpoints = ["/not_found_as_http_error"] if CHERRYPY_VERSION >= (3, 2): - _error_endpoints.extend(['/not_found_as_str_http_error', - '/bad_http_error']) + _error_endpoints.extend(["/not_found_as_str_http_error", "/bad_http_error"]) -@pytest.mark.parametrize('endpoint', _error_endpoints) -@pytest.mark.parametrize('ignore_overrides,expected_errors', [ - ([], ['cherrypy._cperror:HTTPError']), - ([404, 500], []), -]) +@pytest.mark.parametrize("endpoint", _error_endpoints) +@pytest.mark.parametrize( + "ignore_overrides,expected_errors", + [ + ([], ["cherrypy._cperror:HTTPError"]), + ([404, 500], []), + ], +) def test_ignore_status_code(endpoint, ignore_overrides, expected_errors): - @validate_transaction_errors(errors=expected_errors) @override_ignore_status_codes(ignore_overrides) def _test(): @@ -189,5 +184,5 @@ def _test(): @validate_transaction_errors(errors=[]) def test_ignore_status_unexpected_param(): - response = test_application.get('/?arg=1', status=404) - response.mustcontain(no=['INDEX RESPONSE']) + response = test_application.get("/?arg=1", status=404) + response.mustcontain(no=["INDEX RESPONSE"]) diff --git a/tests/framework_cherrypy/test_dispatch.py b/tests/framework_cherrypy/test_dispatch.py index 64dccb2146..bc756fa2ec 100644 --- a/tests/framework_cherrypy/test_dispatch.py +++ b/tests/framework_cherrypy/test_dispatch.py @@ -15,7 +15,6 @@ import pytest import webtest -from newrelic.packages import six from testing_support.validators.validate_transaction_errors import validate_transaction_errors @@ -27,7 +26,7 @@ requires_cherrypy32 = pytest.mark.skipif(not is_ge_cherrypy32, reason="The dispatch mechanism was only added in CherryPy 3.2.") -class Resource(object): +class Resource(): def _cp_dispatch(self, vpath): raise RuntimeError('dispatch error') @@ -40,12 +39,8 @@ def _cp_dispatch(self, vpath): application = cherrypy.Application(Resource(), '/', conf) test_application = webtest.TestApp(application) -if six.PY3: - _test_dispatch_exception_errors = ['builtins:RuntimeError'] -else: - _test_dispatch_exception_errors = ['exceptions:RuntimeError'] @requires_cherrypy32 -@validate_transaction_errors(errors=_test_dispatch_exception_errors) +@validate_transaction_errors(errors=['builtins:RuntimeError']) def test_dispatch_exception(): response = test_application.get('/sub/a/b', status=500) diff --git a/tests/framework_cherrypy/test_resource.py b/tests/framework_cherrypy/test_resource.py index 385d28d91e..4d6f949fb9 100644 --- a/tests/framework_cherrypy/test_resource.py +++ b/tests/framework_cherrypy/test_resource.py @@ -19,7 +19,7 @@ import cherrypy -class Resource(object): +class Resource(): exposed = True diff --git a/tests/framework_cherrypy/test_routes.py b/tests/framework_cherrypy/test_routes.py index 9111a29ced..464fcc7c1e 100644 --- a/tests/framework_cherrypy/test_routes.py +++ b/tests/framework_cherrypy/test_routes.py @@ -21,7 +21,7 @@ import cherrypy -class EndPoint(object): +class EndPoint(): def index(self): return 'INDEX RESPONSE' diff --git a/tests/framework_django/conftest.py b/tests/framework_django/conftest.py index 8a43ef5c90..bfc90d0e08 100644 --- a/tests/framework_django/conftest.py +++ b/tests/framework_django/conftest.py @@ -12,21 +12,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, - 'debug.log_autorum_middleware': True, - 'feature_flag': set(['django.instrumentation.inclusion-tags.r1']), + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_autorum_middleware": True, + "feature_flag": set(["django.instrumentation.inclusion-tags.r1"]), + "security.agent.enabled": True, + "security.enabled": True, + "security.mode": "IAST", + "security.validator_service_url": "wss://csec-staging.nr-data.net" } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (framework_django)', - default_settings=_default_settings) + app_name="Python Agent Test (framework_django)", default_settings=_default_settings +) diff --git a/tests/framework_django/middleware.py b/tests/framework_django/middleware.py index 2d9e794467..0e6669ae3f 100644 --- a/tests/framework_django/middleware.py +++ b/tests/framework_django/middleware.py @@ -19,7 +19,7 @@ class Custom410(Exception): pass -class ExceptionTo410Middleware(object): +class ExceptionTo410Middleware(): def __init__(self, get_response=None): self.get_response = get_response diff --git a/tests/framework_django/templates/main.html b/tests/framework_django/templates/main.html index bcf5afda39..5de5a534a3 100644 --- a/tests/framework_django/templates/main.html +++ b/tests/framework_django/templates/main.html @@ -26,6 +26,5 @@

My First Heading

My first paragraph.

{% show_results %} - {% newrelic_browser_timing_footer %} diff --git a/tests/framework_django/test_application.py b/tests/framework_django/test_application.py index 1f2616b0fa..82501707b2 100644 --- a/tests/framework_django/test_application.py +++ b/tests/framework_django/test_application.py @@ -12,24 +12,33 @@ # See the License for the specific language governing permissions and # limitations under the License. -from testing_support.fixtures import ( - override_application_settings, - override_generic_settings, override_ignore_status_codes) -from testing_support.validators.validate_code_level_metrics import validate_code_level_metrics -from newrelic.hooks.framework_django import django_settings -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_transaction_errors import validate_transaction_errors - import os import django +from testing_support.fixtures import ( + override_application_settings, + override_generic_settings, + override_ignore_status_codes, +) +from testing_support.validators.validate_code_level_metrics import ( + validate_code_level_metrics, +) +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) + +from newrelic.hooks.framework_django import django_settings -DJANGO_VERSION = tuple(map(int, django.get_version().split('.')[:2])) -DJANGO_SETTINGS_MODULE = os.environ.get('DJANGO_SETTINGS_MODULE', None) +DJANGO_VERSION = tuple(map(int, django.get_version().split(".")[:2])) +DJANGO_SETTINGS_MODULE = os.environ.get("DJANGO_SETTINGS_MODULE", None) def target_application(): from _target_application import _target_application + return _target_application @@ -37,272 +46,233 @@ def target_application(): # MIDDLEWARE defined in the version-specific Django settings.py file. _test_django_pre_1_10_middleware_scoped_metrics = [ - (('Function/django.middleware.common:' - 'CommonMiddleware.process_request'), 1), - (('Function/django.contrib.sessions.middleware:' - 'SessionMiddleware.process_request'), 1), - (('Function/django.contrib.auth.middleware:' - 'AuthenticationMiddleware.process_request'), 1), - (('Function/django.contrib.messages.middleware:' - 'MessageMiddleware.process_request'), 1), - (('Function/django.middleware.csrf:' - 'CsrfViewMiddleware.process_view'), 1), - (('Function/django.contrib.messages.middleware:' - 'MessageMiddleware.process_response'), 1), - (('Function/django.middleware.csrf:' - 'CsrfViewMiddleware.process_response'), 1), - (('Function/django.contrib.sessions.middleware:' - 'SessionMiddleware.process_response'), 1), - (('Function/django.middleware.common:' - 'CommonMiddleware.process_response'), 1), - (('Function/django.middleware.gzip:' - 'GZipMiddleware.process_response'), 1), - (('Function/newrelic.hooks.framework_django:' - 'browser_timing_insertion'), 1), + (("Function/django.middleware.common:" "CommonMiddleware.process_request"), 1), + (("Function/django.contrib.sessions.middleware:" "SessionMiddleware.process_request"), 1), + (("Function/django.contrib.auth.middleware:" "AuthenticationMiddleware.process_request"), 1), + (("Function/django.contrib.messages.middleware:" "MessageMiddleware.process_request"), 1), + (("Function/django.middleware.csrf:" "CsrfViewMiddleware.process_view"), 1), + (("Function/django.contrib.messages.middleware:" "MessageMiddleware.process_response"), 1), + (("Function/django.middleware.csrf:" "CsrfViewMiddleware.process_response"), 1), + (("Function/django.contrib.sessions.middleware:" "SessionMiddleware.process_response"), 1), + (("Function/django.middleware.common:" "CommonMiddleware.process_response"), 1), + (("Function/django.middleware.gzip:" "GZipMiddleware.process_response"), 1), + (("Function/newrelic.hooks.framework_django:" "browser_timing_insertion"), 1), ] _test_django_post_1_10_middleware_scoped_metrics = [ - ('Function/django.middleware.security:SecurityMiddleware', 1), - ('Function/django.contrib.sessions.middleware:SessionMiddleware', 1), - ('Function/django.middleware.common:CommonMiddleware', 1), - ('Function/django.middleware.csrf:CsrfViewMiddleware', 1), - ('Function/django.contrib.auth.middleware:AuthenticationMiddleware', 1), - ('Function/django.contrib.messages.middleware:MessageMiddleware', 1), - ('Function/django.middleware.clickjacking:XFrameOptionsMiddleware', 1), - ('Function/django.middleware.gzip:GZipMiddleware', 1), + ("Function/django.middleware.security:SecurityMiddleware", 1), + ("Function/django.contrib.sessions.middleware:SessionMiddleware", 1), + ("Function/django.middleware.common:CommonMiddleware", 1), + ("Function/django.middleware.csrf:CsrfViewMiddleware", 1), + ("Function/django.contrib.auth.middleware:AuthenticationMiddleware", 1), + ("Function/django.contrib.messages.middleware:MessageMiddleware", 1), + ("Function/django.middleware.clickjacking:XFrameOptionsMiddleware", 1), + ("Function/django.middleware.gzip:GZipMiddleware", 1), ] _test_django_pre_1_10_url_resolver_scoped_metrics = [ - ('Function/django.core.urlresolvers:RegexURLResolver.resolve', 'present'), + ("Function/django.core.urlresolvers:RegexURLResolver.resolve", "present"), ] _test_django_post_1_10_url_resolver_scoped_metrics = [ - ('Function/django.urls.resolvers:RegexURLResolver.resolve', 'present'), + ("Function/django.urls.resolvers:RegexURLResolver.resolve", "present"), ] _test_django_post_2_0_url_resolver_scoped_metrics = [ - ('Function/django.urls.resolvers:URLResolver.resolve', 'present'), + ("Function/django.urls.resolvers:URLResolver.resolve", "present"), ] _test_application_index_scoped_metrics = [ - ('Function/django.core.handlers.wsgi:WSGIHandler.__call__', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/views:index', 1), + ("Function/django.core.handlers.wsgi:WSGIHandler.__call__", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/views:index", 1), ] if DJANGO_VERSION >= (1, 5): - _test_application_index_scoped_metrics.extend([ - ('Function/django.http.response:HttpResponse.close', 1)]) + _test_application_index_scoped_metrics.extend([("Function/django.http.response:HttpResponse.close", 1)]) if DJANGO_VERSION < (1, 10): - _test_application_index_scoped_metrics.extend( - _test_django_pre_1_10_url_resolver_scoped_metrics) + _test_application_index_scoped_metrics.extend(_test_django_pre_1_10_url_resolver_scoped_metrics) elif DJANGO_VERSION >= (2, 0): - _test_application_index_scoped_metrics.extend( - _test_django_post_2_0_url_resolver_scoped_metrics) + _test_application_index_scoped_metrics.extend(_test_django_post_2_0_url_resolver_scoped_metrics) else: - _test_application_index_scoped_metrics.extend( - _test_django_post_1_10_url_resolver_scoped_metrics) - -if DJANGO_SETTINGS_MODULE == 'settings_0110_old': - _test_application_index_scoped_metrics.extend( - _test_django_pre_1_10_middleware_scoped_metrics) -elif DJANGO_SETTINGS_MODULE == 'settings_0110_new': - _test_application_index_scoped_metrics.extend( - _test_django_post_1_10_middleware_scoped_metrics) + _test_application_index_scoped_metrics.extend(_test_django_post_1_10_url_resolver_scoped_metrics) + +if DJANGO_SETTINGS_MODULE == "settings_0110_old": + _test_application_index_scoped_metrics.extend(_test_django_pre_1_10_middleware_scoped_metrics) +elif DJANGO_SETTINGS_MODULE == "settings_0110_new": + _test_application_index_scoped_metrics.extend(_test_django_post_1_10_middleware_scoped_metrics) elif DJANGO_VERSION < (1, 10): - _test_application_index_scoped_metrics.extend( - _test_django_pre_1_10_middleware_scoped_metrics) + _test_application_index_scoped_metrics.extend(_test_django_pre_1_10_middleware_scoped_metrics) @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('views:index', - scoped_metrics=_test_application_index_scoped_metrics) +@validate_transaction_metrics("views:index", scoped_metrics=_test_application_index_scoped_metrics) @validate_code_level_metrics("views", "index") def test_application_index(): test_application = target_application() - response = test_application.get('') - response.mustcontain('INDEX RESPONSE') + response = test_application.get("") + response.mustcontain("INDEX RESPONSE") -@validate_transaction_metrics('views:exception') +@validate_transaction_metrics("views:exception") @validate_code_level_metrics("views", "exception") def test_application_exception(): test_application = target_application() - test_application.get('/exception', status=500) + test_application.get("/exception", status=500) _test_application_not_found_scoped_metrics = [ - ('Function/django.core.handlers.wsgi:WSGIHandler.__call__', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), + ("Function/django.core.handlers.wsgi:WSGIHandler.__call__", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), ] if DJANGO_VERSION >= (1, 5): - _test_application_not_found_scoped_metrics.extend([ - ('Function/django.http.response:HttpResponseNotFound.close', 1)]) + _test_application_not_found_scoped_metrics.extend([("Function/django.http.response:HttpResponseNotFound.close", 1)]) if DJANGO_VERSION < (1, 10): - _test_application_not_found_scoped_metrics.extend( - _test_django_pre_1_10_url_resolver_scoped_metrics) + _test_application_not_found_scoped_metrics.extend(_test_django_pre_1_10_url_resolver_scoped_metrics) elif DJANGO_VERSION >= (2, 0): - _test_application_not_found_scoped_metrics.extend( - _test_django_post_2_0_url_resolver_scoped_metrics) + _test_application_not_found_scoped_metrics.extend(_test_django_post_2_0_url_resolver_scoped_metrics) else: - _test_application_not_found_scoped_metrics.extend( - _test_django_post_1_10_url_resolver_scoped_metrics) + _test_application_not_found_scoped_metrics.extend(_test_django_post_1_10_url_resolver_scoped_metrics) -if DJANGO_SETTINGS_MODULE == 'settings_0110_old': - _test_application_not_found_scoped_metrics.extend( - _test_django_pre_1_10_middleware_scoped_metrics) +if DJANGO_SETTINGS_MODULE == "settings_0110_old": + _test_application_not_found_scoped_metrics.extend(_test_django_pre_1_10_middleware_scoped_metrics) # The `CsrfViewMiddleware.process_view` isn't called for 404 Not Found. _test_application_not_found_scoped_metrics.remove( - ('Function/django.middleware.csrf:CsrfViewMiddleware.process_view', 1)) -elif DJANGO_SETTINGS_MODULE == 'settings_0110_new': - _test_application_not_found_scoped_metrics.extend( - _test_django_post_1_10_middleware_scoped_metrics) + ("Function/django.middleware.csrf:CsrfViewMiddleware.process_view", 1) + ) +elif DJANGO_SETTINGS_MODULE == "settings_0110_new": + _test_application_not_found_scoped_metrics.extend(_test_django_post_1_10_middleware_scoped_metrics) elif DJANGO_VERSION < (1, 10): - _test_application_not_found_scoped_metrics.extend( - _test_django_pre_1_10_middleware_scoped_metrics) + _test_application_not_found_scoped_metrics.extend(_test_django_pre_1_10_middleware_scoped_metrics) # The `CsrfViewMiddleware.process_view` isn't called for 404 Not Found. _test_application_not_found_scoped_metrics.remove( - ('Function/django.middleware.csrf:CsrfViewMiddleware.process_view', 1)) + ("Function/django.middleware.csrf:CsrfViewMiddleware.process_view", 1) + ) @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('django.views.debug:technical_404_response', - scoped_metrics=_test_application_not_found_scoped_metrics) +@validate_transaction_metrics( + "django.views.debug:technical_404_response", scoped_metrics=_test_application_not_found_scoped_metrics +) def test_application_not_found(): test_application = target_application() - test_application.get('/not_found', status=404) + test_application.get("/not_found", status=404) @override_ignore_status_codes([403]) @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('views:permission_denied') +@validate_transaction_metrics("views:permission_denied") @validate_code_level_metrics("views", "permission_denied") def test_ignored_status_code(): test_application = target_application() - test_application.get('/permission_denied', status=403) + test_application.get("/permission_denied", status=403) @override_ignore_status_codes([410]) @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('views:middleware_410') +@validate_transaction_metrics("views:middleware_410") @validate_code_level_metrics("views", "middleware_410") def test_middleware_ignore_status_codes(): test_application = target_application() - test_application.get('/middleware_410', status=410) + test_application.get("/middleware_410", status=410) _test_application_cbv_scoped_metrics = [ - ('Function/django.core.handlers.wsgi:WSGIHandler.__call__', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/views:MyView', 1), - ('Function/views:MyView.get', 1), + ("Function/django.core.handlers.wsgi:WSGIHandler.__call__", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/views:MyView", 1), + ("Function/views:MyView.get", 1), ] if DJANGO_VERSION >= (1, 5): - _test_application_cbv_scoped_metrics.extend([ - ('Function/django.http.response:HttpResponse.close', 1)]) + _test_application_cbv_scoped_metrics.extend([("Function/django.http.response:HttpResponse.close", 1)]) if DJANGO_VERSION < (1, 10): - _test_application_cbv_scoped_metrics.extend( - _test_django_pre_1_10_url_resolver_scoped_metrics) + _test_application_cbv_scoped_metrics.extend(_test_django_pre_1_10_url_resolver_scoped_metrics) elif DJANGO_VERSION >= (2, 0): - _test_application_cbv_scoped_metrics.extend( - _test_django_post_2_0_url_resolver_scoped_metrics) + _test_application_cbv_scoped_metrics.extend(_test_django_post_2_0_url_resolver_scoped_metrics) else: - _test_application_cbv_scoped_metrics.extend( - _test_django_post_1_10_url_resolver_scoped_metrics) - -if DJANGO_SETTINGS_MODULE == 'settings_0110_old': - _test_application_cbv_scoped_metrics.extend( - _test_django_pre_1_10_middleware_scoped_metrics) -elif DJANGO_SETTINGS_MODULE == 'settings_0110_new': - _test_application_cbv_scoped_metrics.extend( - _test_django_post_1_10_middleware_scoped_metrics) + _test_application_cbv_scoped_metrics.extend(_test_django_post_1_10_url_resolver_scoped_metrics) + +if DJANGO_SETTINGS_MODULE == "settings_0110_old": + _test_application_cbv_scoped_metrics.extend(_test_django_pre_1_10_middleware_scoped_metrics) +elif DJANGO_SETTINGS_MODULE == "settings_0110_new": + _test_application_cbv_scoped_metrics.extend(_test_django_post_1_10_middleware_scoped_metrics) elif DJANGO_VERSION < (1, 10): - _test_application_cbv_scoped_metrics.extend( - _test_django_pre_1_10_middleware_scoped_metrics) + _test_application_cbv_scoped_metrics.extend(_test_django_pre_1_10_middleware_scoped_metrics) @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('views:MyView.get', - scoped_metrics=_test_application_cbv_scoped_metrics) +@validate_transaction_metrics("views:MyView.get", scoped_metrics=_test_application_cbv_scoped_metrics) @validate_code_level_metrics("views.MyView", "get") def test_application_cbv(): test_application = target_application() - response = test_application.get('/cbv') - response.mustcontain('CBV RESPONSE') + response = test_application.get("/cbv") + response.mustcontain("CBV RESPONSE") _test_application_deferred_cbv_scoped_metrics = [ - ('Function/django.core.handlers.wsgi:WSGIHandler.__call__', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/views:deferred_cbv', 1), - ('Function/views:MyView.get', 1), + ("Function/django.core.handlers.wsgi:WSGIHandler.__call__", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/views:deferred_cbv", 1), + ("Function/views:MyView.get", 1), ] if DJANGO_VERSION >= (1, 5): - _test_application_deferred_cbv_scoped_metrics.extend([ - ('Function/django.http.response:HttpResponse.close', 1)]) + _test_application_deferred_cbv_scoped_metrics.extend([("Function/django.http.response:HttpResponse.close", 1)]) if DJANGO_VERSION < (1, 10): - _test_application_deferred_cbv_scoped_metrics.extend( - _test_django_pre_1_10_url_resolver_scoped_metrics) + _test_application_deferred_cbv_scoped_metrics.extend(_test_django_pre_1_10_url_resolver_scoped_metrics) elif DJANGO_VERSION >= (2, 0): - _test_application_deferred_cbv_scoped_metrics.extend( - _test_django_post_2_0_url_resolver_scoped_metrics) + _test_application_deferred_cbv_scoped_metrics.extend(_test_django_post_2_0_url_resolver_scoped_metrics) else: - _test_application_deferred_cbv_scoped_metrics.extend( - _test_django_post_1_10_url_resolver_scoped_metrics) - -if DJANGO_SETTINGS_MODULE == 'settings_0110_old': - _test_application_deferred_cbv_scoped_metrics.extend( - _test_django_pre_1_10_middleware_scoped_metrics) -elif DJANGO_SETTINGS_MODULE == 'settings_0110_new': - _test_application_deferred_cbv_scoped_metrics.extend( - _test_django_post_1_10_middleware_scoped_metrics) + _test_application_deferred_cbv_scoped_metrics.extend(_test_django_post_1_10_url_resolver_scoped_metrics) + +if DJANGO_SETTINGS_MODULE == "settings_0110_old": + _test_application_deferred_cbv_scoped_metrics.extend(_test_django_pre_1_10_middleware_scoped_metrics) +elif DJANGO_SETTINGS_MODULE == "settings_0110_new": + _test_application_deferred_cbv_scoped_metrics.extend(_test_django_post_1_10_middleware_scoped_metrics) elif DJANGO_VERSION < (1, 10): - _test_application_deferred_cbv_scoped_metrics.extend( - _test_django_pre_1_10_middleware_scoped_metrics) + _test_application_deferred_cbv_scoped_metrics.extend(_test_django_pre_1_10_middleware_scoped_metrics) @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('views:deferred_cbv', - scoped_metrics=_test_application_deferred_cbv_scoped_metrics) +@validate_transaction_metrics("views:deferred_cbv", scoped_metrics=_test_application_deferred_cbv_scoped_metrics) @validate_code_level_metrics("views", "deferred_cbv") def test_application_deferred_cbv(): test_application = target_application() - response = test_application.get('/deferred_cbv') - response.mustcontain('CBV RESPONSE') + response = test_application.get("/deferred_cbv") + response.mustcontain("CBV RESPONSE") _test_html_insertion_settings = { - 'browser_monitoring.enabled': True, - 'browser_monitoring.auto_instrument': True, - 'js_agent_loader': u'', + "browser_monitoring.enabled": True, + "browser_monitoring.auto_instrument": True, + "js_agent_loader": "", } @override_application_settings(_test_html_insertion_settings) def test_html_insertion_django_middleware(): test_application = target_application() - response = test_application.get('/html_insertion', status=200) + response = test_application.get("/html_insertion", status=200) # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. - response.mustcontain('NREUM HEADER', 'NREUM.info') + response.mustcontain("NREUM HEADER", "NREUM.info") @override_application_settings(_test_html_insertion_settings) @@ -311,23 +281,22 @@ def test_html_insertion_django_gzip_middleware_enabled(): # GZipMiddleware only fires if given the following header. - gzip_header = {'Accept-Encoding': 'gzip'} - response = test_application.get('/gzip_html_insertion', status=200, - headers=gzip_header) + gzip_header = {"Accept-Encoding": "gzip"} + response = test_application.get("/gzip_html_insertion", status=200, headers=gzip_header) # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. # The response.text will already be gunzipped - response.mustcontain('NREUM HEADER', 'NREUM.info') + response.mustcontain("NREUM HEADER", "NREUM.info") _test_html_insertion_settings_disabled = { - 'browser_monitoring.enabled': False, - 'browser_monitoring.auto_instrument': False, - 'js_agent_loader': u'', + "browser_monitoring.enabled": False, + "browser_monitoring.auto_instrument": False, + "js_agent_loader": "", } @@ -337,264 +306,238 @@ def test_html_insertion_django_gzip_middleware_disabled(): # GZipMiddleware only fires if given the following header. - gzip_header = {'Accept-Encoding': 'gzip'} - response = test_application.get('/gzip_html_insertion', status=200, - headers=gzip_header) + gzip_header = {"Accept-Encoding": "gzip"} + response = test_application.get("/gzip_html_insertion", status=200, headers=gzip_header) # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. # The response.text will already be gunzipped - response.mustcontain(no=['NREUM HEADER', 'NREUM.info']) + response.mustcontain(no=["NREUM HEADER", "NREUM.info"]) _test_html_insertion_manual_settings = { - 'browser_monitoring.enabled': True, - 'browser_monitoring.auto_instrument': True, - 'js_agent_loader': u'', + "browser_monitoring.enabled": True, + "browser_monitoring.auto_instrument": True, + "js_agent_loader": "", } @override_application_settings(_test_html_insertion_manual_settings) def test_html_insertion_manual_django_middleware(): test_application = target_application() - response = test_application.get('/html_insertion_manual', status=200) + response = test_application.get("/html_insertion_manual", status=200) # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. - response.mustcontain(no=['NREUM HEADER', 'NREUM.info']) + response.mustcontain(no=["NREUM HEADER", "NREUM.info"]) @override_application_settings(_test_html_insertion_settings) def test_html_insertion_unnamed_attachment_header_django_middleware(): test_application = target_application() - response = test_application.get( - '/html_insertion_unnamed_attachment_header', status=200) + response = test_application.get("/html_insertion_unnamed_attachment_header", status=200) # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. - response.mustcontain(no=['NREUM HEADER', 'NREUM.info']) + response.mustcontain(no=["NREUM HEADER", "NREUM.info"]) @override_application_settings(_test_html_insertion_settings) def test_html_insertion_named_attachment_header_django_middleware(): test_application = target_application() - response = test_application.get( - '/html_insertion_named_attachment_header', status=200) + response = test_application.get("/html_insertion_named_attachment_header", status=200) # The 'NREUM HEADER' value comes from our override for the header. # The 'NREUM.info' value comes from the programmatically generated - # footer added by the agent. + # header added by the agent. - response.mustcontain(no=['NREUM HEADER', 'NREUM.info']) + response.mustcontain(no=["NREUM HEADER", "NREUM.info"]) _test_html_insertion_settings = { - 'browser_monitoring.enabled': True, - 'browser_monitoring.auto_instrument': False, - 'js_agent_loader': u'', + "browser_monitoring.enabled": True, + "browser_monitoring.auto_instrument": False, + "js_agent_loader": "", } @override_application_settings(_test_html_insertion_settings) def test_html_insertion_manual_tag_instrumentation(): test_application = target_application() - response = test_application.get('/template_tags') + response = test_application.get("/template_tags") # Assert that the instrumentation is not inappropriately escaped - response.mustcontain('', - no=['<!-- NREUM HEADER -->']) + response.mustcontain("", no=["<!-- NREUM HEADER -->"]) _test_application_inclusion_tag_scoped_metrics = [ - ('Function/django.core.handlers.wsgi:WSGIHandler.__call__', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/views:inclusion_tag', 1), - ('Template/Render/main.html', 1), + ("Function/django.core.handlers.wsgi:WSGIHandler.__call__", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/views:inclusion_tag", 1), + ("Template/Render/main.html", 1), ] if DJANGO_VERSION < (1, 9): - _test_application_inclusion_tag_scoped_metrics.extend([ - ('Template/Include/results.html', 1)]) + _test_application_inclusion_tag_scoped_metrics.extend([("Template/Include/results.html", 1)]) if DJANGO_VERSION < (1, 10): - _test_application_inclusion_tag_scoped_metrics.extend( - _test_django_pre_1_10_url_resolver_scoped_metrics) + _test_application_inclusion_tag_scoped_metrics.extend(_test_django_pre_1_10_url_resolver_scoped_metrics) elif DJANGO_VERSION >= (2, 0): - _test_application_inclusion_tag_scoped_metrics.extend( - _test_django_post_2_0_url_resolver_scoped_metrics) + _test_application_inclusion_tag_scoped_metrics.extend(_test_django_post_2_0_url_resolver_scoped_metrics) else: - _test_application_inclusion_tag_scoped_metrics.extend( - _test_django_post_1_10_url_resolver_scoped_metrics) - -if DJANGO_SETTINGS_MODULE == 'settings_0110_old': - _test_application_inclusion_tag_scoped_metrics.extend( - _test_django_pre_1_10_middleware_scoped_metrics) -elif DJANGO_SETTINGS_MODULE == 'settings_0110_new': - _test_application_inclusion_tag_scoped_metrics.extend( - _test_django_post_1_10_middleware_scoped_metrics) + _test_application_inclusion_tag_scoped_metrics.extend(_test_django_post_1_10_url_resolver_scoped_metrics) + +if DJANGO_SETTINGS_MODULE == "settings_0110_old": + _test_application_inclusion_tag_scoped_metrics.extend(_test_django_pre_1_10_middleware_scoped_metrics) +elif DJANGO_SETTINGS_MODULE == "settings_0110_new": + _test_application_inclusion_tag_scoped_metrics.extend(_test_django_post_1_10_middleware_scoped_metrics) elif DJANGO_VERSION < (1, 10): - _test_application_inclusion_tag_scoped_metrics.extend( - _test_django_pre_1_10_middleware_scoped_metrics) + _test_application_inclusion_tag_scoped_metrics.extend(_test_django_pre_1_10_middleware_scoped_metrics) try: _test_application_inclusion_tag_scoped_metrics.remove( - (('Function/newrelic.hooks.framework_django:' - 'browser_timing_insertion'), 1) + (("Function/newrelic.hooks.framework_django:" "browser_timing_insertion"), 1) ) except ValueError: pass @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('views:inclusion_tag', - scoped_metrics=_test_application_inclusion_tag_scoped_metrics) +@validate_transaction_metrics("views:inclusion_tag", scoped_metrics=_test_application_inclusion_tag_scoped_metrics) @validate_code_level_metrics("views", "inclusion_tag") def test_application_inclusion_tag(): test_application = target_application() - response = test_application.get('/inclusion_tag') - response.mustcontain('Inclusion tag') + response = test_application.get("/inclusion_tag") + response.mustcontain("Inclusion tag") _test_inclusion_tag_template_tags_scoped_metrics = [ - ('Function/django.core.handlers.wsgi:WSGIHandler.__call__', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/views:inclusion_tag', 1), - ('Template/Render/main.html', 1), + ("Function/django.core.handlers.wsgi:WSGIHandler.__call__", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/views:inclusion_tag", 1), + ("Template/Render/main.html", 1), ] if DJANGO_VERSION < (1, 9): - _test_inclusion_tag_template_tags_scoped_metrics.extend([ - ('Template/Include/results.html', 1), - ('Template/Tag/show_results', 1)]) + _test_inclusion_tag_template_tags_scoped_metrics.extend( + [("Template/Include/results.html", 1), ("Template/Tag/show_results", 1)] + ) -_test_inclusion_tag_settings = { - 'instrumentation.templates.inclusion_tag': '*' -} +_test_inclusion_tag_settings = {"instrumentation.templates.inclusion_tag": "*"} if DJANGO_VERSION < (1, 10): - _test_inclusion_tag_template_tags_scoped_metrics.extend( - _test_django_pre_1_10_url_resolver_scoped_metrics) + _test_inclusion_tag_template_tags_scoped_metrics.extend(_test_django_pre_1_10_url_resolver_scoped_metrics) elif DJANGO_VERSION >= (2, 0): - _test_inclusion_tag_template_tags_scoped_metrics.extend( - _test_django_post_2_0_url_resolver_scoped_metrics) + _test_inclusion_tag_template_tags_scoped_metrics.extend(_test_django_post_2_0_url_resolver_scoped_metrics) else: - _test_inclusion_tag_template_tags_scoped_metrics.extend( - _test_django_post_1_10_url_resolver_scoped_metrics) + _test_inclusion_tag_template_tags_scoped_metrics.extend(_test_django_post_1_10_url_resolver_scoped_metrics) -if DJANGO_SETTINGS_MODULE == 'settings_0110_old': - _test_inclusion_tag_template_tags_scoped_metrics.extend( - _test_django_pre_1_10_middleware_scoped_metrics) -elif DJANGO_SETTINGS_MODULE == 'settings_0110_new': - _test_inclusion_tag_template_tags_scoped_metrics.extend( - _test_django_post_1_10_middleware_scoped_metrics) +if DJANGO_SETTINGS_MODULE == "settings_0110_old": + _test_inclusion_tag_template_tags_scoped_metrics.extend(_test_django_pre_1_10_middleware_scoped_metrics) +elif DJANGO_SETTINGS_MODULE == "settings_0110_new": + _test_inclusion_tag_template_tags_scoped_metrics.extend(_test_django_post_1_10_middleware_scoped_metrics) elif DJANGO_VERSION < (1, 10): - _test_inclusion_tag_template_tags_scoped_metrics.extend( - _test_django_pre_1_10_middleware_scoped_metrics) + _test_inclusion_tag_template_tags_scoped_metrics.extend(_test_django_pre_1_10_middleware_scoped_metrics) try: _test_inclusion_tag_template_tags_scoped_metrics.remove( - (('Function/newrelic.hooks.framework_django:' - 'browser_timing_insertion'), 1) + (("Function/newrelic.hooks.framework_django:" "browser_timing_insertion"), 1) ) except ValueError: pass @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('views:inclusion_tag', - scoped_metrics=_test_inclusion_tag_template_tags_scoped_metrics) +@validate_transaction_metrics("views:inclusion_tag", scoped_metrics=_test_inclusion_tag_template_tags_scoped_metrics) @override_generic_settings(django_settings, _test_inclusion_tag_settings) @validate_code_level_metrics("views", "inclusion_tag") def test_inclusion_tag_template_tag_metric(): test_application = target_application() - response = test_application.get('/inclusion_tag') - response.mustcontain('Inclusion tag') + response = test_application.get("/inclusion_tag") + response.mustcontain("Inclusion tag") _test_template_render_exception_scoped_metrics_base = [ - ('Function/django.core.handlers.wsgi:WSGIHandler.__call__', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), + ("Function/django.core.handlers.wsgi:WSGIHandler.__call__", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), ] if DJANGO_VERSION < (1, 5): _test_template_render_exception_scoped_metrics_base.append( - ('Function/django.http:HttpResponseServerError.close', 1)) + ("Function/django.http:HttpResponseServerError.close", 1) + ) elif DJANGO_VERSION < (1, 8): _test_template_render_exception_scoped_metrics_base.append( - ('Function/django.http.response:HttpResponseServerError.close', 1)) + ("Function/django.http.response:HttpResponseServerError.close", 1) + ) else: - _test_template_render_exception_scoped_metrics_base.append( - ('Function/django.http.response:HttpResponse.close', 1)) + _test_template_render_exception_scoped_metrics_base.append(("Function/django.http.response:HttpResponse.close", 1)) if DJANGO_VERSION < (1, 10): - _test_template_render_exception_scoped_metrics_base.extend( - _test_django_pre_1_10_url_resolver_scoped_metrics) + _test_template_render_exception_scoped_metrics_base.extend(_test_django_pre_1_10_url_resolver_scoped_metrics) elif DJANGO_VERSION >= (2, 0): - _test_template_render_exception_scoped_metrics_base.extend( - _test_django_post_2_0_url_resolver_scoped_metrics) + _test_template_render_exception_scoped_metrics_base.extend(_test_django_post_2_0_url_resolver_scoped_metrics) else: - _test_template_render_exception_scoped_metrics_base.extend( - _test_django_post_1_10_url_resolver_scoped_metrics) - -if DJANGO_SETTINGS_MODULE == 'settings_0110_old': - _test_template_render_exception_scoped_metrics_base.extend( - _test_django_pre_1_10_middleware_scoped_metrics) -elif DJANGO_SETTINGS_MODULE == 'settings_0110_new': - _test_template_render_exception_scoped_metrics_base.extend( - _test_django_post_1_10_middleware_scoped_metrics) + _test_template_render_exception_scoped_metrics_base.extend(_test_django_post_1_10_url_resolver_scoped_metrics) + +if DJANGO_SETTINGS_MODULE == "settings_0110_old": + _test_template_render_exception_scoped_metrics_base.extend(_test_django_pre_1_10_middleware_scoped_metrics) +elif DJANGO_SETTINGS_MODULE == "settings_0110_new": + _test_template_render_exception_scoped_metrics_base.extend(_test_django_post_1_10_middleware_scoped_metrics) elif DJANGO_VERSION < (1, 10): - _test_template_render_exception_scoped_metrics_base.extend( - _test_django_pre_1_10_middleware_scoped_metrics) + _test_template_render_exception_scoped_metrics_base.extend(_test_django_pre_1_10_middleware_scoped_metrics) if DJANGO_VERSION < (1, 9): - _test_template_render_exception_errors = [ - 'django.template.base:TemplateSyntaxError'] + _test_template_render_exception_errors = ["django.template.base:TemplateSyntaxError"] else: - _test_template_render_exception_errors = [ - 'django.template.exceptions:TemplateSyntaxError'] + _test_template_render_exception_errors = ["django.template.exceptions:TemplateSyntaxError"] -_test_template_render_exception_function_scoped_metrics = list( - _test_template_render_exception_scoped_metrics_base) -_test_template_render_exception_function_scoped_metrics.extend([ - ('Function/views:render_exception_function', 1), -]) +_test_template_render_exception_function_scoped_metrics = list(_test_template_render_exception_scoped_metrics_base) +_test_template_render_exception_function_scoped_metrics.extend( + [ + ("Function/views:render_exception_function", 1), + ] +) @validate_transaction_errors(errors=_test_template_render_exception_errors) -@validate_transaction_metrics('views:render_exception_function', - scoped_metrics=_test_template_render_exception_function_scoped_metrics) +@validate_transaction_metrics( + "views:render_exception_function", scoped_metrics=_test_template_render_exception_function_scoped_metrics +) @validate_code_level_metrics("views", "render_exception_function") def test_template_render_exception_function(): test_application = target_application() - test_application.get('/render_exception_function', status=500) + test_application.get("/render_exception_function", status=500) -_test_template_render_exception_class_scoped_metrics = list( - _test_template_render_exception_scoped_metrics_base) -_test_template_render_exception_class_scoped_metrics.extend([ - ('Function/views:RenderExceptionClass', 1), - ('Function/views:RenderExceptionClass.get', 1), -]) +_test_template_render_exception_class_scoped_metrics = list(_test_template_render_exception_scoped_metrics_base) +_test_template_render_exception_class_scoped_metrics.extend( + [ + ("Function/views:RenderExceptionClass", 1), + ("Function/views:RenderExceptionClass.get", 1), + ] +) @validate_transaction_errors(errors=_test_template_render_exception_errors) -@validate_transaction_metrics('views:RenderExceptionClass.get', - scoped_metrics=_test_template_render_exception_class_scoped_metrics) +@validate_transaction_metrics( + "views:RenderExceptionClass.get", scoped_metrics=_test_template_render_exception_class_scoped_metrics +) @validate_code_level_metrics("views.RenderExceptionClass", "get") def test_template_render_exception_class(): test_application = target_application() - test_application.get('/render_exception_class', status=500) + test_application.get("/render_exception_class", status=500) diff --git a/tests/framework_django/test_asgi_application.py b/tests/framework_django/test_asgi_application.py index 4570427662..39e99db2f0 100644 --- a/tests/framework_django/test_asgi_application.py +++ b/tests/framework_django/test_asgi_application.py @@ -45,7 +45,7 @@ ] rollup_metrics = scoped_metrics + [ - ('Python/Framework/Django/%s' % django.get_version(), 1), + (f'Python/Framework/Django/{django.get_version()}', 1), ] @@ -107,7 +107,7 @@ def test_asgi_class_based_view(application, url, view_name): @validate_transaction_errors(errors=[]) @validate_transaction_metrics(view_name, - scoped_metrics=[('Function/' + view_name, 1)] + scoped_metrics, + scoped_metrics=[(f"Function/{view_name}", 1)] + scoped_metrics, rollup_metrics=rollup_metrics) @validate_code_level_metrics(namespace, func) def _test(): diff --git a/tests/framework_django/views.py b/tests/framework_django/views.py index c5ce1526c7..9a5b33e1d8 100644 --- a/tests/framework_django/views.py +++ b/tests/framework_django/views.py @@ -12,22 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. +from django.core.exceptions import PermissionDenied from django.http import HttpResponse -from django.views.generic.base import View, TemplateView from django.shortcuts import render -from django.core.exceptions import PermissionDenied +from django.views.generic.base import TemplateView, View from middleware import Custom410 -from newrelic.api.transaction import (get_browser_timing_header, - get_browser_timing_footer) +from newrelic.api.transaction import get_browser_timing_header def index(request): - return HttpResponse('INDEX RESPONSE') + return HttpResponse("INDEX RESPONSE") def exception(request): - raise RuntimeError('exception') + raise RuntimeError("exception") def permission_denied(request): @@ -40,7 +39,7 @@ def middleware_410(request): class MyView(View): def get(self, request): - return HttpResponse('CBV RESPONSE') + return HttpResponse("CBV RESPONSE") def deferred_cbv(request): @@ -48,69 +47,76 @@ def deferred_cbv(request): def html_insertion(request): - return HttpResponse('Some header' - '

My First Heading

My first paragraph.

' - '') + return HttpResponse( + "Some header" + "

My First Heading

My first paragraph.

" + "" + ) def html_insertion_content_length(request): - content = ('Some header' - '

My First Heading

My first paragraph.

' - '') + content = ( + "Some header" + "

My First Heading

My first paragraph.

" + "" + ) response = HttpResponse(content) - response['Content-Length'] = len(content) + response["Content-Length"] = len(content) return response def html_insertion_manual(request): header = get_browser_timing_header() - footer = get_browser_timing_footer() - header = get_browser_timing_header() - footer = get_browser_timing_footer() - assert header == '' - assert footer == '' + assert header == "" - return HttpResponse('Some header' - '

My First Heading

My first paragraph.

' - '') + return HttpResponse( + "Some header" + "

My First Heading

My first paragraph.

" + "" + ) def html_insertion_unnamed_attachment_header(request): - response = HttpResponse('Some header' - '

My First Heading

My first paragraph.

' - '') - response['Content-Disposition'] = 'attachment' + response = HttpResponse( + "Some header" + "

My First Heading

My first paragraph.

" + "" + ) + response["Content-Disposition"] = "attachment" return response def html_insertion_named_attachment_header(request): - response = HttpResponse('Some header' - '

My First Heading

My first paragraph.

' - '') - response['Content-Disposition'] = 'Attachment; filename="X"' + response = HttpResponse( + "Some header" + "

My First Heading

My first paragraph.

" + "" + ) + response["Content-Disposition"] = 'Attachment; filename="X"' return response def inclusion_tag(request): - return render(request, 'main.html', {}, content_type="text/html") + return render(request, "main.html", {}, content_type="text/html") def template_tags(request): - return render(request, 'main.html', {}, content_type="text/html") + return render(request, "main.html", {}, content_type="text/html") def render_exception_function(request): - return render(request, 'render_exception.html') + return render(request, "render_exception.html") class RenderExceptionClass(TemplateView): - template_name = 'render_exception.html' + template_name = "render_exception.html" def gzip_html_insertion(request): # contents must be at least 200 bytes for gzip middleware to work - contents = '*' * 200 - return HttpResponse('Some header' - '

My First Heading

%s

' % contents) + contents = "*" * 200 + return HttpResponse( + f"Some header

My First Heading

{contents}

" + ) diff --git a/tests/framework_falcon/_target_application.py b/tests/framework_falcon/_target_application.py index e78e5bb633..258bc82af7 100644 --- a/tests/framework_falcon/_target_application.py +++ b/tests/framework_falcon/_target_application.py @@ -39,14 +39,14 @@ class BadPutRequest(ValueError): pass -class Index(object): +class Index(): def on_get(self, req, resp): """Handles GET requests""" resp.content_type = 'application/json' resp.data = b'{"status": "ok"}' -class BadResponse(object): +class BadResponse(): def on_get(self, req, resp): raise BadGetRequest() diff --git a/tests/framework_falcon/conftest.py b/tests/framework_falcon/conftest.py index fd43715c6e..8b781fed07 100644 --- a/tests/framework_falcon/conftest.py +++ b/tests/framework_falcon/conftest.py @@ -13,24 +13,27 @@ # limitations under the License. import pytest - -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 - +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (framework_falcon)', - default_settings=_default_settings) + app_name="Python Agent Test (framework_falcon)", default_settings=_default_settings +) @pytest.fixture() def app(): from _target_application import _target_application + return _target_application diff --git a/tests/framework_falcon/test_application.py b/tests/framework_falcon/test_application.py index 6b64c8c678..89175cee1d 100644 --- a/tests/framework_falcon/test_application.py +++ b/tests/framework_falcon/test_application.py @@ -26,7 +26,7 @@ def test_basic(app): _test_basic_metrics = ( - ('Function/' + app.name_prefix + '.__call__', 1), + (f"Function/{app.name_prefix}.__call__", 1), ('Function/_target_application:Index.on_get', 1), ) @@ -45,7 +45,7 @@ def _test(): @validate_transaction_errors(errors=[]) def test_ignored_status_code(app): - @validate_transaction_metrics(app.name_prefix + '._handle_exception') + @validate_transaction_metrics(f"{app.name_prefix}._handle_exception") def _test(): app.get('/foobar', status=404) @@ -56,7 +56,7 @@ def _test(): def test_error_recorded(app): @validate_transaction_errors(errors=[app.not_found_error]) - @validate_transaction_metrics(app.name_prefix + '._handle_exception') + @validate_transaction_metrics(f"{app.name_prefix}._handle_exception") def _test(): app.get('/foobar', status=404) diff --git a/tests/framework_fastapi/conftest.py b/tests/framework_fastapi/conftest.py index d65398ffb9..e53e597061 100644 --- a/tests/framework_fastapi/conftest.py +++ b/tests/framework_fastapi/conftest.py @@ -13,13 +13,16 @@ # limitations under the License. import pytest -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 newrelic_caplog as caplog, ) - _default_settings = { + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. "transaction_tracer.explain_threshold": 0.0, "transaction_tracer.transaction_threshold": 0.0, "transaction_tracer.stack_trace_threshold": 0.0, diff --git a/tests/framework_fastapi/test_application.py b/tests/framework_fastapi/test_application.py index 85d230e26b..135a7268f3 100644 --- a/tests/framework_fastapi/test_application.py +++ b/tests/framework_fastapi/test_application.py @@ -29,7 +29,7 @@ def test_application(caplog, app, endpoint, transaction_name): caplog.set_level(logging.ERROR) - @validate_transaction_metrics(transaction_name, scoped_metrics=[("Function/" + transaction_name, 1)]) + @validate_transaction_metrics(transaction_name, scoped_metrics=[(f"Function/{transaction_name}", 1)]) @validate_code_level_metrics(*transaction_name.split(":")) def _test(): response = app.get(endpoint) diff --git a/tests/framework_flask/_test_blueprints.py b/tests/framework_flask/_test_blueprints.py index 11236eb33c..e811f6cf99 100644 --- a/tests/framework_flask/_test_blueprints.py +++ b/tests/framework_flask/_test_blueprints.py @@ -13,70 +13,82 @@ # limitations under the License. import webtest - -from flask import Flask -from flask import Blueprint -from werkzeug.routing import Rule - from conftest import is_flask_v2 as nested_blueprint_support +from conftest import is_not_flask_v2_3 as before_app_first_request_support +from flask import Blueprint, Flask +from werkzeug.routing import Rule # Blueprints are only available in 0.7.0 onwards. -blueprint = Blueprint('blueprint', __name__) +blueprint = Blueprint("blueprint", __name__) application = Flask(__name__) -@blueprint.route('/index') + +@blueprint.route("/index") def index_page(): - return 'BLUEPRINT INDEX RESPONSE' + return "BLUEPRINT INDEX RESPONSE" -@blueprint.endpoint('endpoint') + +@blueprint.endpoint("endpoint") def endpoint_page(): - return 'BLUEPRINT ENDPOINT RESPONSE' + return "BLUEPRINT ENDPOINT RESPONSE" + + +# < Flask v2.3.0 +if before_app_first_request_support: + + @blueprint.before_app_first_request + def before_app_first_request(): + pass -@blueprint.before_app_first_request -def before_app_first_request(): - pass @blueprint.before_request def before_request(): pass + @blueprint.before_app_request def before_app_request(): pass + @blueprint.after_request def after_request(response): return response + @blueprint.after_app_request def after_app_request(response): return response + @blueprint.teardown_request def teardown_request(exc): pass + @blueprint.teardown_app_request def teardown_app_request(exc): pass + # Support for nested blueprints was added in Flask 2.0 if nested_blueprint_support: - parent = Blueprint('parent', __name__, url_prefix='/parent') - child = Blueprint('child', __name__, url_prefix='/child') + parent = Blueprint("parent", __name__, url_prefix="/parent") + child = Blueprint("child", __name__, url_prefix="/child") parent.register_blueprint(child) - @child.route('/nested') + @child.route("/nested") def nested_page(): - return 'PARENT NESTED RESPONSE' + return "PARENT NESTED RESPONSE" + application.register_blueprint(parent) application.register_blueprint(blueprint) -application.url_map.add(Rule('/endpoint', endpoint='endpoint')) +application.url_map.add(Rule("/endpoint", endpoint="endpoint")) _test_application = webtest.TestApp(application) diff --git a/tests/framework_flask/_test_compress.py b/tests/framework_flask/_test_compress.py index f3c9fbf2be..e0c90d56a5 100644 --- a/tests/framework_flask/_test_compress.py +++ b/tests/framework_flask/_test_compress.py @@ -12,20 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -try: - from io import BytesIO as IO -except ImportError: - import StringIO as IO +from io import BytesIO import webtest - -from flask import Flask -from flask import Response -from flask import send_file +from flask import Flask, Response, send_file from flask_compress import Compress -from newrelic.api.transaction import (get_browser_timing_header, - get_browser_timing_footer) +from newrelic.api.transaction import get_browser_timing_header application = Flask(__name__) @@ -33,59 +26,59 @@ compress.init_app(application) -@application.route('/compress') +@application.route("/compress") def index_page(): - return '' + 500 * 'X' + '' + return f"{500 * 'X'}" -@application.route('/html_insertion') +@application.route("/html_insertion") def html_insertion(): - return ('Some header' - '

My First Heading

My first paragraph.

' - '') + return ( + "Some header" + "

My First Heading

My first paragraph.

" + "" + ) -@application.route('/html_insertion_manual') +@application.route("/html_insertion_manual") def html_insertion_manual(): header = get_browser_timing_header() - footer = get_browser_timing_footer() - header = get_browser_timing_header() - footer = get_browser_timing_footer() - assert header == '' - assert footer == '' + assert header == "" - return ('Some header' - '

My First Heading

My first paragraph.

' - '') + return ( + "Some header" + "

My First Heading

My first paragraph.

" + "" + ) -@application.route('/html_insertion_unnamed_attachment_header') +@application.route("/html_insertion_unnamed_attachment_header") def html_insertion_unnamed_attachment_header(): response = Response( - response='Some header' - '

My First Heading

My first paragraph.

' - '') - response.headers.add('Content-Disposition', - 'attachment') + response="Some header" + "

My First Heading

My first paragraph.

" + "" + ) + response.headers.add("Content-Disposition", "attachment") return response -@application.route('/html_insertion_named_attachment_header') +@application.route("/html_insertion_named_attachment_header") def html_insertion_named_attachment_header(): response = Response( - response='Some header' - '

My First Heading

My first paragraph.

' - '') - response.headers.add('Content-Disposition', - 'attachment; filename="X"') + response="Some header" + "

My First Heading

My first paragraph.

" + "" + ) + response.headers.add("Content-Disposition", 'attachment; filename="X"') return response -@application.route('/html_served_from_file') +@application.route("/html_served_from_file") def html_served_from_file(): - file = IO() + file = BytesIO() contents = b""" Some header

My First Heading

My first paragraph.

@@ -93,12 +86,12 @@ def html_served_from_file(): """ file.write(contents) file.seek(0) - return send_file(file, mimetype='text/html') + return send_file(file, mimetype="text/html") -@application.route('/text_served_from_file') +@application.route("/text_served_from_file") def text_served_from_file(): - file = IO() + file = BytesIO() contents = b""" Some header

My First Heading

My first paragraph.

@@ -106,17 +99,19 @@ def text_served_from_file(): """ file.write(contents) file.seek(0) - return send_file(file, mimetype='text/plain') + return send_file(file, mimetype="text/plain") _test_application = webtest.TestApp(application) -@application.route('/empty_content_type') +@application.route("/empty_content_type") def empty_content_type(): response = Response( - response='Some header' - '

My First Heading

My first paragraph.

' - '', mimetype='') + response="Some header" + "

My First Heading

My first paragraph.

" + "", + mimetype="", + ) assert response.mimetype is None return response diff --git a/tests/framework_flask/_test_middleware.py b/tests/framework_flask/_test_middleware.py index 41abb8b8cc..e477315c36 100644 --- a/tests/framework_flask/_test_middleware.py +++ b/tests/framework_flask/_test_middleware.py @@ -13,33 +13,48 @@ # limitations under the License. import webtest - +from conftest import is_not_flask_v2_3 as before_first_request_support from flask import Flask application = Flask(__name__) -@application.before_first_request -def before_first_request(): - pass +# < Flask v2.3.0 +if before_first_request_support: + + @application.before_first_request + def before_first_request(): + pass + + +# >= Flask v2.3.0 +def application_app_context(): + with application.app_context(): + pass + @application.before_request def before_request(): pass + @application.after_request def after_request(response): return response + @application.teardown_request def teardown_request(exc): pass + @application.teardown_appcontext def teardown_appcontext(exc): pass -@application.route('/middleware') + +@application.route("/middleware") def index_page(): - return 'INDEX RESPONSE' + return "INDEX RESPONSE" + _test_application = webtest.TestApp(application) diff --git a/tests/framework_flask/conftest.py b/tests/framework_flask/conftest.py index f90ed9b512..c52a11de9f 100644 --- a/tests/framework_flask/conftest.py +++ b/tests/framework_flask/conftest.py @@ -15,29 +15,48 @@ import platform import pytest -from flask import __version__ as flask_version +from flask import ( + __version__ as flask_version, # required for python 3.7 in lieu of get_package_version_tuple +) -from testing_support.fixtures import collector_agent_registration_fixture, collector_available_fixture # noqa: F401; pylint: disable=W0611 +from newrelic.common.package_version_utils import get_package_version_tuple +try: + FLASK_VERSION = tuple(int(v) for v in flask_version.split(".")) +except: + # This does not work for Python 3.7 for v2.2.5 + # This only works for flaskmaster + FLASK_VERSION = get_package_version_tuple("flask") + +from testing_support.fixtures import ( # noqa: F401; pylint: disable=W0611 + collector_agent_registration_fixture, + collector_available_fixture, +) _default_settings = { - 'transaction_tracer.explain_threshold': 0.0, - 'transaction_tracer.transaction_threshold': 0.0, - 'transaction_tracer.stack_trace_threshold': 0.0, - 'debug.log_data_collector_payloads': True, - 'debug.record_transaction_failure': True, - 'debug.log_autorum_middleware': True, + "package_reporting.enabled": False, # Turn off package reporting for testing as it causes slow downs. + "transaction_tracer.explain_threshold": 0.0, + "transaction_tracer.transaction_threshold": 0.0, + "transaction_tracer.stack_trace_threshold": 0.0, + "debug.log_data_collector_payloads": True, + "debug.record_transaction_failure": True, + "debug.log_autorum_middleware": True, + "security.agent.enabled": True, + "security.enabled": True, + "security.mode": "IAST", + "security.validator_service_url": "wss://csec-staging.nr-data.net" } collector_agent_registration = collector_agent_registration_fixture( - app_name='Python Agent Test (framework_flask)', - default_settings=_default_settings) + app_name="Python Agent Test (framework_flask)", default_settings=_default_settings +) -is_flask_v2 = int(flask_version.split('.')[0]) >= 2 +is_flask_v2 = FLASK_VERSION[0] >= 2 +is_not_flask_v2_3 = FLASK_VERSION < (2, 3, 0) is_pypy = platform.python_implementation() == "PyPy" async_handler_support = is_flask_v2 and not is_pypy skip_if_not_async_handler_support = pytest.mark.skipif( not async_handler_support, - reason="Requires async handler support. (Flask >=v2.0.0)", -) \ No newline at end of file + reason="Requires async handler support. (Flask >=v2.0.0, CPython)", +) diff --git a/tests/framework_flask/test_application.py b/tests/framework_flask/test_application.py index de7a430191..b1250828dd 100644 --- a/tests/framework_flask/test_application.py +++ b/tests/framework_flask/test_application.py @@ -13,23 +13,28 @@ # limitations under the License. import pytest - +from conftest import async_handler_support, skip_if_not_async_handler_support from testing_support.fixtures import ( override_application_settings, - validate_tt_parenting) -from testing_support.validators.validate_code_level_metrics import validate_code_level_metrics -from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics -from testing_support.validators.validate_transaction_errors import validate_transaction_errors - -from newrelic.packages import six + validate_tt_parenting, +) +from testing_support.validators.validate_code_level_metrics import ( + validate_code_level_metrics, +) +from testing_support.validators.validate_transaction_errors import ( + validate_transaction_errors, +) +from testing_support.validators.validate_transaction_metrics import ( + validate_transaction_metrics, +) -from conftest import async_handler_support, skip_if_not_async_handler_support try: # The __version__ attribute was only added in 0.7.0. # Flask team does not use semantic versioning during development. from flask import __version__ as flask_version - flask_version = tuple([int(v) for v in flask_version.split('.')]) + + flask_version = tuple([int(v) for v in flask_version.split(".")]) is_gt_flask060 = True is_dev_version = False except ValueError: @@ -39,8 +44,7 @@ is_gt_flask060 = False is_dev_version = False -requires_endpoint_decorator = pytest.mark.skipif(not is_gt_flask060, - reason="The endpoint decorator is not supported.") +requires_endpoint_decorator = pytest.mark.skipif(not is_gt_flask060, reason="The endpoint decorator is not supported.") def target_application(): @@ -48,10 +52,7 @@ def target_application(): # issues whereby the agent needs to be initialised before Flask is # imported and the routes configured. Normally pytest only runs the # global fixture which will initialise the agent after each test - # file is imported, which is too late. We also can't do application - # creation within a function as we will then get view handler - # functions are different between Python 2 and 3, with the latter - # showing scope in path. + # file is imported, which is too late. if not async_handler_support: from _test_application import _test_application @@ -61,226 +62,248 @@ def target_application(): _test_application_index_scoped_metrics = [ - ('Function/flask.app:Flask.wsgi_app', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/_test_application:index_page', 1), - ('Function/werkzeug.wsgi:ClosingIterator.close', 1)] + ("Function/flask.app:Flask.wsgi_app", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/_test_application:index_page", 1), + ("Function/werkzeug.wsgi:ClosingIterator.close", 1), +] _test_application_index_tt_parenting = ( - 'TransactionNode', [ - ('FunctionNode', [ - ('FunctionNode', [ - ('FunctionNode', []), - ('FunctionNode', []), - ('FunctionNode', []), - # some flask versions have more FunctionNodes here, as appended - # below - ]), - ]), - ('FunctionNode', []), - ('FunctionNode', [ - ('FunctionNode', []), - ]), - ] + "TransactionNode", + [ + ( + "FunctionNode", + [ + ( + "FunctionNode", + [ + ("FunctionNode", []), + ("FunctionNode", []), + ("FunctionNode", []), + # some flask versions have more FunctionNodes here, as appended + # below + ], + ), + ], + ), + ("FunctionNode", []), + ( + "FunctionNode", + [ + ("FunctionNode", []), + ], + ), + ], ) if is_dev_version or (is_gt_flask060 and flask_version >= (0, 7)): _test_application_index_tt_parenting[1][0][1][0][1].append( - ('FunctionNode', []), + ("FunctionNode", []), ) if is_dev_version or (is_gt_flask060 and flask_version >= (0, 9)): _test_application_index_tt_parenting[1][0][1][0][1].append( - ('FunctionNode', []), + ("FunctionNode", []), ) + @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('_test_application:index_page', - scoped_metrics=_test_application_index_scoped_metrics) +@validate_transaction_metrics("_test_application:index_page", scoped_metrics=_test_application_index_scoped_metrics) @validate_tt_parenting(_test_application_index_tt_parenting) @validate_code_level_metrics("_test_application", "index_page") def test_application_index(): application = target_application() - response = application.get('/index') - response.mustcontain('INDEX RESPONSE') + response = application.get("/index") + response.mustcontain("INDEX RESPONSE") + _test_application_async_scoped_metrics = [ - ('Function/flask.app:Flask.wsgi_app', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/_test_application_async:async_page', 1), - ('Function/werkzeug.wsgi:ClosingIterator.close', 1)] + ("Function/flask.app:Flask.wsgi_app", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/_test_application_async:async_page", 1), + ("Function/werkzeug.wsgi:ClosingIterator.close", 1), +] + @skip_if_not_async_handler_support @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('_test_application_async:async_page', - scoped_metrics=_test_application_async_scoped_metrics) +@validate_transaction_metrics( + "_test_application_async:async_page", scoped_metrics=_test_application_async_scoped_metrics +) @validate_tt_parenting(_test_application_index_tt_parenting) @validate_code_level_metrics("_test_application_async", "async_page") def test_application_async(): application = target_application() - response = application.get('/async') - response.mustcontain('ASYNC RESPONSE') + response = application.get("/async") + response.mustcontain("ASYNC RESPONSE") + _test_application_endpoint_scoped_metrics = [ - ('Function/flask.app:Flask.wsgi_app', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/_test_application:endpoint_page', 1), - ('Function/werkzeug.wsgi:ClosingIterator.close', 1)] + ("Function/flask.app:Flask.wsgi_app", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/_test_application:endpoint_page", 1), + ("Function/werkzeug.wsgi:ClosingIterator.close", 1), +] @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('_test_application:endpoint_page', - scoped_metrics=_test_application_endpoint_scoped_metrics) +@validate_transaction_metrics( + "_test_application:endpoint_page", scoped_metrics=_test_application_endpoint_scoped_metrics +) @validate_code_level_metrics("_test_application", "endpoint_page") def test_application_endpoint(): application = target_application() - response = application.get('/endpoint') - response.mustcontain('ENDPOINT RESPONSE') + response = application.get("/endpoint") + response.mustcontain("ENDPOINT RESPONSE") _test_application_error_scoped_metrics = [ - ('Function/flask.app:Flask.wsgi_app', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/_test_application:error_page', 1), - ('Function/flask.app:Flask.handle_exception', 1), - ('Function/werkzeug.wsgi:ClosingIterator.close', 1), - ('Function/flask.app:Flask.handle_user_exception', 1), - ('Function/flask.app:Flask.handle_user_exception', 1)] - - -if six.PY3: - _test_application_error_errors = ['builtins:RuntimeError'] -else: - _test_application_error_errors = ['exceptions:RuntimeError'] - - -@validate_transaction_errors(errors=_test_application_error_errors) -@validate_transaction_metrics('_test_application:error_page', - scoped_metrics=_test_application_error_scoped_metrics) + ("Function/flask.app:Flask.wsgi_app", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/_test_application:error_page", 1), + ("Function/flask.app:Flask.handle_exception", 1), + ("Function/werkzeug.wsgi:ClosingIterator.close", 1), + ("Function/flask.app:Flask.handle_user_exception", 1), + ("Function/flask.app:Flask.handle_user_exception", 1), +] + + +@validate_transaction_errors(errors=["builtins:RuntimeError"]) +@validate_transaction_metrics("_test_application:error_page", scoped_metrics=_test_application_error_scoped_metrics) @validate_code_level_metrics("_test_application", "error_page") def test_application_error(): application = target_application() - application.get('/error', status=500, expect_errors=True) + application.get("/error", status=500, expect_errors=True) _test_application_abort_404_scoped_metrics = [ - ('Function/flask.app:Flask.wsgi_app', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/_test_application:abort_404_page', 1), - ('Function/flask.app:Flask.handle_http_exception', 1), - ('Function/werkzeug.wsgi:ClosingIterator.close', 1), - ('Function/flask.app:Flask.handle_user_exception', 1)] + ("Function/flask.app:Flask.wsgi_app", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/_test_application:abort_404_page", 1), + ("Function/flask.app:Flask.handle_http_exception", 1), + ("Function/werkzeug.wsgi:ClosingIterator.close", 1), + ("Function/flask.app:Flask.handle_user_exception", 1), +] @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('_test_application:abort_404_page', - scoped_metrics=_test_application_abort_404_scoped_metrics) +@validate_transaction_metrics( + "_test_application:abort_404_page", scoped_metrics=_test_application_abort_404_scoped_metrics +) @validate_code_level_metrics("_test_application", "abort_404_page") def test_application_abort_404(): application = target_application() - application.get('/abort_404', status=404) + application.get("/abort_404", status=404) _test_application_exception_404_scoped_metrics = [ - ('Function/flask.app:Flask.wsgi_app', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/_test_application:exception_404_page', 1), - ('Function/flask.app:Flask.handle_http_exception', 1), - ('Function/werkzeug.wsgi:ClosingIterator.close', 1), - ('Function/flask.app:Flask.handle_user_exception', 1)] + ("Function/flask.app:Flask.wsgi_app", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/_test_application:exception_404_page", 1), + ("Function/flask.app:Flask.handle_http_exception", 1), + ("Function/werkzeug.wsgi:ClosingIterator.close", 1), + ("Function/flask.app:Flask.handle_user_exception", 1), +] @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('_test_application:exception_404_page', - scoped_metrics=_test_application_exception_404_scoped_metrics) +@validate_transaction_metrics( + "_test_application:exception_404_page", scoped_metrics=_test_application_exception_404_scoped_metrics +) @validate_code_level_metrics("_test_application", "exception_404_page") def test_application_exception_404(): application = target_application() - application.get('/exception_404', status=404) + application.get("/exception_404", status=404) _test_application_not_found_scoped_metrics = [ - ('Function/flask.app:Flask.wsgi_app', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/flask.app:Flask.handle_http_exception', 1), - ('Function/werkzeug.wsgi:ClosingIterator.close', 1), - ('Function/flask.app:Flask.handle_user_exception', 1)] + ("Function/flask.app:Flask.wsgi_app", 1), + ("Python/WSGI/Application", 1), + ("Python/WSGI/Response", 1), + ("Python/WSGI/Finalize", 1), + ("Function/flask.app:Flask.handle_http_exception", 1), + ("Function/werkzeug.wsgi:ClosingIterator.close", 1), + ("Function/flask.app:Flask.handle_user_exception", 1), +] @validate_transaction_errors(errors=[]) -@validate_transaction_metrics('flask.app:Flask.handle_http_exception', - scoped_metrics=_test_application_not_found_scoped_metrics) +@validate_transaction_metrics( + "flask.app:Flask.handle_http_exception", scoped_metrics=_test_application_not_found_scoped_metrics +) def test_application_not_found(): application = target_application() - application.get('/missing', status=404) + application.get("/missing", status=404) _test_application_render_template_string_scoped_metrics = [ - ('Function/flask.app:Flask.wsgi_app', 1), - ('Python/WSGI/Application', 1), - ('Python/WSGI/Response', 1), - ('Python/WSGI/Finalize', 1), - ('Function/_test_application:template_string', 1), - ('Function/werkzeug.wsgi:ClosingIterator.close', 1), - ('Template/Compile/