diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml index 83d8e54..f982bf0 100644 --- a/.github/workflows/cd.yml +++ b/.github/workflows/cd.yml @@ -1,5 +1,5 @@ --- -## continous deployment workflow +## continuous deployment workflow name: cd on: @@ -17,6 +17,15 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 + fetch-tags: false + ref: ${{ github.sha || 'main' }} + - uses: actions/setup-python@v5 + with: + python-version-file: .python-version + cache: 'pip' + cache-dependency-path: requirements.txt - name: Lint run: make lint build: @@ -57,4 +66,3 @@ jobs: publish: true env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d25ce0..ee94b59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,5 +1,5 @@ --- -## continous integration workflow +## continuous integration workflow name: ci on: @@ -18,6 +18,11 @@ jobs: fetch-depth: 0 fetch-tags: false ref: ${{ github.sha || 'main' }} + - uses: actions/setup-python@v5 + with: + python-version-file: .python-version + cache: 'pip' + cache-dependency-path: requirements.txt - name: Lint run: make lint build: @@ -43,4 +48,3 @@ jobs: with: name: test-results path: "**/junit-*.xml" - diff --git a/.gitignore b/.gitignore index 8d07011..e653045 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,3 @@ -__init__.py .gradle .idea .venv @@ -11,3 +10,8 @@ gha-creds-*.json release.properties target venv +pytest_otel.egg-info +junit-test_*.xml +.mypy_cache +.pytest_cache +tests/tests.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..702fc56 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,50 @@ +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + hooks: + - id: check-case-conflict + - id: check-executables-have-shebangs + exclude: (.+.bat$|.+.ps1$|^target/) + - id: check-json + - id: check-merge-conflict + - id: check-yaml + args: [--allow-multiple-documents] + - id: check-xml + - id: end-of-file-fixer + - id: mixed-line-ending + - id: trailing-whitespace + +- repo: https://github.com/detailyang/pre-commit-shell.git + rev: master + hooks: + - id: shell-lint + name: "Shellscript: lint" + stages: [commit] + types: [shell] + +- repo: https://github.com/adrienverge/yamllint.git + rev: v1.25.0 + hooks: + - id: yamllint + name: "Yaml: lint" + args: ['-c','.yamlint.yml'] + stages: [commit] + types: [yaml] + exclude: (^.pre-commit-config.yaml$|^.pre-commit-hooks.yaml$) + +- repo: https://github.com/elastic/apm-pipeline-library.git + rev: current + hooks: + - id: check-bash-syntax + - id: check-abstract-classes-and-trait # TODO: this hook won't be needed once the CI migration is completed + - id: check-jsonslurper-class # TODO: this hook won't be needed once the CI migration is completed + - id: check-unicode-non-breaking-spaces + - id: remove-unicode-non-breaking-spaces + - id: check-en-dashes + - id: remove-en-dashes + +- repo: https://github.com/codespell-project/codespell + rev: v2.1.0 + hooks: + - id: codespell + args: ['--ignore-words-list','pullrequest,pullrequests'] diff --git a/.yamlint.yml b/.yamlint.yml new file mode 100644 index 0000000..20055ca --- /dev/null +++ b/.yamlint.yml @@ -0,0 +1,10 @@ +extends: default + +rules: + # 120 chars should be enough, but don't fail if a line is longer + line-length: + max: 120 + level: warning + + indentation: + level: warning diff --git a/LICENSE.txt b/LICENSE.txt deleted file mode 100644 index 5cffc6f..0000000 --- a/LICENSE.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [2020] [Elastic B.V.] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/docs/demos/jaeger/config/otel-collector-config.yaml b/docs/demos/jaeger/config/otel-collector-config.yaml index 295acea..6ce3aff 100644 --- a/docs/demos/jaeger/config/otel-collector-config.yaml +++ b/docs/demos/jaeger/config/otel-collector-config.yaml @@ -7,8 +7,8 @@ receivers: grpc: exporters: - logging: - loglevel: debug + debug: + verbosity: detailed jaeger: endpoint: jaeger-all-in-one:14250 tls: diff --git a/docs/demos/jaeger/docker-compose.yml b/docs/demos/jaeger/docker-compose.yml index 74f66ff..9f2839c 100644 --- a/docs/demos/jaeger/docker-compose.yml +++ b/docs/demos/jaeger/docker-compose.yml @@ -4,7 +4,7 @@ version: '3.7' services: jaeger-all-in-one: - image: jaegertracing/all-in-one:latest + image: jaegertracing/all-in-one:1.46.0 ports: - "16686:16686" - "14268" @@ -13,7 +13,7 @@ services: - pytest_otel otel-collector: - image: otel/opentelemetry-collector-contrib-dev:latest + image: otel/opentelemetry-collector-contrib:0.94.0 command: ["--config=/etc/otel-collector-config.yaml", "${OTELCOL_ARGS}"] healthcheck: interval: 10s diff --git a/setup.cfg b/setup.cfg index 361196f..f519fbf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,9 +3,9 @@ name = pytest_otel description = pytest-otel report OpenTelemetry traces about test executed long_description = file: README.md long_description_content_type = text/markdown -url = https://github.com/elastic/apm-pipeline-library/tree/main/resources/scripts/pytest_otel +url = https://github.com/kuisathaverat/pytest_otel maintainer = Ivan Fernandez Calvo -version = 1.4.0 +version = 1.4.1 license = Apache-2.0 license_file = LICENSE.txt platforms = any @@ -27,8 +27,8 @@ classifiers = Topic :: Utilities keywords = pytest, otel, opentelemetry, debug project_urls = - Source=https://github.com/elastic/apm-pipeline-library/tree/main/resources/scripts/pytest_otel - Tracker=https://github.com/elastic/apm-pipeline-library/issues + Source=https://github.com/kuisathaverat/pytest_otel + Tracker=https://github.com/kuisathaverat/pytest_otel/issues [options] packages = find: diff --git a/src/pytest_otel/__init__.py b/src/pytest_otel/__init__.py new file mode 100644 index 0000000..77db6ae --- /dev/null +++ b/src/pytest_otel/__init__.py @@ -0,0 +1,360 @@ +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +import logging +import os +import sys +import traceback + +import _pytest._code +import _pytest.skipping +import pytest +from opentelemetry import trace +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import SERVICE_NAME, Resource +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter +from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator +from opentelemetry.trace.status import Status, StatusCode + +# from opentelemetry import metrics +# from opentelemetry.sdk.metrics.export import ConsoleMetricsExporter +# from opentelemetry.sdk.metrics.export.controller import PushController +# from opentelemetry.ext.otcollector.metrics_exporter import CollectorMetricsExporter +# from opentelemetry.sdk.metrics import Counter, MeterProvider + +LOGGER = logging.getLogger("pytest_otel") +service_name = None +traceparent = None +session_name = None +tracer = None +insecure = None +in_memory_span_exporter = False +otel_span_file_output = None +otel_exporter = None +spans = {} +outcome = None +otel_debug = False +# errors_counter = None +# failed_counter = None +# skipped_counter = None +# total_counter = None +# controller = None + +def pytest_addoption(parser): + """Init command line arguments""" + group = parser.getgroup("pytest-otel", "report OpenTelemetry traces for tests executed.") + + group.addoption( + "--otel-endpoint", + dest="endpoint", + help="URL for the APM server.(OTEL_EXPORTER_OTLP_ENDPOINT)", + ) + group.addoption( + "--otel-headers", + dest="headers", + help="Additional headers to send (i.e.: key1=value1,key2=value2).(OTEL_EXPORTER_OTLP_HEADERS)", # noqa: E501 + ) + group.addoption( + "--otel-service-name", + dest="service_name", + default="Pytest_Otel_reporter", + help="Name of the service.(OTEL_SERVICE_NAME)", + ) + group.addoption( + "--otel-session-name", + dest="session_name", + default="Test Suite", + help="Name for the Main span reported.", + ) + group.addoption( + "--otel-traceparent", + dest="traceparent", + help="Trace parent.(TRACEPARENT) see https://www.w3.org/TR/trace-context-1/#trace-context-http-headers-format", # noqa: E501 + ) + group.addoption( + "--otel-insecure", + dest="insecure", + default=False, + help="Disables TLS.(OTEL_EXPORTER_OTLP_INSECURE)", + ) + group.addoption( + "--otel-span-file-output", + dest="otel_span_file_output", + default="./otel-traces-file-output.json", + help="If the Otel endpoint is not set, the spans will be saved to a file (./otel-traces-file-output.txt)", + ) + group.addoption( + "--otel-debug", + dest="otel_debug", + default=False, + help="", + ) + +def init_otel(): + """Init the OpenTelemetry settings""" + global tracer, session_name, service_name, insecure, otel_exporter, errors_counter, failed_counter, skipped_counter, total_counter, controller # noqa: E501 + LOGGER.debug("Init Otel : {}".format(service_name)) + trace.set_tracer_provider( + TracerProvider( + resource=Resource.create({SERVICE_NAME: service_name}), + ) + ) + + if in_memory_span_exporter: + otel_exporter = InMemorySpanExporter() + trace.get_tracer_provider().add_span_processor(SimpleSpanProcessor(otel_exporter)) + # metrics_exporter = ConsoleMetricsExporter() + else: + otel_exporter = OTLPSpanExporter() + trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(otel_exporter)) + # metrics_exporter = CollectorMetricsExporter() + + tracer = trace.get_tracer(session_name) + + # metrics.set_meter_provider(MeterProvider()) + # meter = metrics.get_meter(session_name, True) + # controller = PushController(meter, metrics_exporter, 5) + # + # errors_counter = meter.create_counter( + # name="tests.error", + # description="number of error tests", + # unit="1", + # value_type=int + # ) + # failed_counter = meter.create_counter( + # name="tests.failed", + # description="number of failed tests", + # unit="1", + # value_type=int + # ) + # skipped_counter = meter.create_counter( + # name="tests.skipped", + # description="number of skipped tests", + # unit="1", + # value_type=int + # ) + # total_counter = meter.create_counter( + # name="tests.total", + # description="total number of tests", + # unit="1", + # value_type=int + # ) + + +def start_span(span_name, context=None, kind=None): + """Starts a span with the name, context, and kind passed as parameters""" + global tracer, spans + spans[span_name] = tracer.start_span( + span_name, context=context, record_exception=True, set_status_on_exception=True, + kind=kind + ) + LOGGER.debug("The {} transaction start_span.".format(span_name)) + return spans[span_name] + + +def end_span(span_name, outcome): + """Ends a span identified by its name""" + global spans + status = convertOutcome(outcome) + spans[span_name].set_status(status) + spans[span_name].set_attribute("tests.status", outcome) + spans[span_name].end() + LOGGER.debug("The {} transaction ends. -> {}".format(span_name, status)) + return spans[span_name] + + +def convertOutcome(outcome): + """Convert from pytest outcome to OpenTelemetry status code""" + if outcome == "passed": + return Status(status_code=StatusCode.OK) + elif (outcome == "failed" + or outcome == "interrupted" + or outcome == "internal_error" + or outcome == "usage_error" + or outcome == "no_tests_collected" + ): + return Status(status_code=StatusCode.ERROR) + else: + return Status(status_code=StatusCode.UNSET) + +# def update_metrics(outcome): +# """Update the metrics with the test result""" +# if (outcome == "interrupted" +# or outcome == "internal_error" +# or outcome == "usage_error" +# or outcome == "no_tests_collected" +# ): +# errors_counter.add(1) +# elif (outcome == "failed"): +# failed_counter.add(1) +# elif (outcome == "skipped"): +# skipped_counter.add(1) + + +def exitCodeToOutcome(exit_code): + """convert pytest ExitCode to outcome""" + if exit_code == 0: + return "passed" + elif exit_code == 1: + return "failed" + elif exit_code == 2: + return "interrupted" + elif exit_code == 3: + return "internal_error" + elif exit_code == 4: + return "usage_error" + elif exit_code == 4: + return "no_tests_collected" + else: + return "failed" + +def traceparent_context(traceparent): + """Extracts the trace context from the TRACEPARENT passed""" + carrier = {} + carrier["traceparent"] = traceparent + return TraceContextTextMapPropagator().extract(carrier=carrier) + + +def pytest_sessionstart(session): + """Uses the commandline parameter to define the environment variables used by OpenTelemetry""" + global service_name, traceparent, session_name, insecure, in_memory_span_exporter, otel_span_file_output, otel_debug + config = session.config + if config.getoption("otel_debug"): + LOGGER.setLevel(logging.DEBUG) + otel_debug = True + service_name = config.getoption("service_name") + session_name = config.getoption("session_name") + traceparent = config.getoption("traceparent") + endpoint = config.getoption("endpoint") + headers = config.getoption("headers") + insecure = config.getoption("insecure") + if endpoint is not None: + os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint + if headers is not None: + os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = headers + if service_name is not None: + os.environ["OTEL_SERVICE_NAME"] = service_name + if insecure: + os.environ["OTEL_EXPORTER_OTLP_INSECURE"] = f'{insecure}' + if traceparent is None: + traceparent = os.getenv("TRACEPARENT", None) + if len(os.getenv("OTEL_EXPORTER_OTLP_ENDPOINT", "")) == 0: + in_memory_span_exporter = True + otel_span_file_output = config.getoption("otel_span_file_output") + init_otel() + span = start_span(session_name, traceparent_context(traceparent), trace.SpanKind.SERVER) + + +def pytest_runtest_setup(item): # noqa: U100 + """Clean the global outcome on every test""" + global outcome + outcome = None + + +def pytest_report_teststatus(report): + """Set the final outcome to the reported outcome""" + global outcome + outcome = report.outcome + + +def pytest_sessionfinish(session, exitstatus): # noqa: U100 + """Ends the parent Opentelemetry span with the session outcome""" + global session_name, outcome, in_memory_span_exporter, otel_exporter + LOGGER.debug("Session transaction Ends") + end_span(session_name, exitCodeToOutcome(exitstatus)) + LOGGER.debug("in_memory_span_exporter {}".format(in_memory_span_exporter)) + if in_memory_span_exporter: + print() + print("Using on memory OpenTelemetry exporter") + span_list = otel_exporter.get_finished_spans() + print("Number of spans: {}".format(len(span_list))) + if otel_debug: + json = "[\n" + for i in range(len(span_list)): + if i > 0: + json += "," + json += span_list[i].to_json() + json += "\n]\n" + with open(otel_span_file_output, 'w', encoding='utf-8') as output: + output.write(json) + print(json) + + +@pytest.hookimpl(hookwrapper=True) +def pytest_runtest_call(item): + global outcome, session_name, spans + with tracer.start_as_current_span( + "Running {}".format(item.name), + context=trace.set_span_in_context(spans[session_name]), + record_exception=True, + set_status_on_exception=True, + ) as span: + #total_counter.add(1) + LOGGER.debug("Test {} starts - {}".format(item.name, span.get_span_context())) + span.set_attribute("tests.name", item.name) + info = yield + LOGGER.debug("Test {} ends - {}".format(item.name, span.get_span_context())) + + if hasattr(info, "_excinfo"): + if info._excinfo: + (info_class, info_msg, info_trace) = info._excinfo + if info_class.__name__ == 'Failed': + outcome = "failed" + span.set_attribute("tests.message", "{}".format(info_msg)) + if hasattr(sys, "last_value") and hasattr(sys, "last_traceback") and hasattr(sys, "last_type"): + longrepr = "" + last_value = getattr(sys, 'last_value') + last_traceback = getattr(sys, 'last_traceback') + last_type = getattr(sys, 'last_type') + + if not isinstance(last_value, _pytest._code.ExceptionInfo): + outcome = "failed" + longrepr = last_value + elif isinstance(last_value, _pytest._code.skip.Exception): + outcome = "skipped" + r = last_value._getreprcrash() + longrepr = (str(r.path), r.lineno, r.message) + else: + outcome = "failed" + style = item.config.getoption("tbstyle", "auto") + longrepr = item._repr_failure_py(last_value, style=style) + + stack_trace = repr(traceback.format_exception(last_type, last_value, last_traceback)) + span.set_attribute("tests.error", "{}".format(stack_trace)) + if hasattr(last_value, "args") and len(getattr(last_value, 'args', [])) > 0: + span.set_attribute("tests.message", "{}" + .format(last_value.args[0])) + + if longrepr: + span.set_attribute("tests.message", "{}".format(longrepr)) + elif last_value: + span.set_attribute("tests.message", "{}".format(last_value)) + elif last_type: + span.set_attribute("tests.message", "{}".format(last_type)) + + skipping = getattr(_pytest, 'skipping', None) + if skipping: + key = getattr(skipping, 'xfailed_key', None) + xfailed = item._store.get(key, None) + reason = getattr(xfailed, 'reason', None) + if reason : + span.set_attribute("tests.message", "{}".format(reason)) + + #update_metrics(outcome) + status = convertOutcome(outcome) + span.set_status(status) + span.set_attribute("tests.status", "{}".format(outcome)) + + +@pytest.hookimpl() +def pytest_runtest_logreport(report): + global session_name, spans + test_name = report.nodeid.split("::")[0] + + if report.failed and report.when == "teardown": + span = spans[test_name] + span.set_attribute("tests.systemerr", report.capstderr) + span.set_attribute("tests.systemout", report.capstdout) + span.set_attribute("tests.duration", getattr(report, "duration", 0.0)) diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml index 3c10999..d932d4e 100644 --- a/tests/docker-compose.yml +++ b/tests/docker-compose.yml @@ -4,7 +4,7 @@ version: "3.7" services: otel-collector: - image: otel/opentelemetry-collector:latest + image: otel/opentelemetry-collector:0.94.0 user: ${UID}:${GID} command: ["--config=/etc/otel-collector.yaml"] volumes: diff --git a/tests/it/utils/__init__.py b/tests/it/utils/__init__.py new file mode 100644 index 0000000..ed1f4de --- /dev/null +++ b/tests/it/utils/__init__.py @@ -0,0 +1,97 @@ +import pytest +import json +import time +import os +import socket +import subprocess + +SPAN_KIND_INTERNAL = 1 +SPAN_KIND_SERVER = 2 + +STATUS_CODE_OK = 1 +STATUS_CODE_ERROR = 2 + + +def is_portListening(host, port): + """Check a port in a host is liostening""" + a_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + location = (host, port) + result_of_check = a_socket.connect_ex(location) + if result_of_check == 0: + return True + else: + return False + + +def getSize(filename): + """return the size of a file""" + if os.path.isfile(filename): + st = os.stat(filename) + return st.st_size + else: + return 0 + + +def waitForFileContent(filename): + """wait for a file has content""" + while getSize(filename) < 1: + time.sleep(5) + subprocess.check_output(f"docker cp $(docker ps -q --filter expose=4317):/tmp/tests.json {filename}", + stderr=subprocess.STDOUT, shell=True) + with open(filename, encoding='utf-8') as input: + print(input.read()) + + +def assertAttrKeyValue(attributes, key, value): + """check the value of a key in attributes""" + realValue = '' + for attr in attributes: + if attr['key'] == key: + realValue = attr["value"]["stringValue"] + assert realValue == value, f'attribute {key} is not {value}: {realValue}' + + +def assertTestSuit(span, outcome, status): + """check attributes of a test suit span""" + assert span["kind"] == SPAN_KIND_SERVER, f'span kind is not server: {span["kind"]}' + assert span["status"]["code"] == status, f'status code is not {status}: {span["status"]["code"]}' + if outcome is not None: + assertAttrKeyValue(span["attributes"], 'tests.status', outcome) + assert len(span["parentSpanId"]) == 0, f'parent span id is not empty: {span["parentSpanId"]}' + return True + + +def assertSpan(span, name, outcome, status): + """check attributes of a span""" + assert span["kind"] == SPAN_KIND_INTERNAL, f'span kind is not internal: {span["kind"]}' + assert span["status"]["code"] == status, f'status code is not {status}: {span["status"]["code"]}' + assertAttrKeyValue(span["attributes"], 'tests.name', name) + if outcome is not None: + assertAttrKeyValue(span["attributes"], 'tests.status', outcome) + assert len(span["parentSpanId"]) > 0, f'parent span id is empty: {span["parentSpanId"]}' + return True + + +def assertTest(pytester, name, ts_outcome, ts_status, outcome, status): + """check a test results are correct""" + pytester.runpytest("--otel-endpoint=http://127.0.0.1:4317", "--otel-service-name=pytest_otel", "--otel-debug=True", "-rsx") + filename = "./tests.json" + waitForFileContent(filename) + foundTest = False + foundTestSuit = False + with open(filename, encoding='utf-8') as input: + spans_output = json.loads(input.readline()) + print(f""" + spans_output {spans_output} + resourceSpans {spans_output['resourceSpans']} + """) + for resourceSpan in spans_output['resourceSpans']: + for instrumentationLibrarySpan in resourceSpan['scopeSpans']: + for span in instrumentationLibrarySpan['spans']: + if span["name"] == f"Running {name}": + foundTest = assertSpan(span, name, outcome, status) + if span["name"] == "Test Suite": + foundTestSuit = assertTestSuit(span, ts_outcome, ts_status) + assert foundTest or name is None, f'test {name} not found' + assert foundTestSuit, 'test suit not found' + os.remove(filename) diff --git a/tests/otel-collector.yaml b/tests/otel-collector.yaml index 413c5b7..8cdd89f 100644 --- a/tests/otel-collector.yaml +++ b/tests/otel-collector.yaml @@ -8,8 +8,8 @@ receivers: endpoint: 0.0.0.0:4317 exporters: - logging: - logLevel: debug + debug: + verbosity: detailed file: path: /tmp/tests.json processors: @@ -23,5 +23,5 @@ service: processors: - batch exporters: - - logging + - debug - file